chg: [relationships] add messages mentions -> chats + user-accounts

bin/importer/feeders/abstract_chats_feeder.py

@@ -483,4 +483,41 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
                             obj.add_relationship(chat_obj.get_global_id(), 'in')
         # -FORWARD- #
 
+        ## MENTION ##
+        if self.get_json_meta().get('mentions'):
+            for mention in self.get_json_meta()['mentions'].get('chats', []):
+                m_obj = self._process_chat(mention, date, new_objs=new_objs)
+                if m_obj:
+                    for chat_obj in chats_objs:
+                        if chat_obj.type == 'chat':
+                            chat_obj.add_relationship(m_obj.get_global_id(), 'mention')
+
+                    # TODO PERF
+                    # TODO Keep message obj + chat obj in global var
+                    for obj in objs:
+                        if obj.type == 'message':
+                            obj.add_relationship(m_obj.get_global_id(), 'mention')
+                            for chat_obj in chats_objs:
+                                if chat_obj.type == 'chat':
+                                    obj.add_relationship(chat_obj.get_global_id(), 'in')
+
+            for mention in self.get_json_meta()['mentions'].get('users', []):
+                m_obj = self._process_user(mention, date, timestamp, new_objs=new_objs)  # TODO date, timestamp ???
+                if m_obj:
+                    for chat_obj in chats_objs:
+                        if chat_obj.type == 'chat':
+                            chat_obj.add_relationship(m_obj.get_global_id(), 'mention')
+
+                    # TODO PERF
+                    # TODO Keep message obj + chat obj in global var
+                    for obj in objs:
+                        if obj.type == 'message':
+                            obj.add_relationship(m_obj.get_global_id(), 'mention')
+                            for chat_obj in chats_objs:
+                                if chat_obj.type == 'chat':
+                                    obj.add_relationship(chat_obj.get_global_id(), 'in')
+
+
+         # -MENTION- #
+
         return new_objs | objs

bin/lib/chats_viewer.py

@@ -490,6 +490,35 @@ def get_user_account_chats_chord(subtype, user_id):
         chord['data'].append({'source': user_account_gid, 'target': chat_g_id, 'value': nb[chat_g_id]})
     return chord
 
+def get_user_account_mentions_chord(subtype, user_id):
+    chord = {'meta': {}, 'data': []}
+    nb = {}
+    user_account = UsersAccount.UserAccount(user_id, subtype)
+    user_account_gid = user_account.get_global_id()
+    label = get_chat_user_account_label(user_account_gid)
+    if label:
+        chord['meta'][user_account_gid] = label
+    else:
+        chord['meta'][user_account_gid] = user_account_gid
+
+    for mess in user_account.get_messages():
+        m = Messages.Message(mess[9:])
+        for rel in m.get_obj_relationships(relationships={'mention'}, filter_types={'chat', 'user_account'}):
+            if rel:
+                if not rel['target'] in nb:
+                    nb[rel['target']] = 0
+                nb[rel['target']] += 1
+
+    for g_id in nb:
+        label = get_chat_user_account_label(g_id)
+        if label:
+            chord['meta'][g_id] = label
+        else:
+            chord['meta'][g_id] = g_id
+        chord['data'].append({'source': user_account_gid, 'target': g_id, 'value': nb[g_id]})
+    return chord
+
+
 def _get_chat_card_meta_options():
     return {'created_at', 'icon', 'info', 'nb_participants', 'origin_link', 'subchannels', 'tags_safe', 'threads', 'translation', 'username'}
 

bin/lib/objects/UsersAccount.py

@@ -139,11 +139,12 @@ class UserAccount(AbstractSubtypeObject):
 
     def get_messages(self):
         messages = []
-        for mess in self.get_correlation('message'):
-            messages.append(f'message:{mess}')
+        correl = self.get_correlation('message')
+        if 'message' in correl:
+            for mess_id in correl['message']:
+                messages.append(f'message:{mess_id}')
         return messages
 
-
     def get_messages_by_chat_obj(self, chat_obj):
         messages = []
         for mess in self.get_correlation_iter_obj(chat_obj, 'message'):

bin/lib/objects/abstract_object.py

@@ -24,7 +24,7 @@ from lib.ConfigLoader import ConfigLoader
 from lib import Duplicate
 from lib.correlations_engine import get_nb_correlations, get_correlations, add_obj_correlation, delete_obj_correlation, delete_obj_correlations, exists_obj_correlation, is_obj_correlated, get_nb_correlation_by_correl_type, get_obj_inter_correlation
 from lib.Investigations import is_object_investigated, get_obj_investigations, delete_obj_investigations
-from lib.relationships_engine import get_obj_nb_relationships, add_obj_relationship
+from lib.relationships_engine import get_obj_nb_relationships, get_obj_relationships, add_obj_relationship
 from lib.Language import get_obj_languages, add_obj_language, remove_obj_language, detect_obj_language, get_obj_language_stats, get_obj_translation, set_obj_translation, delete_obj_translation, get_obj_main_language
 from lib.Tracker import is_obj_tracked, get_obj_trackers, delete_obj_trackers
 
@@ -299,6 +299,9 @@ class AbstractObject(ABC):
     def get_nb_relationships(self, filter=[]):
         return get_obj_nb_relationships(self.get_global_id())
 
+    def get_obj_relationships(self, relationships=set(), filter_types=set()):
+        return get_obj_relationships(self.get_global_id(), relationships=relationships, filter_types=filter_types)
+
     def add_relationship(self, obj2_global_id, relationship, source=True):
         # is source
         if source:

bin/lib/objects/ail_objects.py

@@ -594,6 +594,13 @@ def get_chat_relationships_cord_graph(obj_type, subtype, obj_id):
         return data
     return []
 
+def get_chat_relationships_mentions_cord_graph(obj_type, subtype, obj_id):
+    if obj_type == 'chat':
+        obj_global_id = get_obj_global_id(obj_type, subtype, obj_id)
+        data = relationships_engine.get_chat_mentions_stats(obj_global_id)
+        return data
+    return []
+
 # --- RELATIONSHIPS --- #
 
 

bin/lib/relationships_engine.py

@@ -22,7 +22,7 @@ RELATIONSHIPS = {
     "mention"
 }
 
-RELATIONSHIPS_OBJS = {
+RELATIONSHIPS_OBJS = { # TODO forward user-account
     "forwarded_from": {
         'chat': {'message'},
         'message': {'chat', 'user-account'}
@@ -35,7 +35,11 @@ RELATIONSHIPS_OBJS = {
         'chat': {'message'},
         'message': {'chat'}
     },
-    "mention": {}
+    "mention": {
+        'chat': {'chat', 'user-account', 'message'},
+        'message': {'chat', 'user-account'},
+        'user-account': {'chat', 'message'},
+    },
 }
 
 def get_relationships():
@@ -221,3 +225,55 @@ def get_chat_forward_stats(obj_global_id): # objs_hidden
 
     return data
 
+##################################################################
+
+def get_chat_mention_stats_in(obj_global_id):
+    nb = {}
+    for rel in get_obj_relationships(obj_global_id, relationships={'mention'}, filter_types={'message'}):
+        chat_mess = rel['source'].split('/')
+        chat_source = f'chat:{chat_mess[0][9:]}:{chat_mess[2]}'
+        if chat_source not in nb:
+            nb[chat_source] = 0
+        nb[chat_source] += 1
+    return nb
+
+def get_chat_mention_stats_out(obj_global_id):
+    nb = {}
+    for rel in get_obj_relationships(obj_global_id, relationships={'in'}, filter_types={'message'}):
+        r = get_obj_relationships(rel['source'], relationships={'mention'}, filter_types={'chat', 'user-account'})
+        if r:
+            if not r[0]['target'] in nb:
+                nb[r[0]['target']] = 0
+            nb[r[0]['target']] += 1
+    #     chat_mess = rel['source'].split('/')
+    #
+    #     chat_target = f'chat:{chat_mess[0][9:]}:{chat_mess[2]}'
+    #     print(chat_target, chat, chat_target == chat)
+    #     if chat is None or chat_target == chat:
+    #         if chat_target not in nb:
+    #             nb[chat_target] = 0
+    #         nb[chat_target] += 1
+    #
+    # print(json.dumps(nb, indent=4))
+    return nb
+
+def get_chat_mentions_stats(obj_global_id): # objs_hidden
+    data = []
+
+    # print(get_obj_relationships(obj_global_id, relationships=['forwarded_from'], filter_types=['message']))
+
+    out_mess = get_chat_mention_stats_in(obj_global_id)
+    in_mess = get_chat_mention_stats_out(obj_global_id)
+    #
+    for target in in_mess:
+        data.append({'source': obj_global_id, 'target': target, 'value': in_mess[target]})
+
+    for source in out_mess:
+        data.append({'source': source, 'target': obj_global_id, 'value': out_mess[source]})
+
+    # print()
+    # print(in_mess)
+    # print()
+    # print(out_mess)
+
+    return data

var/www/blueprints/chats_explorer.py

@@ -306,6 +306,15 @@ def objects_user_account_chats_chord_json():
     json_graph = chats_viewer.get_user_account_chats_chord(subtype, user_id)
     return jsonify(json_graph)
 
+@chats_explorer.route("/objects/user-account_mentions_chord_json", methods=['GET']) # TODO API
+@login_required
+@login_read_only
+def objects_user_account_mentions_chord_json():
+    subtype = request.args.get('subtype')
+    user_id = request.args.get('id')
+    json_graph = chats_viewer.get_user_account_mentions_chord(subtype, user_id)
+    return jsonify(json_graph)
+
 @chats_explorer.route("/objects/user-account/chat", methods=['GET'])
 @login_required
 @login_read_only

var/www/blueprints/correlation.py

@@ -283,6 +283,19 @@ def relationships_chord_graph_json():
 
     return jsonify({'meta': meta, 'data': chat_json_graph})
 
+@correlation.route('/relationships/chord_mentions_graph_json')
+@login_required
+@login_read_only
+def relationships_chord_mentions_graph_json():
+    obj_id = request.args.get('id')
+    subtype = request.args.get('subtype')
+    obj_type = request.args.get('type')
+
+    chat_json_graph = ail_objects.get_chat_relationships_mentions_cord_graph(obj_type, subtype, obj_id)
+    meta = chats_viewer.enrich_chat_relationships_labels(chat_json_graph)
+
+    return jsonify({'meta': meta, 'data': chat_json_graph})
+
 @correlation.route('/relationship/show', methods=['GET', 'POST'])
 @login_required
 @login_read_only

var/www/templates/chats_explorer/user_account.html

@@ -50,6 +50,9 @@
                     <h4>Numbers of Messages Posted by Chat:</h4>
                     <div id="chord_user_chats" style="max-width: 900px"></div>
 
+                    <h4>Numbers of Mentions:</h4>
+                    <div id="chord_mentions" style="max-width: 900px"></div>
+
                     <h4>User Chats:</h4>
                     {% for meta_chats in meta['chats'] %}
                         <div class="my-2">
@@ -92,6 +95,10 @@ d3.json(url).then(function(data) {
     create_directed_chord_diagram('#chord_user_chats', data, mouseover_tooltip_ail_obj, mouseout_tooltip_ail_obj);
 });
 
+let url2 = "{{ url_for('chats_explorer.objects_user_account_mentions_chord_json') }}?subtype={{ meta["subtype"] }}&id={{ meta["id"] }}"
+d3.json(url2).then(function(data) {
+    create_directed_chord_diagram('#chord_mentions', data, mouseover_tooltip_ail_obj, mouseout_tooltip_ail_obj);
+});
 
 </script>
 

var/www/templates/correlation/show_relationship.html

@@ -248,6 +248,10 @@
                                                     <input class="form-check-input" type="checkbox" value="True" id="message_Check" name="message_Check" {%if "message" in dict_object["filter"]%}checked{%endif%}>
                                                     <label class="form-check-label" for="message_Check">Message</label>
                                                 </div>
+                                                <div class="form-check">
+                                                    <input class="form-check-input" type="checkbox" value="True" id="user-account_Check" name="user-account_Check" {%if "user-account" in dict_object["filter"]%}checked{%endif%}>
+                                                    <label class="form-check-label" for="user-account_Check">User-Account</label>
+                                                </div>
                                             </li>
 
 {#                                        <li class="list-group-item text-left">#}
@@ -351,8 +355,12 @@
                         </div>
                     </div>
 
+                    <h3>Forwards</h3>
                     <div id="chart_test" style="max-width: 900px"></div>
 
+                    <h3>Mentions</h3>
+                    <div id="chart_mentions" style="max-width: 900px"></div>
+
 
                     <table id="table_graph_node_objects">
                     </table>
@@ -381,6 +389,11 @@ $(document).ready(function(){
             create_directed_chord_diagram('#chart_test', data, mouseover_tooltip_ail_obj, mouseout_tooltip_ail_obj);
         });
 
+        let url2 = "{{ url_for('correlation.relationships_chord_mentions_graph_json') }}?id={{ dict_object["correlation_id"] }}&type={{ dict_object["object_type"] }}{% if 'type_id' in dict_object["metadata"] %}&subtype={{ dict_object["metadata"]["type_id"] }}{% endif %}"
+        d3.json(url2).then(function(data) {
+            create_directed_chord_diagram('#chart_mentions', data, mouseover_tooltip_ail_obj, mouseout_tooltip_ail_obj);
+        });
+
 
 });
 
@@ -808,6 +821,10 @@ window.all_graph = all_graph;
 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 
 
+/*
+*/
+
+
 
 </script>