mirror of https://github.com/CIRCL/AIL-framework
chg: [relationships] messages foraward between chats/user-account + chat chord diagram
parent
70db33caaf
commit
c1a2bc7eb8
|
@ -128,7 +128,7 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
|
||||||
return self.json_data['meta'].get('reply_to', {}).get('message_id')
|
return self.json_data['meta'].get('reply_to', {}).get('message_id')
|
||||||
|
|
||||||
def get_message_forward(self):
|
def get_message_forward(self):
|
||||||
return self.json_data['meta'].get('forward')
|
return self.json_data['meta'].get('forward', {})
|
||||||
|
|
||||||
def get_message_content(self):
|
def get_message_content(self):
|
||||||
decoded = base64.standard_b64decode(self.json_data['data'])
|
decoded = base64.standard_b64decode(self.json_data['data'])
|
||||||
|
@ -291,6 +291,39 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
|
||||||
# else:
|
# else:
|
||||||
# # ADD NEW MESSAGE REF (used by discord)
|
# # ADD NEW MESSAGE REF (used by discord)
|
||||||
|
|
||||||
|
##########################################################################################
|
||||||
|
|
||||||
|
def _process_user(self, meta, date, timestamp, new_objs=None): #################33 timestamp
|
||||||
|
user_account = UsersAccount.UserAccount(meta['id'], self.get_chat_instance_uuid())
|
||||||
|
if meta.get('username'):
|
||||||
|
username = Username(meta['username'], self.get_chat_protocol())
|
||||||
|
# TODO timeline or/and correlation ????
|
||||||
|
user_account.add_correlation(username.type, username.get_subtype(r_str=True), username.id)
|
||||||
|
user_account.update_username_timeline(username.get_global_id(), timestamp) # TODO time.time !!!! (time when meta are retrieved)
|
||||||
|
|
||||||
|
# Username---Message
|
||||||
|
username.add(date) # TODO # correlation message ??? ###############################################################
|
||||||
|
|
||||||
|
# ADDITIONAL METAS
|
||||||
|
if meta.get('firstname'):
|
||||||
|
user_account.set_first_name(meta['firstname'])
|
||||||
|
if meta.get('lastname'):
|
||||||
|
user_account.set_last_name(meta['lastname'])
|
||||||
|
if meta.get('phone'):
|
||||||
|
user_account.set_phone(meta['phone'])
|
||||||
|
|
||||||
|
if meta.get('icon'):
|
||||||
|
img = Images.create(meta['icon'], b64=True)
|
||||||
|
img.add(date, user_account)
|
||||||
|
user_account.set_icon(img.get_global_id())
|
||||||
|
new_objs.add(img)
|
||||||
|
|
||||||
|
if meta.get('info'):
|
||||||
|
user_account.set_info(meta['info'])
|
||||||
|
|
||||||
|
user_account.add(date)
|
||||||
|
return user_account
|
||||||
|
|
||||||
def process_sender(self, new_objs, obj, date, timestamp):
|
def process_sender(self, new_objs, obj, date, timestamp):
|
||||||
meta = self.json_data['meta'].get('sender')
|
meta = self.json_data['meta'].get('sender')
|
||||||
if not meta:
|
if not meta:
|
||||||
|
@ -339,6 +372,7 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
|
||||||
if self.obj:
|
if self.obj:
|
||||||
objs.add(self.obj)
|
objs.add(self.obj)
|
||||||
new_objs = set()
|
new_objs = set()
|
||||||
|
chats_objs = set()
|
||||||
|
|
||||||
date, timestamp = self.get_message_date_timestamp()
|
date, timestamp = self.get_message_date_timestamp()
|
||||||
|
|
||||||
|
@ -397,14 +431,14 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
|
||||||
print(obj.id)
|
print(obj.id)
|
||||||
|
|
||||||
# CHAT
|
# CHAT
|
||||||
chat_objs = self.process_chat(new_objs, obj, date, timestamp, reply_id=reply_id)
|
curr_chats_objs = self.process_chat(new_objs, obj, date, timestamp, reply_id=reply_id)
|
||||||
|
|
||||||
# SENDER # TODO HANDLE NULL SENDER
|
# SENDER # TODO HANDLE NULL SENDER
|
||||||
user_account = self.process_sender(new_objs, obj, date, timestamp)
|
user_account = self.process_sender(new_objs, obj, date, timestamp)
|
||||||
|
|
||||||
if user_account:
|
if user_account:
|
||||||
# UserAccount---ChatObjects
|
# UserAccount---ChatObjects
|
||||||
for obj_chat in chat_objs:
|
for obj_chat in curr_chats_objs:
|
||||||
user_account.add_correlation(obj_chat.type, obj_chat.get_subtype(r_str=True), obj_chat.id)
|
user_account.add_correlation(obj_chat.type, obj_chat.get_subtype(r_str=True), obj_chat.id)
|
||||||
|
|
||||||
# if chat: # TODO Chat---Username correlation ???
|
# if chat: # TODO Chat---Username correlation ???
|
||||||
|
@ -416,24 +450,37 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
|
||||||
# -> subchannel ?
|
# -> subchannel ?
|
||||||
# -> thread id ?
|
# -> thread id ?
|
||||||
|
|
||||||
|
chats_objs.update(curr_chats_objs)
|
||||||
|
|
||||||
#######################################################################
|
#######################################################################
|
||||||
|
|
||||||
## FORWARD ##
|
## FORWARD ##
|
||||||
# chat_fwd = None
|
chat_fwd = None
|
||||||
# if self.get_json_meta().get('forward'):
|
user_fwd = None
|
||||||
# meta_fwd = self.get_message_forward()
|
if self.get_json_meta().get('forward'):
|
||||||
# if meta_fwd['chat']:
|
meta_fwd = self.get_message_forward()
|
||||||
# chat_fwd = self._process_chat(meta_fwd['chat'], date, new_objs=new_objs)
|
if meta_fwd.get('chat'):
|
||||||
# for chat_obj in chat_objs:
|
chat_fwd = self._process_chat(meta_fwd['chat'], date, new_objs=new_objs)
|
||||||
# if chat_obj.type == 'chat':
|
for chat_obj in chats_objs:
|
||||||
# chat_fwd.add_relationship(chat_obj.get_global_id(), 'forward')
|
if chat_obj.type == 'chat':
|
||||||
#
|
chat_fwd.add_relationship(chat_obj.get_global_id(), 'forwarded_to')
|
||||||
# # TODO chat_fwd -> message
|
if meta_fwd.get('user'):
|
||||||
# if chat_fwd:
|
user_fwd = self._process_user(meta_fwd['user'], date, timestamp, new_objs=new_objs) # TODO date, timestamp ???
|
||||||
# for obj in objs:
|
for chat_obj in chats_objs:
|
||||||
# if obj.type == 'message':
|
if chat_obj.type == 'chat':
|
||||||
# chat_fwd.add_relationship(obj.get_global_id(), 'forward')
|
user_fwd.add_relationship(chat_obj.get_global_id(), 'forwarded_to')
|
||||||
|
|
||||||
|
# TODO chat_fwd -> message
|
||||||
|
if chat_fwd or user_fwd:
|
||||||
|
for obj in objs:
|
||||||
|
if obj.type == 'message':
|
||||||
|
if chat_fwd:
|
||||||
|
obj.add_relationship(chat_fwd.get_global_id(), 'forwarded_from')
|
||||||
|
if user_fwd:
|
||||||
|
obj.add_relationship(user_fwd.get_global_id(), 'forwarded_from')
|
||||||
|
for chat_obj in chats_objs:
|
||||||
|
if chat_obj.type == 'chat':
|
||||||
|
obj.add_relationship(chat_obj.get_global_id(), 'in')
|
||||||
# -FORWARD- #
|
# -FORWARD- #
|
||||||
|
|
||||||
return new_objs | objs
|
return new_objs | objs
|
||||||
|
|
|
@ -515,6 +515,53 @@ def fix_correlations_subchannel_message():
|
||||||
|
|
||||||
#### API ####
|
#### API ####
|
||||||
|
|
||||||
|
def get_chat_user_account_label(chat_gid):
|
||||||
|
label = None
|
||||||
|
obj_type, subtype, obj_id = chat_gid.split(':', 2)
|
||||||
|
if obj_type == 'chat':
|
||||||
|
obj = get_obj_chat(obj_type, subtype, obj_id)
|
||||||
|
username = obj.get_username()
|
||||||
|
if username:
|
||||||
|
username = username.split(':', 2)[2]
|
||||||
|
name = obj.get_name()
|
||||||
|
if username and name:
|
||||||
|
label = f'{username} - {name}'
|
||||||
|
elif username:
|
||||||
|
label = username
|
||||||
|
elif name:
|
||||||
|
label = name
|
||||||
|
|
||||||
|
elif obj_type == 'user-account':
|
||||||
|
obj = UsersAccount.UserAccount(obj_id, subtype)
|
||||||
|
username = obj.get_username()
|
||||||
|
if username:
|
||||||
|
username = username.split(':', 2)[2]
|
||||||
|
name = obj.get_name()
|
||||||
|
if username and name:
|
||||||
|
label = f'{username} - {name}'
|
||||||
|
elif username:
|
||||||
|
label = username
|
||||||
|
elif name:
|
||||||
|
label = name
|
||||||
|
return label
|
||||||
|
|
||||||
|
def enrich_chat_relationships_labels(relationships):
|
||||||
|
meta = {}
|
||||||
|
for row in relationships:
|
||||||
|
if row['source'] not in meta:
|
||||||
|
label = get_chat_user_account_label(row['source'])
|
||||||
|
if label:
|
||||||
|
meta[row['source']] = label
|
||||||
|
else:
|
||||||
|
meta[row['source']] = row['source']
|
||||||
|
|
||||||
|
if row['target'] not in meta:
|
||||||
|
label = get_chat_user_account_label(row['target'])
|
||||||
|
if label:
|
||||||
|
meta[row['target']] = label
|
||||||
|
else:
|
||||||
|
meta[row['target']] = row['target']
|
||||||
|
return meta
|
||||||
def api_get_chat_service_instance(chat_instance_uuid):
|
def api_get_chat_service_instance(chat_instance_uuid):
|
||||||
chat_instance = ChatServiceInstance(chat_instance_uuid)
|
chat_instance = ChatServiceInstance(chat_instance_uuid)
|
||||||
if not chat_instance.exists():
|
if not chat_instance.exists():
|
||||||
|
|
|
@ -63,6 +63,16 @@ class UserAccount(AbstractSubtypeObject):
|
||||||
def get_last_name(self):
|
def get_last_name(self):
|
||||||
return self._get_field('lastname')
|
return self._get_field('lastname')
|
||||||
|
|
||||||
|
def get_name(self):
|
||||||
|
first_name = self.get_first_name()
|
||||||
|
last_name = self.get_last_name()
|
||||||
|
if first_name and last_name:
|
||||||
|
return f'{first_name} {last_name}'
|
||||||
|
elif first_name:
|
||||||
|
return first_name
|
||||||
|
elif last_name:
|
||||||
|
return last_name
|
||||||
|
|
||||||
def get_phone(self):
|
def get_phone(self):
|
||||||
return self._get_field('phone')
|
return self._get_field('phone')
|
||||||
|
|
||||||
|
|
|
@ -587,6 +587,12 @@ def get_relationships_graph_node(obj_type, subtype, obj_id, relationships=[], fi
|
||||||
"links": links,
|
"links": links,
|
||||||
"meta": meta}
|
"meta": meta}
|
||||||
|
|
||||||
|
def get_chat_relationships_cord_graph(obj_type, subtype, obj_id):
|
||||||
|
if obj_type == 'chat':
|
||||||
|
obj_global_id = get_obj_global_id(obj_type, subtype, obj_id)
|
||||||
|
data = relationships_engine.get_chat_forward_stats(obj_global_id)
|
||||||
|
return data
|
||||||
|
return []
|
||||||
|
|
||||||
# --- RELATIONSHIPS --- #
|
# --- RELATIONSHIPS --- #
|
||||||
|
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
# -*-coding:UTF-8 -*
|
# -*-coding:UTF-8 -*
|
||||||
|
import json
|
||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
|
@ -16,13 +16,23 @@ config_loader = None
|
||||||
|
|
||||||
|
|
||||||
RELATIONSHIPS = {
|
RELATIONSHIPS = {
|
||||||
"forward", # forwarded_to
|
"forwarded_from",
|
||||||
|
"forwarded_to", # forwarded_to
|
||||||
|
"in",
|
||||||
"mention"
|
"mention"
|
||||||
}
|
}
|
||||||
|
|
||||||
RELATIONSHIPS_OBJS = {
|
RELATIONSHIPS_OBJS = {
|
||||||
"forward": {
|
"forwarded_from": {
|
||||||
'chat': {'chat', 'message'},
|
'chat': {'message'},
|
||||||
|
'message': {'chat', 'user-account'}
|
||||||
|
},
|
||||||
|
"forwarded_to": {
|
||||||
|
'chat': {'chat'},
|
||||||
|
'user-account': {'chat'},
|
||||||
|
},
|
||||||
|
"in": {
|
||||||
|
'chat': {'message'},
|
||||||
'message': {'chat'}
|
'message': {'chat'}
|
||||||
},
|
},
|
||||||
"mention": {}
|
"mention": {}
|
||||||
|
@ -51,10 +61,12 @@ def sanityze_obj_types(relationship, obj_type, filter_types):
|
||||||
objs_types = get_relationship_objs(relationship, obj_type)
|
objs_types = get_relationship_objs(relationship, obj_type)
|
||||||
if filter_types:
|
if filter_types:
|
||||||
filter_types = objs_types.intersection(filter_types)
|
filter_types = objs_types.intersection(filter_types)
|
||||||
if not filter_types:
|
else:
|
||||||
filter_types = objs_types
|
filter_types = objs_types
|
||||||
if not filter_types:
|
# if not filter_types:
|
||||||
return []
|
# filter_types = objs_types
|
||||||
|
# if not filter_types:
|
||||||
|
# return []
|
||||||
return filter_types
|
return filter_types
|
||||||
|
|
||||||
# TODO check obj_type
|
# TODO check obj_type
|
||||||
|
@ -149,3 +161,63 @@ def _get_relationship_graph(obj_global_id, links, nodes, meta, level, max_nodes,
|
||||||
|
|
||||||
# done.add(rel['id'])
|
# done.add(rel['id'])
|
||||||
|
|
||||||
|
# # # # # # # # # # # # # # # # # # # # # # # ## # # # # # # ## # # # # # # ## # # # # # # ## # # # # # # ## # # # #
|
||||||
|
|
||||||
|
def get_chat_forward_stats_out(obj_global_id):
|
||||||
|
nb = {}
|
||||||
|
for rel in get_obj_relationships(obj_global_id, relationships={'forwarded_from'}, filter_types={'message'}):
|
||||||
|
chat_mess = rel['source'].split('/')
|
||||||
|
|
||||||
|
chat_target = f'chat:{chat_mess[0][9:]}:{chat_mess[2]}'
|
||||||
|
if chat_target not in nb:
|
||||||
|
nb[chat_target] = 0
|
||||||
|
nb[chat_target] += 1
|
||||||
|
return nb
|
||||||
|
|
||||||
|
def get_chat_forward_stats_in(obj_global_id):
|
||||||
|
nb = {}
|
||||||
|
for rel in get_obj_relationships(obj_global_id, relationships={'in'}, filter_types={'message'}):
|
||||||
|
r = get_obj_relationships(rel['source'], relationships={'forwarded_from'}, filter_types={'chat', 'user-account'})
|
||||||
|
if r:
|
||||||
|
if not r[0]['target'] in nb:
|
||||||
|
nb[r[0]['target']] = 0
|
||||||
|
nb[r[0]['target']] += 1
|
||||||
|
# chat_mess = rel['source'].split('/')
|
||||||
|
#
|
||||||
|
# chat_target = f'chat:{chat_mess[0][9:]}:{chat_mess[2]}'
|
||||||
|
# print(chat_target, chat, chat_target == chat)
|
||||||
|
# if chat is None or chat_target == chat:
|
||||||
|
# if chat_target not in nb:
|
||||||
|
# nb[chat_target] = 0
|
||||||
|
# nb[chat_target] += 1
|
||||||
|
#
|
||||||
|
# print(json.dumps(nb, indent=4))
|
||||||
|
return nb
|
||||||
|
|
||||||
|
def get_chat_forward_stats(obj_global_id): # objs_hidden
|
||||||
|
data = []
|
||||||
|
|
||||||
|
# print(get_obj_relationships(obj_global_id, relationships=['forwarded_from'], filter_types=['message']))
|
||||||
|
|
||||||
|
in_mess = get_chat_forward_stats_out(obj_global_id)
|
||||||
|
|
||||||
|
out_mess = get_chat_forward_stats_in(obj_global_id)
|
||||||
|
|
||||||
|
|
||||||
|
# if rel['target'] == obj_global_id:
|
||||||
|
# # print(rel)
|
||||||
|
# nb_chats[rel['source']] = get_chat_forward_stats_out(rel['source'], chat=obj_global_id)
|
||||||
|
#
|
||||||
|
for target in in_mess:
|
||||||
|
data.append({'source': obj_global_id, 'target': target, 'value': in_mess[target]})
|
||||||
|
|
||||||
|
for source in out_mess:
|
||||||
|
data.append({'source': source, 'target': obj_global_id, 'value': out_mess[source]})
|
||||||
|
|
||||||
|
# print()
|
||||||
|
# print(in_mess)
|
||||||
|
# print()
|
||||||
|
# print(out_mess)
|
||||||
|
|
||||||
|
return data
|
||||||
|
|
||||||
|
|
|
@ -24,6 +24,7 @@ sys.path.append(os.environ['AIL_BIN'])
|
||||||
# Import Project packages
|
# Import Project packages
|
||||||
##################################
|
##################################
|
||||||
from lib.objects import ail_objects
|
from lib.objects import ail_objects
|
||||||
|
from lib import chats_viewer
|
||||||
from lib import Tag
|
from lib import Tag
|
||||||
|
|
||||||
bootstrap_label = Flask_config.bootstrap_label
|
bootstrap_label = Flask_config.bootstrap_label
|
||||||
|
@ -269,6 +270,18 @@ def relationships_graph_node_json():
|
||||||
json_graph = ail_objects.get_relationships_graph_node(obj_type, subtype, obj_id, relationships=relationships, filter_types=filter_types, max_nodes=max_nodes, level=level, flask_context=True)
|
json_graph = ail_objects.get_relationships_graph_node(obj_type, subtype, obj_id, relationships=relationships, filter_types=filter_types, max_nodes=max_nodes, level=level, flask_context=True)
|
||||||
return jsonify(json_graph)
|
return jsonify(json_graph)
|
||||||
|
|
||||||
|
@correlation.route('/relationships/chord_graph_json')
|
||||||
|
@login_required
|
||||||
|
@login_read_only
|
||||||
|
def relationships_chord_graph_json():
|
||||||
|
obj_id = request.args.get('id')
|
||||||
|
subtype = request.args.get('subtype')
|
||||||
|
obj_type = request.args.get('type')
|
||||||
|
|
||||||
|
chat_json_graph = ail_objects.get_chat_relationships_cord_graph(obj_type, subtype, obj_id)
|
||||||
|
meta = chats_viewer.enrich_chat_relationships_labels(chat_json_graph)
|
||||||
|
|
||||||
|
return jsonify({'meta': meta, 'data': chat_json_graph})
|
||||||
|
|
||||||
@correlation.route('/relationship/show', methods=['GET', 'POST'])
|
@correlation.route('/relationship/show', methods=['GET', 'POST'])
|
||||||
@login_required
|
@login_required
|
||||||
|
|
|
@ -16,7 +16,8 @@
|
||||||
<script src="{{ url_for('static', filename='js/helper.js')}}"></script>
|
<script src="{{ url_for('static', filename='js/helper.js')}}"></script>
|
||||||
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
|
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
|
||||||
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
|
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
|
||||||
<script src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
|
<!--<script src="{{ url_for('static', filename='js/d3.min.js') }}"></script>-->
|
||||||
|
<script src="https://d3js.org/d3.v6.min.js"></script>
|
||||||
|
|
||||||
<style>
|
<style>
|
||||||
.icon_legend {
|
.icon_legend {
|
||||||
|
@ -71,6 +72,7 @@
|
||||||
|
|
||||||
.blured {
|
.blured {
|
||||||
filter: blur(5px);
|
filter: blur(5px);
|
||||||
|
max-width: 200px;
|
||||||
}
|
}
|
||||||
|
|
||||||
.graph_panel {
|
.graph_panel {
|
||||||
|
@ -210,8 +212,16 @@
|
||||||
|
|
||||||
<li class="list-group-item text-left">
|
<li class="list-group-item text-left">
|
||||||
<div class="form-check">
|
<div class="form-check">
|
||||||
<input class="form-check-input" type="checkbox" value="True" id="relationship_forward_Check" name="relationship_forward_Check" {%if "forward" in dict_object["relationships"]%}checked{%endif%}>
|
<input class="form-check-input" type="checkbox" value="True" id="relationship_forwarded_from_Check" name="relationship_forwarded_from_Check" {%if "forwarded_from" in dict_object["relationships"]%}checked{%endif%}>
|
||||||
<label class="form-check-label" for="relationship_forward_Check">Forward</label>
|
<label class="form-check-label" for="relationship_forwarded_from_Check">Forwarded From</label>
|
||||||
|
</div>
|
||||||
|
<div class="form-check">
|
||||||
|
<input class="form-check-input" type="checkbox" value="True" id="relationship_forwarded_to_Check" name="relationship_forwarded_to_Check" {%if "forwarded_to" in dict_object["relationships"]%}checked{%endif%}>
|
||||||
|
<label class="form-check-label" for="relationship_forwarded_to_Check">Forwarded To</label>
|
||||||
|
</div>
|
||||||
|
<div class="form-check">
|
||||||
|
<input class="form-check-input" type="checkbox" value="True" id="relationship_in_Check" name="relationship_in_Check" {%if "in" in dict_object["relationships"]%}checked{%endif%}>
|
||||||
|
<label class="form-check-label" for="relationship_in_Check">Forward</label>
|
||||||
</div>
|
</div>
|
||||||
<div class="form-check">
|
<div class="form-check">
|
||||||
<input class="form-check-input" type="checkbox" value="True" id="relationship_mention_Check" name="relationship_mention_Check" {%if "mention" in dict_object["relationships"]%}checked{%endif%}>
|
<input class="form-check-input" type="checkbox" value="True" id="relationship_mention_Check" name="relationship_mention_Check" {%if "mention" in dict_object["relationships"]%}checked{%endif%}>
|
||||||
|
@ -337,10 +347,17 @@
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
</div>
|
<div id="chart_test" style="max-width: 900px"></div>
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
|
|
||||||
|
<table id="table_graph_node_objects">
|
||||||
|
</table>
|
||||||
|
|
||||||
|
<div id="timeline"></div>
|
||||||
|
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
<script>
|
<script>
|
||||||
|
|
||||||
var all_graph = {};
|
var all_graph = {};
|
||||||
|
@ -350,6 +367,14 @@ $(document).ready(function(){
|
||||||
|
|
||||||
all_graph.node_graph = create_graph("{{ url_for('correlation.relationships_graph_node_json') }}?id={{ dict_object["correlation_id"] }}&type={{ dict_object["object_type"] }}&level={{ dict_object["level"] }}&relationships={{ dict_object["relationships_str"] }}&filter={{ dict_object["filter_str"] }}&max_nodes={{dict_object["max_nodes"]}}{% if 'type_id' in dict_object["metadata"] %}&subtype={{ dict_object["metadata"]["type_id"] }}{% endif %}&hidden={{ dict_object["hidden_str"] }}");
|
all_graph.node_graph = create_graph("{{ url_for('correlation.relationships_graph_node_json') }}?id={{ dict_object["correlation_id"] }}&type={{ dict_object["object_type"] }}&level={{ dict_object["level"] }}&relationships={{ dict_object["relationships_str"] }}&filter={{ dict_object["filter_str"] }}&max_nodes={{dict_object["max_nodes"]}}{% if 'type_id' in dict_object["metadata"] %}&subtype={{ dict_object["metadata"]["type_id"] }}{% endif %}&hidden={{ dict_object["hidden_str"] }}");
|
||||||
all_graph.onResize();
|
all_graph.onResize();
|
||||||
|
|
||||||
|
let url = "{{ url_for('correlation.relationships_chord_graph_json') }}?id={{ dict_object["correlation_id"] }}&type={{ dict_object["object_type"] }}{% if 'type_id' in dict_object["metadata"] %}&subtype={{ dict_object["metadata"]["type_id"] }}{% endif %}"
|
||||||
|
d3.json(url).then(function(data) {
|
||||||
|
const diagram = createDirectedChordDiagram(data);
|
||||||
|
document.getElementById('chart_test').appendChild(diagram);
|
||||||
|
});
|
||||||
|
|
||||||
|
|
||||||
});
|
});
|
||||||
|
|
||||||
const blur_slider_correlation = $('#blur-slider-correlation');
|
const blur_slider_correlation = $('#blur-slider-correlation');
|
||||||
|
@ -577,7 +602,18 @@ d3.json(url)
|
||||||
|
|
||||||
d3.select("body").on("keypress", keypressed)
|
d3.select("body").on("keypress", keypressed)
|
||||||
|
|
||||||
|
let table_obj = document.getElementById("table_graph_node_objects")
|
||||||
|
for (let i=0; i<data.nodes.length; i++) {
|
||||||
|
let newRow = table_obj.insertRow();
|
||||||
|
let newCell = newRow.insertCell();
|
||||||
|
let newText = document.createTextNode(data.nodes[i].id);
|
||||||
|
newCell.appendChild(newText);
|
||||||
|
|
||||||
|
//console.log(data.nodes[i])
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
//// -----------------------------------------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
})
|
})
|
||||||
.catch(function(error) {
|
.catch(function(error) {
|
||||||
|
@ -588,19 +624,19 @@ d3.json(url)
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
function zoomed() {
|
function zoomed(event, d) {
|
||||||
container_graph.attr("transform", d3.event.transform);
|
container_graph.attr("transform", event.transform);
|
||||||
}
|
}
|
||||||
|
|
||||||
function doubleclick (d) {
|
function doubleclick (d) {
|
||||||
window.open(d.url, '_blank');
|
window.open(d.url, '_blank');
|
||||||
}
|
}
|
||||||
|
|
||||||
function keypressed () {
|
function keypressed (event, d) {
|
||||||
//console.log(d3.event.keyCode)
|
//console.log(event.keyCode)
|
||||||
//console.log(currentObject.id)
|
//console.log(currentObject.id)
|
||||||
// hide node, H or h key
|
// hide node, H or h key
|
||||||
if ((d3.event.keyCode === 72 || d3.event.keyCode === 104) && currentObject) {
|
if ((event.keyCode === 72 || event.keyCode === 104) && currentObject) {
|
||||||
window.location.href = correl_link + "&hide=" + currentObject.id
|
window.location.href = correl_link + "&hide=" + currentObject.id
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -610,29 +646,29 @@ function click (d) {
|
||||||
console.log('clicked')
|
console.log('clicked')
|
||||||
}
|
}
|
||||||
|
|
||||||
function drag_start(d) {
|
function drag_start(event, d) {
|
||||||
if (!d3.event.active) simulation.alphaTarget(0.3).restart();
|
if (!event.active) simulation.alphaTarget(0.3).restart();
|
||||||
d.fx = d.x;
|
d.fx = d.x;
|
||||||
d.fy = d.y;
|
d.fy = d.y;
|
||||||
}
|
}
|
||||||
|
|
||||||
function dragged(d) {
|
function dragged(event, d) {
|
||||||
d.fx = d3.event.x;
|
d.fx = event.x;
|
||||||
d.fy = d3.event.y;
|
d.fy = event.y;
|
||||||
}
|
}
|
||||||
|
|
||||||
function drag_end(d) {
|
function drag_end(event, d) {
|
||||||
if (!d3.event.active) simulation.alphaTarget(0);
|
if (!event.active) simulation.alphaTarget(0);
|
||||||
d.fx = d.x;
|
d.fx = d.x;
|
||||||
d.fy = d.y;
|
d.fy = d.y;
|
||||||
}
|
}
|
||||||
|
|
||||||
function mouseovered(d) {
|
function mouseovered(event, d, obj_gid, obj_label, additional_text) {
|
||||||
|
|
||||||
currentObject = d;
|
currentObject = d;
|
||||||
|
|
||||||
var d3_pageX = d3.event.pageX;
|
var d3_pageX = event.pageX;
|
||||||
var d3_pageY = d3.event.pageY;
|
var d3_pageY = event.pageY;
|
||||||
|
|
||||||
if (d.popover) {
|
if (d.popover) {
|
||||||
div.html(d.popover)
|
div.html(d.popover)
|
||||||
|
@ -645,7 +681,15 @@ if (d.popover) {
|
||||||
blur_tooltip();
|
blur_tooltip();
|
||||||
} else {
|
} else {
|
||||||
|
|
||||||
var pop_header = "<div class=\"card text-white\" style=\"max-width: 25rem;\"><div class=\"card-header bg-dark pb-0 border-white\"><h6>"+ sanitize_text(d.text) +"</h6></div>"
|
var pop_header = "<div class=\"card text-white\" style=\"max-width: 25rem;\"><div class=\"card-header bg-dark pb-0 border-white\"><h6>"
|
||||||
|
if (obj_label) {
|
||||||
|
pop_header = pop_header + sanitize_text(obj_label)
|
||||||
|
} else if (obj_gid) {
|
||||||
|
pop_header = pop_header + sanitize_text(obj_gid)
|
||||||
|
} else {
|
||||||
|
pop_header = pop_header + sanitize_text(d.text)
|
||||||
|
}
|
||||||
|
pop_header = pop_header + "</h6></div>"
|
||||||
var spinner = "<div class=\"card-body bg-dark pt-0\"><div class=\"spinner-border text-warning\" role=\"status\"></div> Loading...</div>"
|
var spinner = "<div class=\"card-body bg-dark pt-0\"><div class=\"spinner-border text-warning\" role=\"status\"></div> Loading...</div>"
|
||||||
|
|
||||||
div.html(pop_header + spinner)
|
div.html(pop_header + spinner)
|
||||||
|
@ -656,7 +700,14 @@ if (d.popover) {
|
||||||
.duration(200)
|
.duration(200)
|
||||||
.style("opacity", 1);
|
.style("opacity", 1);
|
||||||
|
|
||||||
$.getJSON("{{ url_for('correlation.get_description') }}?object_id="+ d.id,
|
let description_url = "{{ url_for('correlation.get_description') }}?object_id="
|
||||||
|
if (obj_gid) {
|
||||||
|
description_url = description_url + obj_gid
|
||||||
|
} else {
|
||||||
|
description_url = description_url + d.id
|
||||||
|
}
|
||||||
|
|
||||||
|
$.getJSON(description_url,
|
||||||
function(data){
|
function(data){
|
||||||
var desc = pop_header + "<div class=\"card-body bg-dark pb-1 pt-2\"><dl class=\"row py-0 my-0\">"
|
var desc = pop_header + "<div class=\"card-body bg-dark pb-1 pt-2\"><dl class=\"row py-0 my-0\">"
|
||||||
Object.keys(data).forEach(function(key) {
|
Object.keys(data).forEach(function(key) {
|
||||||
|
@ -674,8 +725,15 @@ if (d.popover) {
|
||||||
desc = desc + "fa-times-circle\"></i>DOWN"
|
desc = desc + "fa-times-circle\"></i>DOWN"
|
||||||
}
|
}
|
||||||
desc = desc + "</div></dd>"
|
desc = desc + "</div></dd>"
|
||||||
} else if (key!="tags" && key!="id" && key!="img" && key!="icon" && key!="link" && key!="type") {
|
} else if (key!=="tags" && key!=="id" && key!="img" && key!=="icon" && key!=="svg_icon" && key!=="link" && key!=="type" && key!=="tags_safe") {
|
||||||
|
if (data[key]) {
|
||||||
|
if ((key==="first_seen" || key==="last_seen") && data[key].length===8) {
|
||||||
|
let date = sanitize_text(data[key])
|
||||||
|
desc = desc + "<dt class=\"col-sm-3 px-0\">" + sanitize_text(key) + "</dt><dd class=\"col-sm-9 px-0\">" + date.slice(0, 4) + "-" + date.slice(4, 6) + "-" + date.slice(6, 8) + "</dd>"
|
||||||
|
} else {
|
||||||
desc = desc + "<dt class=\"col-sm-3 px-0\">" + sanitize_text(key) + "</dt><dd class=\"col-sm-9 px-0\">" + sanitize_text(data[key]) + "</dd>"
|
desc = desc + "<dt class=\"col-sm-3 px-0\">" + sanitize_text(key) + "</dt><dd class=\"col-sm-9 px-0\">" + sanitize_text(data[key]) + "</dd>"
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
desc = desc + "</dl>"
|
desc = desc + "</dl>"
|
||||||
|
@ -699,6 +757,10 @@ if (d.popover) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (additional_text) {
|
||||||
|
desc = desc + "</hr>" + sanitize_text(additional_text)
|
||||||
|
}
|
||||||
|
|
||||||
desc = desc + "</div></div>"
|
desc = desc + "</div></div>"
|
||||||
div.html(desc)
|
div.html(desc)
|
||||||
.style("left", (d3_pageX) + "px")
|
.style("left", (d3_pageX) + "px")
|
||||||
|
@ -736,6 +798,130 @@ all_graph.onResize = function () {
|
||||||
}
|
}
|
||||||
|
|
||||||
window.all_graph = all_graph;
|
window.all_graph = all_graph;
|
||||||
|
|
||||||
|
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||||
|
|
||||||
|
function getMaxCharsToShow(angle, radius) {
|
||||||
|
const approximateCharWidth = 7; // Approximate width of a character in pixels
|
||||||
|
const arcLength = angle * radius; // Length of the arc
|
||||||
|
return Math.floor(arcLength / approximateCharWidth); // Maximum number of characters that can fit
|
||||||
|
}
|
||||||
|
|
||||||
|
function createDirectedChordDiagram(data) {
|
||||||
|
const width = 900;
|
||||||
|
const height = width;
|
||||||
|
const innerRadius = Math.min(width, height) * 0.5 - 20;
|
||||||
|
const outerRadius = innerRadius + 6;
|
||||||
|
|
||||||
|
const labels_meta = data.meta
|
||||||
|
data = data.data
|
||||||
|
|
||||||
|
// Compute a dense matrix from the weighted links in data.
|
||||||
|
var names = Array.from(d3.union(data.flatMap(d => [d.source, d.target])));
|
||||||
|
const index = new Map(names.map((name, i) => [name, i]));
|
||||||
|
const matrix = Array.from(index, () => new Array(names.length).fill(0));
|
||||||
|
for (const {source, target, value} of data) matrix[index.get(source)][index.get(target)] += value;
|
||||||
|
|
||||||
|
const chord = d3.chordDirected()
|
||||||
|
.padAngle(12 / innerRadius)
|
||||||
|
.sortSubgroups(d3.descending)
|
||||||
|
.sortChords(d3.descending);
|
||||||
|
|
||||||
|
const arc = d3.arc()
|
||||||
|
.innerRadius(innerRadius)
|
||||||
|
.outerRadius(outerRadius);
|
||||||
|
|
||||||
|
const ribbon = d3.ribbonArrow()
|
||||||
|
.radius(innerRadius - 0.5)
|
||||||
|
.padAngle(1 / innerRadius);
|
||||||
|
|
||||||
|
const colors = d3.schemeCategory10;
|
||||||
|
|
||||||
|
const svg = d3.create("svg")
|
||||||
|
.attr("width", width)
|
||||||
|
.attr("height", height)
|
||||||
|
.attr("viewBox", [-width / 2, -height / 2, width, height])
|
||||||
|
.attr("style", "width: 100%; height: auto; font: 10px sans-serif;");
|
||||||
|
|
||||||
|
const chords = chord(matrix);
|
||||||
|
|
||||||
|
const textId = `text-${Math.random().toString(36).substring(2, 15)}`;
|
||||||
|
|
||||||
|
svg.append("path")
|
||||||
|
.attr("id", textId)
|
||||||
|
.attr("fill", "none")
|
||||||
|
.attr("d", d3.arc()({outerRadius, startAngle: 0, endAngle: 2 * Math.PI}));
|
||||||
|
|
||||||
|
svg.append("g")
|
||||||
|
.attr("fill-opacity", 0.75)
|
||||||
|
.selectAll("path")
|
||||||
|
.data(chords)
|
||||||
|
.enter()
|
||||||
|
.append("path")
|
||||||
|
.attr("d", ribbon)
|
||||||
|
.attr("fill", d => colors[d.target.index])
|
||||||
|
.style("mix-blend-mode", "multiply")
|
||||||
|
.append("title")
|
||||||
|
.text(d => `${labels_meta[names[d.target.index]]}
|
||||||
|
Forwarded ${d.source.value} Messages From
|
||||||
|
${labels_meta[names[d.source.index]]}`);
|
||||||
|
|
||||||
|
const g = svg.append("g")
|
||||||
|
.selectAll("g")
|
||||||
|
.data(chords.groups)
|
||||||
|
.enter()
|
||||||
|
.append("g");
|
||||||
|
|
||||||
|
g.append("path")
|
||||||
|
.attr("d", arc)
|
||||||
|
.attr("fill", d => colors[d.index])
|
||||||
|
.attr("stroke", "#fff")
|
||||||
|
.on("mouseover", function(event, d) {
|
||||||
|
mouseovered(event, d, names[d.index], labels_meta[names[d.index]]);
|
||||||
|
})
|
||||||
|
.on("mouseout", mouseouted);
|
||||||
|
|
||||||
|
g.each(function(d) {
|
||||||
|
const group = d3.select(this);
|
||||||
|
const angle = d.endAngle - d.startAngle;
|
||||||
|
const text = labels_meta[names[d.index]];
|
||||||
|
const maxCharsToShow = getMaxCharsToShow(angle, outerRadius);
|
||||||
|
|
||||||
|
let displayedText
|
||||||
|
if (maxCharsToShow <= 1) {
|
||||||
|
displayedText = text[0];
|
||||||
|
} else {
|
||||||
|
displayedText = text.length > maxCharsToShow ? text.slice(0, maxCharsToShow - 1) + "…" : text;
|
||||||
|
}
|
||||||
|
|
||||||
|
let info_text = `OUT: ${d3.sum(matrix[d.index])} Messages
|
||||||
|
IN: ${d3.sum(matrix, row => row[d.index])} Messages`
|
||||||
|
|
||||||
|
if (displayedText) {
|
||||||
|
group.append("text")
|
||||||
|
.attr("dy", -3)
|
||||||
|
.append("textPath")
|
||||||
|
.attr("xlink:href", `#${textId}`)
|
||||||
|
.attr("startOffset", d.startAngle * outerRadius)
|
||||||
|
.on("mouseover", function(event, d) {
|
||||||
|
mouseovered(event, d, names[d.index], labels_meta[names[d.index]], info_text);
|
||||||
|
})
|
||||||
|
.on("mouseout", mouseouted)
|
||||||
|
.text(displayedText);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
g.append("title")
|
||||||
|
.text(d => `OUT: ${d3.sum(matrix[d.index])} Messages
|
||||||
|
IN: ${d3.sum(matrix, row => row[d.index])} Messages`);
|
||||||
|
|
||||||
|
return svg.node();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
</body>
|
</body>
|
||||||
|
|
Loading…
Reference in New Issue