mirror of https://github.com/CIRCL/AIL-framework
chg: [relationship] get object relationhips by relationship name + object types
terrtia
parent
86f312cbc3
commit
70db33caaf
|
|
@ -10,6 +10,7 @@ Process Feeder Json (example: Twitter feeder)
|
|||
import datetime
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
from abc import ABC
|
||||
|
||||
|
|
@ -164,6 +165,39 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
|
|||
self.obj = Messages.Message(obj_id)
|
||||
return self.obj
|
||||
|
||||
# TODO handle subchannel
|
||||
def _process_chat(self, meta_chat, date, new_objs=None): #TODO NONE DATE???
|
||||
chat = Chat(meta_chat['id'], self.get_chat_instance_uuid())
|
||||
|
||||
# Obj Daterange
|
||||
chat.add(date)
|
||||
|
||||
if meta_chat.get('name'):
|
||||
chat.set_name(meta_chat['name'])
|
||||
|
||||
if meta_chat.get('info'):
|
||||
chat.set_info(meta_chat['info'])
|
||||
|
||||
if meta_chat.get('date'): # TODO check if already exists
|
||||
chat.set_created_at(int(meta_chat['date']['timestamp']))
|
||||
|
||||
if meta_chat.get('icon'):
|
||||
img = Images.create(meta_chat['icon'], b64=True)
|
||||
img.add(date, chat)
|
||||
chat.set_icon(img.get_global_id())
|
||||
if new_objs:
|
||||
new_objs.add(img)
|
||||
|
||||
if meta_chat.get('username'):
|
||||
username = Username(meta_chat['username'], self.get_chat_protocol())
|
||||
chat.update_username_timeline(username.get_global_id(), int(time.time()))
|
||||
username.add(date, obj=chat) # TODO TODAY DATE
|
||||
|
||||
return chat
|
||||
|
||||
##############################################################################################################################
|
||||
|
||||
|
||||
def process_chat(self, new_objs, obj, date, timestamp, reply_id=None):
|
||||
meta = self.json_data['meta']['chat'] # todo replace me by function
|
||||
chat = Chat(self.get_chat_id(), self.get_chat_instance_uuid())
|
||||
|
|
@ -365,73 +399,6 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
|
|||
# CHAT
|
||||
chat_objs = self.process_chat(new_objs, obj, date, timestamp, reply_id=reply_id)
|
||||
|
||||
# # TODO HANDLE OTHERS OBJECT TYPE
|
||||
# # TODO MAKE IT GENERIC FOR OTHERS CHATS !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
||||
# # Message forward + Discussion
|
||||
# if self.get_json_meta().get('forward'):
|
||||
# discussion_id = self.get_json_meta().get('discussion')
|
||||
# forward_from = self.get_message_forward()
|
||||
#
|
||||
# if discussion_id: # TODO HANDLE FORWARDED MESSAGES FROM EXTERNAL CHANNELS
|
||||
# chat_forward_id = forward_from['from']['id']
|
||||
# message_forward_id = forward_from['from']['channel_post']
|
||||
#
|
||||
# # if chat_forward_id == discussion_id:
|
||||
# # linked_chat = Chat(chat_forward_id, self.get_chat_instance_uuid())
|
||||
# # if linked_chat.exists():
|
||||
# # # create thread
|
||||
# # # add message replies for each childrens
|
||||
#
|
||||
# # TODO HANDLE THREAD
|
||||
# # TODO Change FORWARD META FIELDS
|
||||
# # meta['forward'] = {}
|
||||
# # # CHAT ID
|
||||
# # # SUBCHANNEL ID -> can be None
|
||||
# # # Message ID
|
||||
#
|
||||
# # meta['forward']['origin']
|
||||
# # # same as 'forward'
|
||||
#
|
||||
# if self.get_json_meta().get('forward'):
|
||||
# forward = self.get_message_forward()
|
||||
# f_chat = forward['chat']
|
||||
# f_subchannel = forward.get('subchannel')
|
||||
# f_id = forward.get('id')
|
||||
# if not f_subchannel:
|
||||
# chat_forward = Chat(f_chat, self.get_chat_instance_uuid())
|
||||
# if chat_forward.exists():
|
||||
# for chat_obj in chat_objs:
|
||||
# if chat_obj.type == 'chat':
|
||||
# chat_forward.add_relationship(chat_obj.get_global_id(), 'forward')
|
||||
# # TODO LIST FORWARDED MESSAGES
|
||||
#
|
||||
#
|
||||
# # Discord -> serverID + subchannel ID + message ID
|
||||
# # Telegram -> chat ID + Message ID
|
||||
# # + ORIGIN IDs
|
||||
#
|
||||
#
|
||||
#
|
||||
# # TODO create relationships graph
|
||||
#
|
||||
#
|
||||
# # TODO REMOVE ME
|
||||
# # Message forward # TODO handle subchannel + message ID
|
||||
# if self.get_json_meta().get('forward'):
|
||||
# forward_from = self.get_message_forward()
|
||||
# print('-----------------------------------------------------------')
|
||||
# print(forward_from)
|
||||
# if forward_from:
|
||||
# forward_from_type = forward_from['from']['type']
|
||||
# if forward_from_type == 'channel' or forward_from_type == 'chat':
|
||||
# chat_forward_id = forward_from['from']['id']
|
||||
# chat_forward = Chat(chat_forward_id, self.get_chat_instance_uuid())
|
||||
# if chat_forward.exists():
|
||||
# for chat_obj in chat_objs:
|
||||
# if chat_obj.type == 'chat':
|
||||
# chat_forward.add_relationship(chat_obj.get_global_id(), 'forward')
|
||||
# # chat_forward.add_relationship(obj.get_global_id(), 'forward')
|
||||
|
||||
# SENDER # TODO HANDLE NULL SENDER
|
||||
user_account = self.process_sender(new_objs, obj, date, timestamp)
|
||||
|
||||
|
|
@ -449,4 +416,24 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
|
|||
# -> subchannel ?
|
||||
# -> thread id ?
|
||||
|
||||
|
||||
#######################################################################
|
||||
|
||||
## FORWARD ##
|
||||
# chat_fwd = None
|
||||
# if self.get_json_meta().get('forward'):
|
||||
# meta_fwd = self.get_message_forward()
|
||||
# if meta_fwd['chat']:
|
||||
# chat_fwd = self._process_chat(meta_fwd['chat'], date, new_objs=new_objs)
|
||||
# for chat_obj in chat_objs:
|
||||
# if chat_obj.type == 'chat':
|
||||
# chat_fwd.add_relationship(chat_obj.get_global_id(), 'forward')
|
||||
#
|
||||
# # TODO chat_fwd -> message
|
||||
# if chat_fwd:
|
||||
# for obj in objs:
|
||||
# if obj.type == 'message':
|
||||
# chat_fwd.add_relationship(obj.get_global_id(), 'forward')
|
||||
# -FORWARD- #
|
||||
|
||||
return new_objs | objs
|
||||
|
|
|
|||
|
|
@ -562,15 +562,22 @@ def get_correlations_graph_node(obj_type, subtype, obj_id, filter_types=[], max_
|
|||
|
||||
# --- CORRELATION --- #
|
||||
|
||||
#### RELATIONSHIPS ####
|
||||
|
||||
def get_relationships():
|
||||
return relationships_engine.get_relationships()
|
||||
|
||||
def sanitize_relationships(relationships):
|
||||
return relationships_engine.sanitize_relationships(relationships)
|
||||
def get_obj_nb_relationships(obj_type, subtype, obj_id, filter_types=[]):
|
||||
obj = get_object(obj_type, subtype, obj_id)
|
||||
return obj.get_nb_relationships(filter=filter_types)
|
||||
|
||||
def get_relationships_graph_node(obj_type, subtype, obj_id, filter_types=[], max_nodes=300, level=1,
|
||||
def get_relationships_graph_node(obj_type, subtype, obj_id, relationships=[], filter_types=[], max_nodes=300, level=1,
|
||||
objs_hidden=set(),
|
||||
flask_context=False):
|
||||
obj_global_id = get_obj_global_id(obj_type, subtype, obj_id)
|
||||
nodes, links, meta = relationships_engine.get_relationship_graph(obj_global_id,
|
||||
nodes, links, meta = relationships_engine.get_relationship_graph(obj_global_id, relationships=relationships,
|
||||
filter_types=filter_types,
|
||||
max_nodes=max_nodes, level=level,
|
||||
objs_hidden=objs_hidden)
|
||||
|
|
@ -581,6 +588,9 @@ def get_relationships_graph_node(obj_type, subtype, obj_id, filter_types=[], max
|
|||
"meta": meta}
|
||||
|
||||
|
||||
# --- RELATIONSHIPS --- #
|
||||
|
||||
|
||||
# if __name__ == '__main__':
|
||||
# r = get_objects([{'lvl': 1, 'type': 'item', 'subtype': '', 'id': 'crawled/2020/09/14/circl.lu0f4976a4-dda4-4189-ba11-6618c4a8c951'}])
|
||||
# r = get_misp_objects([Item('crawled/2020/09/14/circl.lu0f4976a4-dda4-4189-ba11-6618c4a8c951'),
|
||||
|
|
|
|||
|
|
@ -16,23 +16,72 @@ config_loader = None
|
|||
|
||||
|
||||
RELATIONSHIPS = {
|
||||
"forward",
|
||||
"forward", # forwarded_to
|
||||
"mention"
|
||||
}
|
||||
|
||||
RELATIONSHIPS_OBJS = {
|
||||
"forward": {
|
||||
'chat': {'chat', 'message'},
|
||||
'message': {'chat'}
|
||||
},
|
||||
"mention": {}
|
||||
}
|
||||
|
||||
def get_relationships():
|
||||
return RELATIONSHIPS
|
||||
|
||||
def sanitize_relationships(relationships):
|
||||
rels = get_relationships()
|
||||
if relationships:
|
||||
relationships = set(relationships).intersection(rels)
|
||||
if not relationships:
|
||||
relationships = rels
|
||||
if not relationships:
|
||||
return []
|
||||
return relationships
|
||||
|
||||
def get_obj_relationships_by_type(obj_global_id, relationship):
|
||||
return r_rel.smembers(f'rel:{relationship}:{obj_global_id}')
|
||||
def get_relationship_obj_types(relationship):
|
||||
return RELATIONSHIPS_OBJS.get(relationship, {})
|
||||
|
||||
def get_obj_nb_relationships_by_type(obj_global_id, relationship):
|
||||
return r_rel.scard(f'rel:{relationship}:{obj_global_id}')
|
||||
def get_relationship_objs(relationship, obj_type):
|
||||
return get_relationship_obj_types(relationship).get(obj_type, set())
|
||||
|
||||
def get_obj_relationships(obj_global_id):
|
||||
relationships = []
|
||||
for relationship in get_relationships():
|
||||
for rel in get_obj_relationships_by_type(obj_global_id, relationship):
|
||||
def sanityze_obj_types(relationship, obj_type, filter_types):
|
||||
objs_types = get_relationship_objs(relationship, obj_type)
|
||||
if filter_types:
|
||||
filter_types = objs_types.intersection(filter_types)
|
||||
if not filter_types:
|
||||
filter_types = objs_types
|
||||
if not filter_types:
|
||||
return []
|
||||
return filter_types
|
||||
|
||||
# TODO check obj_type
|
||||
# TODO sanitize relationships
|
||||
|
||||
def get_obj_relationships_by_type(obj_global_id, relationship, filter_types=set()):
|
||||
obj_type = obj_global_id.split(':', 1)[0]
|
||||
relationships = {}
|
||||
filter_types = sanityze_obj_types(relationship, obj_type, filter_types)
|
||||
for o_type in filter_types:
|
||||
relationships[o_type] = r_rel.smembers(f'rel:{relationship}:{obj_global_id}:{o_type}')
|
||||
return relationships
|
||||
|
||||
def get_obj_nb_relationships_by_type(obj_global_id, relationship, filter_types=set()):
|
||||
obj_type = obj_global_id.split(':', 1)[0]
|
||||
relationships = {}
|
||||
filter_types = sanityze_obj_types(relationship, obj_type, filter_types)
|
||||
for o_type in filter_types:
|
||||
relationships[o_type] = r_rel.scard(f'rel:{relationship}:{obj_global_id}:{o_type}')
|
||||
return relationships
|
||||
|
||||
def get_obj_relationships(obj_global_id, relationships=set(), filter_types=set()):
|
||||
all_relationships = []
|
||||
for relationship in relationships:
|
||||
obj_relationships = get_obj_relationships_by_type(obj_global_id, relationship, filter_types=filter_types)
|
||||
for obj_type in obj_relationships:
|
||||
for rel in obj_relationships[obj_type]:
|
||||
meta = {'relationship': relationship}
|
||||
direction, obj_id = rel.split(':', 1)
|
||||
if direction == 'i':
|
||||
|
|
@ -42,15 +91,12 @@ def get_obj_relationships(obj_global_id):
|
|||
meta['target'] = obj_id
|
||||
meta['source'] = obj_global_id
|
||||
|
||||
if not obj_id.startswith('chat'):
|
||||
continue
|
||||
|
||||
meta['id'] = obj_id
|
||||
# meta['direction'] = direction
|
||||
relationships.append(meta)
|
||||
return relationships
|
||||
all_relationships.append(meta)
|
||||
return all_relationships
|
||||
|
||||
def get_obj_nb_relationships(obj_global_id):
|
||||
def get_obj_nb_relationships(obj_global_id): # TODO###########################################################################################
|
||||
nb = {}
|
||||
for relationship in get_relationships():
|
||||
nb[relationship] = get_obj_nb_relationships_by_type(obj_global_id, relationship)
|
||||
|
|
@ -59,27 +105,27 @@ def get_obj_nb_relationships(obj_global_id):
|
|||
|
||||
# TODO Filter by obj type ???
|
||||
def add_obj_relationship(source, target, relationship):
|
||||
r_rel.sadd(f'rel:{relationship}:{source}', f'o:{target}')
|
||||
r_rel.sadd(f'rel:{relationship}:{target}', f'i:{source}')
|
||||
# r_rel.sadd(f'rels:{source}', relationship)
|
||||
# r_rel.sadd(f'rels:{target}', relationship)
|
||||
source_type = source.split(':', 1)[0]
|
||||
target_type = target.split(':', 1)[0]
|
||||
r_rel.sadd(f'rel:{relationship}:{source}:{target_type}', f'o:{target}')
|
||||
r_rel.sadd(f'rel:{relationship}:{target}:{source_type}', f'i:{source}')
|
||||
|
||||
|
||||
def get_relationship_graph(obj_global_id, filter_types=[], max_nodes=300, level=1, objs_hidden=set()):
|
||||
def get_relationship_graph(obj_global_id, relationships=[], filter_types=[], max_nodes=300, level=1, objs_hidden=set()):
|
||||
links = []
|
||||
nodes = set()
|
||||
meta = {'complete': True, 'objs': set()}
|
||||
done = set()
|
||||
done_link = set()
|
||||
|
||||
_get_relationship_graph(obj_global_id, links, nodes, meta, level, max_nodes, filter_types=filter_types, objs_hidden=objs_hidden, done=done, done_link=done_link)
|
||||
_get_relationship_graph(obj_global_id, links, nodes, meta, level, max_nodes, relationships=relationships, filter_types=filter_types, objs_hidden=objs_hidden, done=done, done_link=done_link)
|
||||
return nodes, links, meta
|
||||
|
||||
def _get_relationship_graph(obj_global_id, links, nodes, meta, level, max_nodes, filter_types=[], objs_hidden=set(), done=set(), done_link=set()):
|
||||
def _get_relationship_graph(obj_global_id, links, nodes, meta, level, max_nodes, relationships=[], filter_types=[], objs_hidden=set(), done=set(), done_link=set()):
|
||||
meta['objs'].add(obj_global_id)
|
||||
nodes.add(obj_global_id)
|
||||
|
||||
for rel in get_obj_relationships(obj_global_id):
|
||||
for rel in get_obj_relationships(obj_global_id, relationships=relationships, filter_types=filter_types):
|
||||
meta['objs'].add(rel['id'])
|
||||
|
||||
if rel['id'] in done:
|
||||
|
|
@ -99,13 +145,7 @@ def _get_relationship_graph(obj_global_id, links, nodes, meta, level, max_nodes,
|
|||
if level > 0:
|
||||
next_level = level - 1
|
||||
|
||||
_get_relationship_graph(rel['id'], links, nodes, meta, next_level, max_nodes, filter_types=filter_types, objs_hidden=objs_hidden, done=done, done_link=done_link)
|
||||
_get_relationship_graph(rel['id'], links, nodes, meta, next_level, max_nodes, relationships=relationships, filter_types=filter_types, objs_hidden=objs_hidden, done=done, done_link=done_link)
|
||||
|
||||
# done.add(rel['id'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
source = ''
|
||||
target = ''
|
||||
add_obj_relationship(source, target, 'forward')
|
||||
# print(get_obj_relationships(source))
|
||||
|
|
|
|||
|
|
@ -263,7 +263,10 @@ def relationships_graph_node_json():
|
|||
max_nodes = sanitise_nb_max_nodes(request.args.get('max_nodes'))
|
||||
level = sanitise_level(request.args.get('level'))
|
||||
|
||||
json_graph = ail_objects.get_relationships_graph_node(obj_type, subtype, obj_id, max_nodes=max_nodes, level=level, flask_context=True)
|
||||
filter_types = ail_objects.sanitize_objs_types(request.args.get('filter', '').split(','))
|
||||
relationships = ail_objects.sanitize_relationships(request.args.get('relationships', '').split(','))
|
||||
|
||||
json_graph = ail_objects.get_relationships_graph_node(obj_type, subtype, obj_id, relationships=relationships, filter_types=filter_types, max_nodes=max_nodes, level=level, flask_context=True)
|
||||
return jsonify(json_graph)
|
||||
|
||||
|
||||
|
|
@ -278,8 +281,28 @@ def show_relationship():
|
|||
max_nodes = request.form.get('max_nb_nodes_in')
|
||||
level = sanitise_level(request.form.get('level'))
|
||||
|
||||
## get all selected relationships
|
||||
relationships = []
|
||||
for relationship in ail_objects.get_relationships():
|
||||
rel_option = request.form.get(f'relationship_{relationship}_Check')
|
||||
if rel_option:
|
||||
relationships.append(relationship)
|
||||
|
||||
relationships = ",".join(relationships)
|
||||
|
||||
## get all selected objects types
|
||||
filter_types = []
|
||||
for ob_type in ail_objects.get_all_objects():
|
||||
correl_option = request.form.get(f'{ob_type}_Check')
|
||||
if correl_option:
|
||||
filter_types.append(ob_type)
|
||||
|
||||
# list as params
|
||||
filter_types = ",".join(filter_types)
|
||||
|
||||
# redirect to keep history and bookmark
|
||||
return redirect(url_for('correlation.show_relationship', type=object_type, subtype=subtype, id=obj_id,
|
||||
filter=filter_types, relationships=relationships,
|
||||
max_nodes=max_nodes, level=level))
|
||||
|
||||
# request.method == 'GET'
|
||||
|
|
@ -290,6 +313,9 @@ def show_relationship():
|
|||
max_nodes = sanitise_nb_max_nodes(request.args.get('max_nodes'))
|
||||
level = sanitise_level(request.args.get('level'))
|
||||
|
||||
filter_types = ail_objects.sanitize_objs_types(request.args.get('filter', '').split(','), default=True)
|
||||
relationships = ail_objects.sanitize_relationships(request.args.get('relationships', '').split(','))
|
||||
|
||||
# check if obj_id exist
|
||||
if not ail_objects.exists_obj(obj_type, subtype, obj_id):
|
||||
return abort(404)
|
||||
|
|
@ -300,6 +326,8 @@ def show_relationship():
|
|||
"object_type": obj_type,
|
||||
"max_nodes": max_nodes, "level": level,
|
||||
"correlation_id": obj_id,
|
||||
"relationships": relationships, "relationships_str": ",".join(relationships),
|
||||
"filter": filter_types, "filter_str": ",".join(filter_types),
|
||||
"metadata": ail_objects.get_object_meta(obj_type, subtype, obj_id, options={'tags', 'info', 'icon', 'username'}, flask_context=True),
|
||||
"nb_relation": ail_objects.get_obj_nb_relationships(obj_type, subtype, obj_id)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -205,13 +205,37 @@
|
|||
<div class="card-body text-center px-0 py-0">
|
||||
|
||||
<ul class="list-group">
|
||||
<li class="list-group-item list-group-item-info">Relationship</li>
|
||||
<li class="list-group-item list-group-item-info">Relationships</li>
|
||||
<form action="{{ url_for('correlation.show_relationship') }}" method="post">
|
||||
|
||||
<li class="list-group-item text-left">
|
||||
<div class="form-check">
|
||||
<input class="form-check-input" type="checkbox" value="True" id="relationship_forward_Check" name="relationship_forward_Check" {%if "forward" in dict_object["relationships"]%}checked{%endif%}>
|
||||
<label class="form-check-label" for="relationship_forward_Check">Forward</label>
|
||||
</div>
|
||||
<div class="form-check">
|
||||
<input class="form-check-input" type="checkbox" value="True" id="relationship_mention_Check" name="relationship_mention_Check" {%if "mention" in dict_object["relationships"]%}checked{%endif%}>
|
||||
<label class="form-check-label" for="relationship_mention_Check">Mention</label>
|
||||
</div>
|
||||
</li>
|
||||
|
||||
<input type="hidden" id="obj_type" name="obj_type" value="{{ dict_object["object_type"] }}">
|
||||
<input type="hidden" id="subtype" name="subtype" value="{{ dict_object["metadata"]["type_id"] }}">
|
||||
<input type="hidden" id="obj_id" name="obj_id" value="{{ dict_object["correlation_id"] }}">
|
||||
|
||||
<li class="list-group-item list-group-item-info">Objects Types</li>
|
||||
|
||||
<li class="list-group-item text-left">
|
||||
<div class="form-check">
|
||||
<input class="form-check-input" type="checkbox" value="True" id="chat_Check" name="chat_Check" {%if "chat" in dict_object["filter"]%}checked{%endif%}>
|
||||
<label class="form-check-label" for="chat_Check">Chat</label>
|
||||
</div>
|
||||
<div class="form-check">
|
||||
<input class="form-check-input" type="checkbox" value="True" id="message_Check" name="message_Check" {%if "message" in dict_object["filter"]%}checked{%endif%}>
|
||||
<label class="form-check-label" for="message_Check">Message</label>
|
||||
</div>
|
||||
</li>
|
||||
|
||||
{# <li class="list-group-item text-left">#}
|
||||
{# <div class="form-check">#}
|
||||
{# <input class="form-check-input" type="checkbox" value="True" id="forwardCheck" name="forwardCheck" {%if "forward" in dict_object["filter"]%}checked{%endif%}>#}
|
||||
|
|
@ -324,7 +348,7 @@ $(document).ready(function(){
|
|||
$("#incomplete_graph").hide();
|
||||
$("#page-Decoded").addClass("active");
|
||||
|
||||
all_graph.node_graph = create_graph("{{ url_for('correlation.relationships_graph_node_json') }}?id={{ dict_object["correlation_id"] }}&type={{ dict_object["object_type"] }}&level={{ dict_object["level"] }}&filter={{ dict_object["filter_str"] }}&max_nodes={{dict_object["max_nodes"]}}{% if 'type_id' in dict_object["metadata"] %}&subtype={{ dict_object["metadata"]["type_id"] }}{% endif %}&hidden={{ dict_object["hidden_str"] }}");
|
||||
all_graph.node_graph = create_graph("{{ url_for('correlation.relationships_graph_node_json') }}?id={{ dict_object["correlation_id"] }}&type={{ dict_object["object_type"] }}&level={{ dict_object["level"] }}&relationships={{ dict_object["relationships_str"] }}&filter={{ dict_object["filter_str"] }}&max_nodes={{dict_object["max_nodes"]}}{% if 'type_id' in dict_object["metadata"] %}&subtype={{ dict_object["metadata"]["type_id"] }}{% endif %}&hidden={{ dict_object["hidden_str"] }}");
|
||||
all_graph.onResize();
|
||||
});
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue