CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

chg: [relationships] add relationship engine + WIP relationships between forwarded messages/chats

dev
terrtia 2024-01-26 15:42:46 +01:00
parent 9c6619aed3
commit 699453f079
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
13 changed files with 1097 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
bin/importer/FeederImporter.py
View File

@ -100,14 +100,15 @@ class FeederImporter(AbstractImporter):
else: else:
objs = set() objs = set()
objs.add(data_obj) if data_obj:
objs.add(data_obj)
for obj in objs: for obj in objs:
if obj.type == 'item': # object save on disk as file (Items) if obj.type == 'item': # object save on disk as file (Items)
gzip64_content = feeder.get_gzip64_content() gzip64_content = feeder.get_gzip64_content()
return obj, f'{feeder_name} {gzip64_content}' return obj, f'{feeder_name} {gzip64_content}'
else: # Messages save on DB else: # Messages save on DB
if obj.exists(): if obj.exists() and obj.type != 'chat':
return obj, f'{feeder_name}' return obj, f'{feeder_name}'
@ -136,4 +137,5 @@ class FeederModuleImporter(AbstractModule):
# Launch Importer # Launch Importer
if __name__ == '__main__': if __name__ == '__main__':
module = FeederModuleImporter() module = FeederModuleImporter()
# module.debug = True
module.run() module.run()

54
bin/importer/feeders/abstract_chats_feeder.py
View File

@ -93,7 +93,10 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
return self.json_data['meta'].get('reactions', []) return self.json_data['meta'].get('reactions', [])
def get_message_timestamp(self): def get_message_timestamp(self):
return self.json_data['meta']['date']['timestamp'] # TODO CREATE DEFAULT TIMESTAMP if not self.json_data['meta'].get('date'):
return None
else:
return self.json_data['meta']['date']['timestamp']
# if self.json_data['meta'].get('date'): # if self.json_data['meta'].get('date'):
# date = datetime.datetime.fromtimestamp( self.json_data['meta']['date']['timestamp']) # date = datetime.datetime.fromtimestamp( self.json_data['meta']['date']['timestamp'])
# date = date.strftime('%Y/%m/%d') # date = date.strftime('%Y/%m/%d')
@ -115,17 +118,29 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
def get_message_reply_id(self): def get_message_reply_id(self):
return self.json_data['meta'].get('reply_to', {}).get('message_id') return self.json_data['meta'].get('reply_to', {}).get('message_id')
def get_message_forward(self):
return self.json_data['meta'].get('forward')
def get_message_content(self): def get_message_content(self):
decoded = base64.standard_b64decode(self.json_data['data']) decoded = base64.standard_b64decode(self.json_data['data'])
return _gunzip_bytes_obj(decoded) return _gunzip_bytes_obj(decoded)
def get_obj(self): # TODO handle others objects -> images, pdf, ... def get_obj(self):
#### TIMESTAMP #### #### TIMESTAMP ####
timestamp = self.get_message_timestamp() timestamp = self.get_message_timestamp()
#### Create Object ID #### #### Create Object ID ####
chat_id = self.get_chat_id() chat_id = self.get_chat_id()
message_id = self.get_message_id() try:
message_id = self.get_message_id()
except KeyError:
if chat_id:
self.obj = Chat(chat_id, self.get_chat_instance_uuid())
return self.obj
else:
self.obj = None
return None
thread_id = self.get_thread_id() thread_id = self.get_thread_id()
# channel id # channel id
# thread id # thread id
@ -236,7 +251,10 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
# # ADD NEW MESSAGE REF (used by discord) # # ADD NEW MESSAGE REF (used by discord)
def process_sender(self, new_objs, obj, date, timestamp): def process_sender(self, new_objs, obj, date, timestamp):
meta = self.json_data['meta']['sender'] meta = self.json_data['meta'].get('sender')
if not meta:
return None
user_account = UsersAccount.UserAccount(meta['id'], self.get_chat_instance_uuid()) user_account = UsersAccount.UserAccount(meta['id'], self.get_chat_instance_uuid())
# date stat + correlation # date stat + correlation
@ -286,8 +304,6 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
# REPLY # REPLY
reply_id = self.get_message_reply_id() reply_id = self.get_message_reply_id()
# TODO Translation
print(self.obj.type) print(self.obj.type)
# TODO FILES + FILES REF # TODO FILES + FILES REF
@ -295,7 +311,7 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
# get object by meta object type # get object by meta object type
if self.obj.type == 'message': if self.obj.type == 'message':
# Content # Content
obj = Messages.create(self.obj.id, self.get_message_content()) # TODO translation obj = Messages.create(self.obj.id, self.get_message_content())
# FILENAME # FILENAME
media_name = self.get_media_name() media_name = self.get_media_name()
@ -305,7 +321,8 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
for reaction in self.get_reactions(): for reaction in self.get_reactions():
obj.add_reaction(reaction['reaction'], int(reaction['count'])) obj.add_reaction(reaction['reaction'], int(reaction['count']))
elif self.obj.type == 'chat':
pass
else: else:
chat_id = self.get_chat_id() chat_id = self.get_chat_id()
thread_id = self.get_thread_id() thread_id = self.get_thread_id()
@ -341,12 +358,29 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
# CHAT # CHAT
chat_objs = self.process_chat(new_objs, obj, date, timestamp, reply_id=reply_id) chat_objs = self.process_chat(new_objs, obj, date, timestamp, reply_id=reply_id)
# Message forward
# if self.get_json_meta().get('forward'):
# forward_from = self.get_message_forward()
# print('-----------------------------------------------------------')
# print(forward_from)
# if forward_from:
# forward_from_type = forward_from['from']['type']
# if forward_from_type == 'channel' or forward_from_type == 'chat':
# chat_forward_id = forward_from['from']['id']
# chat_forward = Chat(chat_forward_id, self.get_chat_instance_uuid())
# if chat_forward.exists():
# for chat_obj in chat_objs:
# if chat_obj.type == 'chat':
# chat_forward.add_relationship(chat_obj.get_global_id(), 'forward')
# # chat_forward.add_relationship(obj.get_global_id(), 'forward')
# SENDER # TODO HANDLE NULL SENDER # SENDER # TODO HANDLE NULL SENDER
user_account = self.process_sender(new_objs, obj, date, timestamp) user_account = self.process_sender(new_objs, obj, date, timestamp)
if user_account:
# UserAccount---ChatObjects # UserAccount---ChatObjects
for obj_chat in chat_objs: for obj_chat in chat_objs:
user_account.add_correlation(obj_chat.type, obj_chat.get_subtype(r_str=True), obj_chat.id) user_account.add_correlation(obj_chat.type, obj_chat.get_subtype(r_str=True), obj_chat.id)
# if chat: # TODO Chat---Username correlation ??? # if chat: # TODO Chat---Username correlation ???
# # Chat---Username => need to handle members and participants # # Chat---Username => need to handle members and participants

3
bin/lib/objects/Chats.py
View File

@ -76,6 +76,7 @@ class Chat(AbstractChatObject):
meta['tags'] = self.get_tags(r_list=True) meta['tags'] = self.get_tags(r_list=True)
if 'icon' in options: if 'icon' in options:
meta['icon'] = self.get_icon() meta['icon'] = self.get_icon()
meta['img'] = meta['icon']
if 'info' in options: if 'info' in options:
meta['info'] = self.get_info() meta['info'] = self.get_info()
if 'participants' in options: if 'participants' in options:
@ -93,6 +94,8 @@ class Chat(AbstractChatObject):
if 'threads' in options: if 'threads' in options:
meta['threads'] = self.get_threads() meta['threads'] = self.get_threads()
print(meta['threads']) print(meta['threads'])
if 'tags_safe' in options:
meta['tags_safe'] = self.is_tags_safe(meta['tags'])
return meta return meta
def get_misp_object(self): def get_misp_object(self):

33
bin/lib/objects/Messages.py
View File

@ -131,7 +131,7 @@ class Message(AbstractObject):
if meta: if meta:
_, user_account_subtype, user_account_id = user_account.split(':', 3) _, user_account_subtype, user_account_id = user_account.split(':', 3)
user_account = UsersAccount.UserAccount(user_account_id, user_account_subtype).get_meta(options={'icon', 'username', 'username_meta'}) user_account = UsersAccount.UserAccount(user_account_id, user_account_subtype).get_meta(options={'icon', 'username', 'username_meta'})
return user_account return user_account
def get_files_names(self): def get_files_names(self):
names = [] names = []
@ -148,15 +148,32 @@ class Message(AbstractObject):
def add_reaction(self, reactions, nb_reaction): def add_reaction(self, reactions, nb_reaction):
r_object.hset(f'meta:reactions:{self.type}::{self.id}', reactions, nb_reaction) r_object.hset(f'meta:reactions:{self.type}::{self.id}', reactions, nb_reaction)
# Update value on import # Interactions between users -> use replies
# reply to -> parent ?
# reply/comment - > children ?
# nb views # nb views
# reactions # MENTIONS -> Messages + Chats
# nb fowards # # relationship -> mention - Chat -> Chat
# room ??? # - Message -> Chat
# message from channel ??? # - Message -> Message ??? fetch mentioned messages
# FORWARDS
# TODO Create forward CHAT -> message
# message (is forwarded) -> message (is forwarded from) ???
# # TODO get source message timestamp
#
# # is forwarded
# # forwarded from -> check if relationship
# # nb forwarded -> scard relationship
#
# Messages -> CHATS -> NB forwarded
# CHAT -> NB forwarded by chats -> NB messages -> parse full set ????
#
#
#
#
#
#
# show users chats
# message media # message media
# flag is deleted -> event or missing from feeder pass ???
def get_translation(self, content=None, source=None, target='fr'): def get_translation(self, content=None, source=None, target='fr'):
""" """

17
bin/lib/objects/abstract_object.py
View File

@ -24,6 +24,7 @@ from lib.ConfigLoader import ConfigLoader
from lib import Duplicate from lib import Duplicate
from lib.correlations_engine import get_nb_correlations, get_correlations, add_obj_correlation, delete_obj_correlation, delete_obj_correlations, exists_obj_correlation, is_obj_correlated, get_nb_correlation_by_correl_type, get_obj_inter_correlation from lib.correlations_engine import get_nb_correlations, get_correlations, add_obj_correlation, delete_obj_correlation, delete_obj_correlations, exists_obj_correlation, is_obj_correlated, get_nb_correlation_by_correl_type, get_obj_inter_correlation
from lib.Investigations import is_object_investigated, get_obj_investigations, delete_obj_investigations from lib.Investigations import is_object_investigated, get_obj_investigations, delete_obj_investigations
from lib.relationships_engine import get_obj_nb_relationships, add_obj_relationship
from lib.Tracker import is_obj_tracked, get_obj_trackers, delete_obj_trackers from lib.Tracker import is_obj_tracked, get_obj_trackers, delete_obj_trackers
logging.config.dictConfig(ail_logger.get_config(name='ail')) logging.config.dictConfig(ail_logger.get_config(name='ail'))
@ -284,6 +285,22 @@ class AbstractObject(ABC):
## -Correlation- ## ## -Correlation- ##
## Relationship ##
def get_nb_relationships(self, filter=[]):
return get_obj_nb_relationships(self.get_global_id())
def add_relationship(self, obj2_global_id, relationship, source=True):
# is source
if source:
print(self.get_global_id(), obj2_global_id, relationship)
add_obj_relationship(self.get_global_id(), obj2_global_id, relationship)
# is target
else:
add_obj_relationship(obj2_global_id, self.get_global_id(), relationship)
## -Relationship- ##
## Parent ## ## Parent ##
def is_parent(self): def is_parent(self):

19
bin/lib/objects/ail_objects.py
View File

@ -10,6 +10,7 @@ sys.path.append(os.environ['AIL_BIN'])
from lib.ConfigLoader import ConfigLoader from lib.ConfigLoader import ConfigLoader
from lib.ail_core import get_all_objects, get_object_all_subtypes from lib.ail_core import get_all_objects, get_object_all_subtypes
from lib import correlations_engine from lib import correlations_engine
from lib import relationships_engine
from lib import btc_ail from lib import btc_ail
from lib import Tag from lib import Tag
@ -468,6 +469,24 @@ def get_correlations_graph_node(obj_type, subtype, obj_id, filter_types=[], max_
# --- CORRELATION --- # # --- CORRELATION --- #
def get_obj_nb_relationships(obj_type, subtype, obj_id, filter_types=[]):
obj = get_object(obj_type, subtype, obj_id)
return obj.get_nb_relationships(filter=filter_types)
def get_relationships_graph_node(obj_type, subtype, obj_id, filter_types=[], max_nodes=300, level=1,
objs_hidden=set(),
flask_context=False):
obj_global_id = get_obj_global_id(obj_type, subtype, obj_id)
nodes, links, meta = relationships_engine.get_relationship_graph(obj_global_id,
filter_types=filter_types,
max_nodes=max_nodes, level=level,
objs_hidden=objs_hidden)
# print(meta)
meta['objs'] = list(meta['objs'])
return {"nodes": create_correlation_graph_nodes(nodes, obj_global_id, flask_context=flask_context),
"links": links,
"meta": meta}
# if __name__ == '__main__': # if __name__ == '__main__':
# r = get_objects([{'lvl': 1, 'type': 'item', 'subtype': '', 'id': 'crawled/2020/09/14/circl.lu0f4976a4-dda4-4189-ba11-6618c4a8c951'}]) # r = get_objects([{'lvl': 1, 'type': 'item', 'subtype': '', 'id': 'crawled/2020/09/14/circl.lu0f4976a4-dda4-4189-ba11-6618c4a8c951'}])

111
bin/lib/relationships_engine.py Executable file
View File

@ -0,0 +1,111 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
import os
import sys
sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
from lib.ConfigLoader import ConfigLoader
config_loader = ConfigLoader()
r_rel = config_loader.get_db_conn("Kvrocks_Relationships")
config_loader = None
RELATIONSHIPS = {
"forward",
"mention"
}
def get_relationships():
return RELATIONSHIPS
def get_obj_relationships_by_type(obj_global_id, relationship):
return r_rel.smembers(f'rel:{relationship}:{obj_global_id}')
def get_obj_nb_relationships_by_type(obj_global_id, relationship):
return r_rel.scard(f'rel:{relationship}:{obj_global_id}')
def get_obj_relationships(obj_global_id):
relationships = []
for relationship in get_relationships():
for rel in get_obj_relationships_by_type(obj_global_id, relationship):
meta = {'relationship': relationship}
direction, obj_id = rel.split(':', 1)
if direction == 'i':
meta['source'] = obj_id
meta['target'] = obj_global_id
else:
meta['target'] = obj_id
meta['source'] = obj_global_id
if not obj_id.startswith('chat'):
continue
meta['id'] = obj_id
# meta['direction'] = direction
relationships.append(meta)
return relationships
def get_obj_nb_relationships(obj_global_id):
nb = {}
for relationship in get_relationships():
nb[relationship] = get_obj_nb_relationships_by_type(obj_global_id, relationship)
return nb
# TODO Filter by obj type ???
def add_obj_relationship(source, target, relationship):
r_rel.sadd(f'rel:{relationship}:{source}', f'o:{target}')
r_rel.sadd(f'rel:{relationship}:{target}', f'i:{source}')
# r_rel.sadd(f'rels:{source}', relationship)
# r_rel.sadd(f'rels:{target}', relationship)
def get_relationship_graph(obj_global_id, filter_types=[], max_nodes=300, level=1, objs_hidden=set()):
links = []
nodes = set()
meta = {'complete': True, 'objs': set()}
done = set()
done_link = set()
_get_relationship_graph(obj_global_id, links, nodes, meta, level, max_nodes, filter_types=filter_types, objs_hidden=objs_hidden, done=done, done_link=done_link)
return nodes, links, meta
def _get_relationship_graph(obj_global_id, links, nodes, meta, level, max_nodes, filter_types=[], objs_hidden=set(), done=set(), done_link=set()):
meta['objs'].add(obj_global_id)
nodes.add(obj_global_id)
for rel in get_obj_relationships(obj_global_id):
meta['objs'].add(rel['id'])
if rel['id'] in done:
continue
if len(nodes) > max_nodes != 0:
meta['complete'] = False
break
nodes.add(rel['id'])
str_link = f"{rel['source']}{rel['target']}{rel['relationship']}"
if str_link not in done_link:
links.append({"source": rel['source'], "target": rel['target'], "relationship": rel['relationship']})
done_link.add(str_link)
if level > 0:
next_level = level - 1
_get_relationship_graph(rel['id'], links, nodes, meta, next_level, max_nodes, filter_types=filter_types, objs_hidden=objs_hidden, done=done, done_link=done_link)
# done.add(rel['id'])
if __name__ == '__main__':
source = ''
target = ''
add_obj_relationship(source, target, 'forward')
# print(get_obj_relationships(source))

1
configs/6383.conf
View File

@ -664,6 +664,7 @@ namespace.db ail_datas
namespace.dup ail_dups namespace.dup ail_dups
namespace.obj ail_objs namespace.obj ail_objs
namespace.tl ail_tls namespace.tl ail_tls
namespace.rel ail_rels
namespace.stat ail_stats namespace.stat ail_stats
namespace.tag ail_tags namespace.tag ail_tags
namespace.track ail_trackers namespace.track ail_trackers

5
configs/core.cfg.sample
View File

@ -196,6 +196,11 @@ host = localhost
port = 6383 port = 6383
password = ail_objs password = ail_objs
[Kvrocks_Relationships]
host = localhost
port = 6383
password = ail_rels
[Kvrocks_Timeline] [Kvrocks_Timeline]
host = localhost host = localhost
port = 6383 port = 6383

63
var/www/blueprints/correlation.py
View File

@ -203,7 +203,7 @@ def get_description():
return Response(json.dumps({"status": "error", "reason": "404 Not Found"}, indent=2, sort_keys=True), mimetype='application/json'), 404 return Response(json.dumps({"status": "error", "reason": "404 Not Found"}, indent=2, sort_keys=True), mimetype='application/json'), 404
# object exist # object exist
else: else:
res = ail_objects.get_object_meta(obj_type, subtype, obj_id, options={'tags', 'tags_safe'}, res = ail_objects.get_object_meta(obj_type, subtype, obj_id, options={'icon', 'tags', 'tags_safe'},
flask_context=True) flask_context=True)
if 'tags' in res: if 'tags' in res:
res['tags'] = list(res['tags']) res['tags'] = list(res['tags'])
@ -292,3 +292,64 @@ def correlation_tags_add():
max_nodes=nb_max, max_nodes=nb_max,
hidden=hidden, hidden_str=",".join(hidden), hidden=hidden, hidden_str=",".join(hidden),
filter=",".join(filter_types))) filter=",".join(filter_types)))
#####################################################################################
@correlation.route('/relationships/graph_node_json')
@login_required
@login_read_only
def relationships_graph_node_json():
obj_id = request.args.get('id')
subtype = request.args.get('subtype')
obj_type = request.args.get('type')
max_nodes = sanitise_nb_max_nodes(request.args.get('max_nodes'))
level = sanitise_level(request.args.get('level'))
json_graph = ail_objects.get_relationships_graph_node(obj_type, subtype, obj_id, max_nodes=max_nodes, level=level, flask_context=True)
return jsonify(json_graph)
@correlation.route('/relationship/show', methods=['GET', 'POST'])
@login_required
@login_read_only
def show_relationship():
if request.method == 'POST':
object_type = request.form.get('obj_type')
subtype = request.form.get('subtype')
obj_id = request.form.get('obj_id')
max_nodes = request.form.get('max_nb_nodes_in')
level = sanitise_level(request.form.get('level'))
# redirect to keep history and bookmark
return redirect(url_for('correlation.show_relationship', type=object_type, subtype=subtype, id=obj_id,
max_nodes=max_nodes, level=level))
# request.method == 'GET'
else:
obj_type = request.args.get('type')
subtype = request.args.get('subtype', '')
obj_id = request.args.get('id')
max_nodes = sanitise_nb_max_nodes(request.args.get('max_nodes'))
level = sanitise_level(request.args.get('level'))
# check if obj_id exist
if not ail_objects.exists_obj(obj_type, subtype, obj_id):
return abort(404)
# object exist
else: # TODO remove old dict key
dict_object = {"type": obj_type,
"id": obj_id,
"object_type": obj_type,
"max_nodes": max_nodes, "level": level,
"correlation_id": obj_id,
"metadata": ail_objects.get_object_meta(obj_type, subtype, obj_id, options={'tags', 'info', 'icon', 'username'}, flask_context=True),
"nb_relation": ail_objects.get_obj_nb_relationships(obj_type, subtype, obj_id)
}
if subtype:
dict_object["subtype"] = subtype
dict_object["metadata"]['type_id'] = subtype
else:
dict_object["subtype"] = ''
dict_object["metadata_card"] = ail_objects.get_object_card_meta(obj_type, subtype, obj_id)
return render_template("show_relationship.html", dict_object=dict_object, bootstrap_label=bootstrap_label,
tags_selector_data=Tag.get_tags_selector_data())

78
var/www/templates/correlation/metadata_card_chat.html Normal file
View File

@ -0,0 +1,78 @@
<link href="{{ url_for('static', filename='css/tags.css') }}" rel="stylesheet" type="text/css" />
<script src="{{ url_for('static', filename='js/tags.js') }}"></script>
{#{% with modal_add_tags=dict_object['metadata_card']['add_tags_modal']%}#}
{# {% include 'modals/add_tags.html' %}#}
{#{% endwith %}#}
{% include 'modals/edit_tag.html' %}
<div class="card my-3">
<div class="card-header" style="background-color:#d9edf7;font-size: 15px">
<h4>{{ dict_object["correlation_id"] }}</h4>
{{ dict_object }}
<div class="text-secondary">{{ dict_object["correlation_id"] }}</div>
<ul class="list-group mb-2">
<li class="list-group-item py-0">
<div class="row">
<div class="col-md-10">
<table class="table">
<thead>
<tr>
<th>Object type</th>
<th>First seen</th>
<th>Last seen</th>
<th>Nb seen</th>
</tr>
</thead>
<tbody>
<tr>
<td>
<svg height="26" width="26">
<g class="nodes">
<circle cx="13" cy="13" r="13" fill="orange"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ dict_object["metadata"]["icon"]["style"] }}" font-size="16px">{{ dict_object["metadata"]["icon"]["icon"] }}</text>
</g>
</svg>
{{ dict_object["object_type"] }}
</td>
<td>{{ dict_object["metadata"]['first_seen'] }}</td>
<td>{{ dict_object["metadata"]['last_seen'] }}</td>
<td>{{ dict_object["metadata"]['nb_seen'] }}</td>
</tr>
</tbody>
</table>
</div>
<div class="col-md-1">
<div id="sparkline"></div>
</div>
</div>
</li>
<li class="list-group-item py-0">
<br>
<div class="mb-3">
Tags:
{% for tag in dict_object["metadata"]['tags'] %}
<button class="btn btn-{{ bootstrap_label[loop.index0 % 5] }}" data-toggle="modal" data-target="#edit_tags_modal"
data-tagid="{{ tag }}" data-objtype="cookie-name" data-objsubtype="" data-objid="{{ dict_object["correlation_id"] }}">
{{ tag }}
</button>
{% endfor %}
<button type="button" class="btn btn-light" data-toggle="modal" data-target="#add_tags_modal">
<i class="far fa-plus-square"></i>
</button>
</div>
</li>
</ul>
{% with obj_type='cookie-name', obj_id=dict_object['correlation_id'], obj_subtype='' %}
{% include 'modals/investigations_register_obj.html' %}
{% endwith %}
<button type="button" class="btn btn-primary" data-toggle="modal" data-target="#investigations_register_obj_modal">
<i class="fas fa-microscope"></i> Investigations
</button>
</div>
</div>

9
var/www/templates/correlation/show_correlation.html
View File

@ -541,6 +541,15 @@ d3.json(url)
.on("drag", dragged) .on("drag", dragged)
.on("end", drag_end)); .on("end", drag_end));
/*
node.append("image")
.attr("xlink:href", "https://circl.lu/assets/images/circl-logo.png")
.attr("height", 20)
.attr("width", 20)
.attr("x", -10)
.attr("y", -10);
*/
node.append("circle") node.append("circle")
.attr("r", function(d) { .attr("r", function(d) {

719
var/www/templates/correlation/show_relationship.html Normal file
View File

@ -0,0 +1,719 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<title>AIL - framework</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/helper.js')}}"></script>
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
<style>
.icon_legend {
color: #0c5460;
background-color: #bee5eb;
}
line.link {
stroke: #666;
}
line.link:hover{
stroke: red;
stroke-width: 2px;
}
.node {
pointer-events: all;
}
circle {
stroke: none;
}
.graph_text_node {
font: 8px sans-serif;
pointer-events: none;
}
.graph_node_icon {
pointer-events: none;
}
.node text {
font: 8px sans-serif;
pointer-events: auto;
}
div.tooltip_graph {
position: absolute;
text-align: center;
padding: 2px;
font: 12px sans-serif;
background: #ebf4fb;
border: 2px solid #b7ddf2;
border-radius: 8px;
pointer-events: none;
color: #000000;
}
.pixelated {
image-rendering: pixelated;
}
.blured {
filter: blur(5px);
}
.graph_panel {
padding: unset;
}
.line_graph {
fill: none;
stroke: steelblue;
stroke-width: 2px;
stroke-linejoin: round;
stroke-linecap: round;
stroke-width: 1.5;
/*attr('stroke', '#bcbd22').*/
}
</style>
</head>
<body>
{% include 'nav_bar.html' %}
<div class="container-fluid">
<div class="row">
{% include 'sidebars/sidebar_objects.html' %}
<div class="col-12 col-lg-10" id="core_content">
{% if dict_object["object_type"] == "pgp" %}
{% include 'correlation/metadata_card_pgp.html' %}
{% elif dict_object["object_type"] == "cryptocurrency" %}
{% include 'correlation/metadata_card_cryptocurrency.html' %}
{% elif dict_object["object_type"] == "username" %}
{% include 'correlation/metadata_card_username.html' %}
{% elif dict_object["object_type"] == "decoded" %}
{% include 'correlation/metadata_card_decoded.html' %}
{% elif dict_object["object_type"] == "chat" %}
{% include 'correlation/metadata_card_chat.html' %}
{% elif dict_object["object_type"] == "cve" %}
{% include 'correlation/metadata_card_cve.html' %}
{% elif dict_object["object_type"] == "domain" %}
{% include 'correlation/metadata_card_domain.html' %}
{% elif dict_object["object_type"] == "screenshot" %}
{% include 'correlation/metadata_card_screenshot.html' %}
{% elif dict_object["object_type"] == "title" %}
{% include 'correlation/metadata_card_title.html' %}
{% elif dict_object["object_type"] == "cookie-name" %}
{% include 'correlation/metadata_card_cookie_name.html' %}
{% elif dict_object["object_type"] == "etag" %}
{% include 'correlation/metadata_card_etag.html' %}
{% elif dict_object["object_type"] == "hhhash" %}
{% include 'correlation/metadata_card_hhhash.html' %}
{% elif dict_object["object_type"] == "item" %}
{% include 'correlation/metadata_card_item.html' %}
{% endif %}
<div class="row">
<div class="col-xl-10">
<div class="card mb-3">
<div class="card-header">
<i class="fas fa-project-diagram"></i> Graph
<span class="float-right">
<div class="card border-secondary">
<div class="card-body py-2">
<div class="row">
<div class="col-md-3 text-center px-0">
<button class="btn btn-sm btn-secondary" onclick="blur_slider_correlation.val(0);blur_tooltip();">
<i class="fas fa-eye-slash"></i>
<span class="label-icon">Hide</span>
</button>
</div>
<div class="col-md-6 text-center pl-0 pt-1">
<input type="range" min="0" max="15" step="0.1" value="10" id="blur-slider-correlation">
</div>
<div class="col-md-3 text-center">
<button class="btn btn-sm btn-secondary" onclick="blur_slider_correlation.val(15);blur_tooltip();">
<i class="fas fa-image"></i>
<span class="label-icon">Full</span>
</button>
</div>
</div>
</div>
</div>
</span>
{# <span class="float-right mx-1">#}
{# {% if dict_object["object_type"] != "" %}#}
{# {% with obj_type=dict_object["object_type"], obj_id=dict_object["correlation_id"], obj_subtype=dict_object["metadata"]["type_id"],obj_lvl=1%}#}
{# {% include 'import_export/block_add_user_object_to_export.html' %}#}
{# {% endwith %}#}
{# {% endif %}#}
{# </span>#}
<span class="float-right mt-2">
<button class="btn btn-primary py-1" onclick="resize_graph();">
<i class="fas fa-sync"></i>&nbsp;Resize Graph
</button>
</span>
<span class="float-right mt-2 mx-1">
<button class="btn btn-primary py-1" onclick="reset_graph();">
<i class="fas fa-undo"></i>&nbsp;Reset Graph
</button>
</span>
<div id="incomplete_graph" class="text-danger mt-3">
<i class="fas fa-exclamation-triangle"></i>&nbsp;Graph Incomplete, Max Nodes Reached.
</div>
</div>
<div class="card-body graph_panel">
<div id="graph_loading" class="ml-3 mt-3">
<div class="spinner-border text-secondary" role="status">
<span class="sr-only">Loading...</span>
</div>
<span>Loading...</span>
</div>
<div id="graph">
</div>
</div>
</div>
{# <p>Press <b>H</b> on an object / node to hide it.</p>#}
{# {% if dict_object["hidden"] %}#}
{# <h5>Hidden objects:</h5>#}
{# {% for obj_hidden in dict_object["hidden"] %}#}
{# {{ obj_hidden }} <br>#}
{# {% endfor %}#}
{# {% endif %}#}
</div>
<div class="col-xl-2">
<div class="card mb-3">
<div class="card-body text-center px-0 py-0">
<ul class="list-group">
<li class="list-group-item list-group-item-info">Relationship</li>
<form action="{{ url_for('correlation.show_relationship') }}" method="post">
<input type="hidden" id="obj_type" name="obj_type" value="{{ dict_object["object_type"] }}">
<input type="hidden" id="subtype" name="subtype" value="{{ dict_object["metadata"]["type_id"] }}">
<input type="hidden" id="obj_id" name="obj_id" value="{{ dict_object["correlation_id"] }}">
{# <li class="list-group-item text-left">#}
{# <div class="form-check">#}
{# <input class="form-check-input" type="checkbox" value="True" id="forwardCheck" name="forwardCheck" {%if "forward" in dict_object["filter"]%}checked{%endif%}>#}
{# <label class="form-check-label" for="forwardCheck">Forwards</label>#}
{# </div>#}
{# <div class="form-check">#}
{# <input class="form-check-input" type="checkbox" value="True" id="mentionCheck" name="mentionCheck" {%if "mention" in dict_object["filter"]%}checked{%endif%}>#}
{# <label class="form-check-label" for="mentionCheck">Mentions</label>#}
{# </div>#}
{##}
{# </li>#}
<li class="list-group-item text-left">
<div class="form-group">
<label for="max_nb_nodes_in">Correlation Depth:</label>
<input class="form-control" type="number" value="{{dict_object["level"]}}" min="0" id="level" name="level">
</div>
</li>
<li class="list-group-item text-left">
<div class="form-group">
<label for="max_nb_nodes_in">Max number of nodes:</label>
<input class="form-control" type="number" value="{{dict_object["max_nodes"]}}" min="0" id="max_nb_nodes_in" name="max_nb_nodes_in">
</div>
<div class="text-center">
<input class="btn btn-primary" type="submit" value="Redraw Graph">
</div>
</li>
</form>
</ul>
<ul class="list-group">
<li class="list-group-item list-group-item-info"><i class="fas fa-info-circle fa-2x"></i></li>
<li class="list-group-item text-left">
<p>Double click on a node to open this object<br><br>
<svg height="26" width="26">
<g class="nodes">
<circle cx="13" cy="13" r="13" fill="orange"></circle>
</g>
</svg>
Current Object<br>
</p>
</li>
</ul>
<ul class="list-group">
<li class="list-group-item list-group-item-secondary"><i class="fas fa-project-diagram"></i> Direct Relationships</li>
<li class="list-group-item text-left">
{% for relationship in dict_object['nb_relation'] %}
<div class="row">
<div class="col-8">
{{ relationship }}
</div>
<div class="col-4">
<span class="badge badge-primary">{{ dict_object['nb_relation'][relationship] }}</span>
</div>
</div>
{% endfor %}
</li>
</ul>
</div>
</div>
</div>
</div>
<div class="row">
<div class="col-xl-10">
{% include 'correlation/legend_graph_correlation.html' %}
</div>
</div>
<div class="card my-2">
<div class="card-header bg-light">
<h4><i class="fas fa-tags"></i> Tags All Objects</h4>
</div>
<div class="card-body">
<form action="{{ url_for('correlation.correlation_tags_add') }}" method='post' onsubmit="SubmitAddTags();">
<input type="hidden" id="tag_obj_type" name="tag_obj_type" value="{{ dict_object["object_type"] }}">
<input type="hidden" id="tag_subtype" name="tag_subtype" value="{{ dict_object["metadata"]["type_id"] }}">
<input type="hidden" id="tag_obj_id" name="tag_obj_id" value="{{ dict_object["correlation_id"] }}">
<input type="hidden" id="tag_level" name="tag_level" value="{{dict_object["level"]}}">
<input type="hidden" id="tag_nb_max" name="tag_nb_max" value="{{dict_object["max_nodes"]}}">
<input type="hidden" id="filter" name="tag_filter" value="{{dict_object["filter_str"]}}">
<input type="hidden" id="tag_hidden" name="tag_hidden" value="{{dict_object["hidden_str"]}}">
{% include 'tags/block_tags_selector.html' %}
<button class="btn btn-primary mt-2">
<i class="fas fa-tag"></i> Add Tags
</button>
</form>
</div>
</div>
</div>
</div>
</div>
<script>
var all_graph = {};
$(document).ready(function(){
$("#incomplete_graph").hide();
$("#page-Decoded").addClass("active");
all_graph.node_graph = create_graph("{{ url_for('correlation.relationships_graph_node_json') }}?id={{ dict_object["correlation_id"] }}&type={{ dict_object["object_type"] }}&level={{ dict_object["level"] }}&filter={{ dict_object["filter_str"] }}&max_nodes={{dict_object["max_nodes"]}}{% if 'type_id' in dict_object["metadata"] %}&subtype={{ dict_object["metadata"]["type_id"] }}{% endif %}&hidden={{ dict_object["hidden_str"] }}");
all_graph.onResize();
});
const blur_slider_correlation = $('#blur-slider-correlation');
function blur_tooltip(){
var image = $('#tooltip_screenshot_correlation')[0];
if (image) {
var blurValue = $('#blur-slider-correlation').val();
blurValue = 15 - blurValue;
image.style.filter = "blur(" + blurValue + "px)";
}
}
$(window).on("resize", function() {
all_graph.onResize();
});
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){
$('#nav_menu').hide();
$('#side_menu').removeClass('border-right')
$('#side_menu').removeClass('col-lg-2')
$('#core_content').removeClass('col-lg-10')
}else{
$('#nav_menu').show();
$('#side_menu').addClass('border-right')
$('#side_menu').addClass('col-lg-2')
$('#core_content').addClass('col-lg-10')
}
}
function SubmitAddTags() {
var tags = ltags.getValue();
var tagsgalaxy = ltagsgalaxies.getValue();
$('#ltags').val(tags);
$('#ltagsgalaxies').val(tagsgalaxy);
return true;
}
</script>
<script>
function resize_graph() {
zoom.translateTo(svg_node, 200, 200);
zoom.scaleTo(svg_node, 2);
}
function reset_graph() {
window.location.href = "{{ url_for('correlation.show_relationship') }}?type={{ dict_object["type"] }}&subtype={{ dict_object["subtype"] }}&id={{ dict_object["id"] }}"
}
</script>
<script>
var correl_link = "{{ url_for('correlation.show_relationship') }}?type={{ dict_object["type"] }}&subtype={{ dict_object["subtype"] }}&id={{ dict_object["id"] }}&max_nodes={{ dict_object["max_nodes"] }}&level={{ dict_object["level"] }}&filter={{ dict_object["filter_str"] }}"
{% if 'hidden_str' in dict_object %}
correl_link = correl_link + "&hidden={{ dict_object["hidden_str"] }}"
{% endif %}
var width = 400,
height = 400;
var link;
var zoom = d3.zoom()
.scaleExtent([.2, 10])
.on("zoom", zoomed);
// Loading:
//$("#graph_loading").show()
//var transform = d3.zoomIdentity;
var color = d3.scaleOrdinal(d3.schemeCategory10);
var div = d3.select("body").append("div")
.attr("class", "tooltip_graph")
.style("opacity", 0);
var simulation = d3.forceSimulation()
.force("link", d3.forceLink().id(function(d) { return d.id; }))
.force("charge", d3.forceManyBody())
.force("center", d3.forceCenter(width / 2, height / 2));
//.on("tick", ticked);
var currentObject = null;
var svg_node = d3.select("#graph").append("svg")
.attr("id", "graph_div")
.attr("width", width)
.attr("height", height)
.call(d3.zoom().scaleExtent([1, 8]).on("zoom", zoomed))
.on("dblclick.zoom", null)
var container_graph = svg_node.append("g");
//.attr("transform", "translate(40,0)")
//.attr("transform", "scale(2)");
function create_graph(url){
d3.json(url)
.then(function(data){
// ----------------------------------------------------------------------------------
svg_node.append("defs").selectAll("marker")
.data(["end"]) // Different link/path types can be defined here
.enter().append("marker")
.attr("id", String)
.attr("viewBox", "0 -5 10 10")
.attr("refX", 25) // Controls the distance of the marker from the node
.attr("refY", 0)
.attr("markerWidth", 6)
.attr("markerHeight", 6)
.attr("orient", "auto")
.append("path")
.attr("d", "M0,-5L10,0L0,5");
// ----------------------------------------------------------------------------------
var link = container_graph.append("g")
/*.selectAll("line")
.data(data.links)
.enter().append("line")
.attr("class", "link")
.attr("marker-end", "url(#end)");*/
.selectAll("g")
.data(data.links)
.enter().append("g");
link.append("line")
.attr("class", "link")
.attr("marker-end", "url(#end)")
link.append("text")
.attr("class", "link-label")
.style("font-size", "6px")
.text(function(d) { return d.relationship; })
//.attr("stroke-width", function(d) { return Math.sqrt(d.value); })
var node = container_graph.selectAll(".node")
.data(data.nodes)
.enter().append("g")
.attr("class", "nodes")
.on("dblclick", doubleclick)
.on("click", click)
.on("mouseover", mouseovered)
.on("mouseout", mouseouted)
.call(d3.drag()
.on("start", drag_start)
.on("drag", dragged)
.on("end", drag_end));
/*
node.append("image")
.attr("xlink:href", "https://circl.lu/assets/images/circl-logo.png")
.attr("height", 20)
.attr("width", 20)
.attr("x", -10)
.attr("y", -10);
*/
node.append("circle")
.attr("r", function(d) {
return (d.style.node_radius);})
.attr("fill", function(d) {
return d.style.node_color;});
node.append('text')
.attr('text-anchor', 'middle')
.attr('dominant-baseline', 'central')
.attr("class", function(d) {
return "graph_node_icon " + d.style.icon_class
})
.attr('font-size', '8px' )
.attr('pointer-events', 'none')
.text(function(d) {
//if(d.hash){
return d.style.icon_text
//}
});
zoom.translateTo(svg_node, 200, 200);
zoom.scaleTo(svg_node, 2);
/* node.append("text")
.attr("dy", 3)
.attr("dx", 7)
.attr("class", "graph_text_node")
//.style("text-anchor", function(d) { return d.children ? "end" : "start"; })
.text(function(d) { return d.id; });*/
simulation
.nodes(data.nodes)
.on("tick", ticked);
simulation.force("link")
.links(data.links);
function ticked() {
link.select("line")
.attr("x1", function(d) { return d.source.x; })
.attr("y1", function(d) { return d.source.y; })
.attr("x2", function(d) { return d.target.x; })
.attr("y2", function(d) { return d.target.y; });
link.select("text")
.attr("x", function(d) {
return (d.source.x + d.target.x) / 2;
})
.attr("y", function(d) {
return (d.source.y + d.target.y) / 2;
});
/*node
.attr("cx", function(d) { return d.x; })
.attr("cy", function(d) { return d.y; });*/
node.attr("transform", function(d) { return "translate(" + d.x + "," + d.y + ")"; });
}
// Loading ...
$("#graph_loading").remove();
if (!data.meta.complete){
$("#incomplete_graph").show();
}
d3.select("body").on("keypress", keypressed)
})
.catch(function(error) {
$("#graph_loading").remove()
svg_node.remove();
d3.select("#graph").append("div")
.text(error);
});
}
function zoomed() {
container_graph.attr("transform", d3.event.transform);
}
function doubleclick (d) {
window.open(d.url, '_blank');
}
function keypressed () {
//console.log(d3.event.keyCode)
//console.log(currentObject.id)
// hide node, H or h key
if ((d3.event.keyCode === 72 || d3.event.keyCode === 104) && currentObject) {
window.location.href = correl_link + "&hide=" + currentObject.id
}
}
function click (d) {
console.log('clicked')
}
function drag_start(d) {
if (!d3.event.active) simulation.alphaTarget(0.3).restart();
d.fx = d.x;
d.fy = d.y;
}
function dragged(d) {
d.fx = d3.event.x;
d.fy = d3.event.y;
}
function drag_end(d) {
if (!d3.event.active) simulation.alphaTarget(0);
d.fx = d.x;
d.fy = d.y;
}
function mouseovered(d) {
currentObject = d;
var d3_pageX = d3.event.pageX;
var d3_pageY = d3.event.pageY;
if (d.popover) {
div.html(d.popover)
.style("left", (d3_pageX) + "px")
.style("top", (d3_pageY - 28) + "px");
div.transition()
.duration(200)
.style("opacity", 1);
blur_tooltip();
} else {
var pop_header = "<div class=\"card text-white\" style=\"max-width: 25rem;\"><div class=\"card-header bg-dark pb-0 border-white\"><h6>"+ sanitize_text(d.text) +"</h6></div>"
var spinner = "<div class=\"card-body bg-dark pt-0\"><div class=\"spinner-border text-warning\" role=\"status\"></div> Loading...</div>"
div.html(pop_header + spinner)
.style("left", (d3_pageX) + "px")
.style("top", (d3_pageY - 28) + "px");
div.transition()
.duration(200)
.style("opacity", 1);
$.getJSON("{{ url_for('correlation.get_description') }}?object_id="+ d.id,
function(data){
var desc = pop_header + "<div class=\"card-body bg-dark pb-1 pt-2\"><dl class=\"row py-0 my-0\">"
Object.keys(data).forEach(function(key) {
if (key=="status") {
desc = desc + "<dt class=\"col-sm-3 px-0\">status</dt><dd class=\"col-sm-9 px-0\"><div class=\"badge badge-pill badge-light flex-row-reverse\" style=\"color:"
if (data["status"]) {
desc = desc + "Green"
} else {
desc = desc + "Red"
}
desc = desc + ";\"><i class=\"fas "
if (data["status"]) {
desc = desc + "fa-check-circle\"></i>UP"
} else {
desc = desc + "fa-times-circle\"></i>DOWN"
}
desc = desc + "</div></dd>"
} else if (key!="tags" && key!="id" && key!="img" && key!="icon" && key!="link" && key!="type") {
desc = desc + "<dt class=\"col-sm-3 px-0\">" + sanitize_text(key) + "</dt><dd class=\"col-sm-9 px-0\">" + sanitize_text(data[key]) + "</dd>"
}
});
desc = desc + "</dl>"
if (data["tags"]) {
data["tags"].forEach(function(tag) {
desc = desc + "<span class=\"badge badge-warning\">"+ sanitize_text(tag) +"</span>";
});
}
if (data["img"]) {
if (data["tags_safe"]) {
if (data["type"] === "screenshot") {
desc = desc + "<img src={{ url_for('objects_item.screenshot', filename="") }}"
} else {
desc = desc + "<img src={{ url_for('objects_image.image', filename="") }}"
}
desc = desc + data["img"] +" class=\"img-thumbnail blured\" id=\"tooltip_screenshot_correlation\" style=\"\"/>";
} else {
desc = desc + "<span class=\"my-2 fa-stack fa-4x\"><i class=\"fas fa-stack-1x fa-image\"></i><i class=\"fas fa-stack-2x fa-ban\" style=\"color:Red\"></i></span>";
}
}
desc = desc + "</div></div>"
div.html(desc)
.style("left", (d3_pageX) + "px")
.style("top", (d3_pageY - 28) + "px");
d.popover = desc
if (data["img"]) {
blur_tooltip();
}
})
.fail(function(error) {
desc = pop_header + "<div class=\"card-body bg-dark pt-0\"><i class=\"fas fa-3x fa-times text-danger\"></i>"+ error.statusText +"</div>"
div.html(desc)
.style("left", (d3_pageX) + "px")
.style("top", (d3_pageY - 28) + "px");
});
}
}
function mouseouted() {
currentObject = null;
div.transition()
.duration(500)
.style("opacity", 0);
//link.select("line")
}
all_graph.onResize = function () {
var aspect = 1000 / 500, all_graph = $("#graph_div");
var targetWidth = all_graph.parent().width();
all_graph.attr("width", targetWidth);
all_graph.attr("height", targetWidth / aspect);
}
window.all_graph = all_graph;
</script>
</body>
</html>