mirror of https://github.com/CIRCL/AIL-framework
chg: [correlation] correlation graph, add an option to hide an object/node by pressing H + reset correlation graph
Terrtia
parent
450ebdd789
commit
482fc21b5e
|
|
@ -170,18 +170,18 @@ def get_obj_str_id(obj_type, subtype, obj_id):
|
|||
subtype = ''
|
||||
return f'{obj_type}:{subtype}:{obj_id}'
|
||||
|
||||
def get_correlations_graph_nodes_links(obj_type, subtype, obj_id, filter_types=[], max_nodes=300, level=1, flask_context=False):
|
||||
def get_correlations_graph_nodes_links(obj_type, subtype, obj_id, filter_types=[], max_nodes=300, level=1, objs_hidden=set(), flask_context=False):
|
||||
links = set()
|
||||
nodes = set()
|
||||
meta = {'complete': True, 'objs': set()}
|
||||
|
||||
obj_str_id = get_obj_str_id(obj_type, subtype, obj_id)
|
||||
|
||||
_get_correlations_graph_node(links, nodes, meta, obj_type, subtype, obj_id, level, max_nodes, filter_types=filter_types, previous_str_obj='')
|
||||
_get_correlations_graph_node(links, nodes, meta, obj_type, subtype, obj_id, level, max_nodes, filter_types=filter_types, objs_hidden=objs_hidden, previous_str_obj='')
|
||||
return obj_str_id, nodes, links, meta
|
||||
|
||||
|
||||
def _get_correlations_graph_node(links, nodes, meta, obj_type, subtype, obj_id, level, max_nodes, filter_types=[], previous_str_obj=''):
|
||||
def _get_correlations_graph_node(links, nodes, meta, obj_type, subtype, obj_id, level, max_nodes, filter_types=[], objs_hidden=set(), previous_str_obj=''):
|
||||
obj_str_id = get_obj_str_id(obj_type, subtype, obj_id)
|
||||
meta['objs'].add(obj_str_id)
|
||||
nodes.add(obj_str_id)
|
||||
|
|
@ -192,6 +192,10 @@ def _get_correlations_graph_node(links, nodes, meta, obj_type, subtype, obj_id,
|
|||
for str_obj in obj_correlations[correl_type]:
|
||||
subtype2, obj2_id = str_obj.split(':', 1)
|
||||
obj2_str_id = get_obj_str_id(correl_type, subtype2, obj2_id)
|
||||
# filter objects to hide
|
||||
if obj2_str_id in objs_hidden:
|
||||
continue
|
||||
|
||||
meta['objs'].add(obj2_str_id)
|
||||
|
||||
if obj2_str_id == previous_str_obj:
|
||||
|
|
@ -205,5 +209,5 @@ def _get_correlations_graph_node(links, nodes, meta, obj_type, subtype, obj_id,
|
|||
|
||||
if level > 0:
|
||||
next_level = level - 1
|
||||
_get_correlations_graph_node(links, nodes, meta, correl_type, subtype2, obj2_id, next_level, max_nodes, filter_types=filter_types, previous_str_obj=obj_str_id)
|
||||
_get_correlations_graph_node(links, nodes, meta, correl_type, subtype2, obj2_id, next_level, max_nodes, filter_types=filter_types, objs_hidden=objs_hidden, previous_str_obj=obj_str_id)
|
||||
|
||||
|
|
|
|||
|
|
@ -338,7 +338,7 @@ def get_obj_correlations(obj_type, subtype, obj_id):
|
|||
obj = get_object(obj_type, subtype, obj_id)
|
||||
return obj.get_correlations()
|
||||
|
||||
def _get_obj_correlations_objs(objs, obj_type, subtype, obj_id, filter_types, lvl, nb_max):
|
||||
def _get_obj_correlations_objs(objs, obj_type, subtype, obj_id, filter_types, lvl, nb_max, objs_hidden):
|
||||
if len(objs) < nb_max or nb_max == 0:
|
||||
if lvl == 0:
|
||||
objs.add((obj_type, subtype, obj_id))
|
||||
|
|
@ -351,16 +351,17 @@ def _get_obj_correlations_objs(objs, obj_type, subtype, obj_id, filter_types, lv
|
|||
for obj2_type in correlations:
|
||||
for str_obj in correlations[obj2_type]:
|
||||
obj2_subtype, obj2_id = str_obj.split(':', 1)
|
||||
_get_obj_correlations_objs(objs, obj2_type, obj2_subtype, obj2_id, filter_types, lvl, nb_max)
|
||||
if get_obj_global_id(obj2_type, obj2_subtype, obj2_id) in objs_hidden:
|
||||
continue # filter object to hide
|
||||
_get_obj_correlations_objs(objs, obj2_type, obj2_subtype, obj2_id, filter_types, lvl, nb_max, objs_hidden)
|
||||
|
||||
def get_obj_correlations_objs(obj_type, subtype, obj_id, filter_types=[], lvl=0, nb_max=300):
|
||||
def get_obj_correlations_objs(obj_type, subtype, obj_id, filter_types=[], lvl=0, nb_max=300, objs_hidden=set()):
|
||||
objs = set()
|
||||
_get_obj_correlations_objs(objs, obj_type, subtype, obj_id, filter_types, lvl, nb_max)
|
||||
_get_obj_correlations_objs(objs, obj_type, subtype, obj_id, filter_types, lvl, nb_max, objs_hidden)
|
||||
return objs
|
||||
|
||||
def obj_correlations_objs_add_tags(obj_type, subtype, obj_id, tags, filter_types=[], lvl=0, nb_max=300):
|
||||
print(nb_max)
|
||||
objs = get_obj_correlations_objs(obj_type, subtype, obj_id, filter_types=filter_types, lvl=lvl, nb_max=nb_max)
|
||||
def obj_correlations_objs_add_tags(obj_type, subtype, obj_id, tags, filter_types=[], lvl=0, nb_max=300, objs_hidden=set()):
|
||||
objs = get_obj_correlations_objs(obj_type, subtype, obj_id, filter_types=filter_types, lvl=lvl, nb_max=nb_max, objs_hidden=objs_hidden)
|
||||
# print(objs)
|
||||
for obj_tuple in objs:
|
||||
obj1_type, subtype1, id1 = obj_tuple
|
||||
|
|
@ -422,10 +423,12 @@ def create_correlation_graph_nodes(nodes_set, obj_str_id, flask_context=True):
|
|||
|
||||
|
||||
def get_correlations_graph_node(obj_type, subtype, obj_id, filter_types=[], max_nodes=300, level=1,
|
||||
objs_hidden=set(),
|
||||
flask_context=False):
|
||||
obj_str_id, nodes, links, meta = correlations_engine.get_correlations_graph_nodes_links(obj_type, subtype, obj_id,
|
||||
filter_types=filter_types,
|
||||
max_nodes=max_nodes, level=level,
|
||||
objs_hidden=objs_hidden,
|
||||
flask_context=flask_context)
|
||||
# print(meta)
|
||||
meta['objs'] = list(meta['objs'])
|
||||
|
|
|
|||
|
|
@ -61,6 +61,13 @@ def sanitise_level(level):
|
|||
level = 2
|
||||
return level
|
||||
|
||||
def sanitise_objs_hidden(objs_hidden):
|
||||
if objs_hidden:
|
||||
objs_hidden = set(objs_hidden.split(',')) # TODO sanitize objects
|
||||
else:
|
||||
objs_hidden = set()
|
||||
return objs_hidden
|
||||
|
||||
# ============= ROUTES ==============
|
||||
@correlation.route('/correlation/show', methods=['GET', 'POST'])
|
||||
@login_required
|
||||
|
|
@ -130,6 +137,10 @@ def show_correlation():
|
|||
max_nodes = sanitise_nb_max_nodes(request.args.get('max_nodes'))
|
||||
mode = sanitise_graph_mode(request.args.get('mode'))
|
||||
level = sanitise_level(request.args.get('level'))
|
||||
objs_hidden = sanitise_objs_hidden(request.args.get('hidden'))
|
||||
obj_to_hide = request.args.get('hide')
|
||||
if obj_to_hide:
|
||||
objs_hidden.add(obj_to_hide)
|
||||
|
||||
related_btc = bool(request.args.get('related_btc', False))
|
||||
|
||||
|
|
@ -139,17 +150,24 @@ def show_correlation():
|
|||
if not ail_objects.exists_obj(obj_type, subtype, obj_id):
|
||||
return abort(404)
|
||||
# object exist
|
||||
else:
|
||||
dict_object = {"object_type": obj_type,
|
||||
"correlation_id": obj_id,
|
||||
else: # TODO remove old dict key
|
||||
dict_object = {"type": obj_type,
|
||||
"id": obj_id,
|
||||
"object_type": obj_type,
|
||||
"max_nodes": max_nodes, "mode": mode, "level": level,
|
||||
"filter": filter_types, "filter_str": ",".join(filter_types),
|
||||
"hidden": objs_hidden, "hidden_str": ",".join(objs_hidden),
|
||||
|
||||
"correlation_id": obj_id,
|
||||
"metadata": ail_objects.get_object_meta(obj_type, subtype, obj_id,
|
||||
options={'tags'}, flask_context=True),
|
||||
"nb_correl": ail_objects.get_obj_nb_correlations(obj_type, subtype, obj_id)
|
||||
}
|
||||
if subtype:
|
||||
dict_object["subtype"] = subtype
|
||||
dict_object["metadata"]['type_id'] = subtype
|
||||
else:
|
||||
dict_object["subtype"] = ''
|
||||
dict_object["metadata_card"] = ail_objects.get_object_card_meta(obj_type, subtype, obj_id, related_btc=related_btc)
|
||||
return render_template("show_correlation.html", dict_object=dict_object, bootstrap_label=bootstrap_label,
|
||||
tags_selector_data=Tag.get_tags_selector_data())
|
||||
|
|
@ -194,9 +212,15 @@ def graph_node_json():
|
|||
max_nodes = sanitise_nb_max_nodes(request.args.get('max_nodes'))
|
||||
level = sanitise_level(request.args.get('level'))
|
||||
|
||||
hidden = request.args.get('hidden')
|
||||
if hidden:
|
||||
hidden = set(hidden.split(','))
|
||||
else:
|
||||
hidden = set()
|
||||
|
||||
filter_types = ail_objects.sanitize_objs_types(request.args.get('filter', '').split(','))
|
||||
|
||||
json_graph = ail_objects.get_correlations_graph_node(obj_type, subtype, obj_id, filter_types=filter_types, max_nodes=max_nodes, level=level, flask_context=True)
|
||||
json_graph = ail_objects.get_correlations_graph_node(obj_type, subtype, obj_id, filter_types=filter_types, max_nodes=max_nodes, level=level, objs_hidden=hidden, flask_context=True)
|
||||
#json_graph = Correlate_object.get_graph_node_object_correlation(obj_type, obj_id, 'union', correlation_names, correlation_objects, requested_correl_type=subtype, max_nodes=max_nodes)
|
||||
return jsonify(json_graph)
|
||||
|
||||
|
|
@ -224,6 +248,7 @@ def correlation_tags_add():
|
|||
nb_max = sanitise_nb_max_nodes(request.form.get('tag_nb_max'))
|
||||
level = sanitise_level(request.form.get('tag_level'))
|
||||
filter_types = ail_objects.sanitize_objs_types(request.form.get('tag_filter', '').split(','))
|
||||
hidden = sanitise_objs_hidden(request.form.get('tag_hidden'))
|
||||
|
||||
if not ail_objects.exists_obj(obj_type, subtype, obj_id):
|
||||
return abort(404)
|
||||
|
|
@ -252,9 +277,11 @@ def correlation_tags_add():
|
|||
|
||||
if tags:
|
||||
ail_objects.obj_correlations_objs_add_tags(obj_type, subtype, obj_id, tags, filter_types=filter_types,
|
||||
objs_hidden=hidden,
|
||||
lvl=level + 1, nb_max=nb_max)
|
||||
return redirect(url_for('correlation.show_correlation',
|
||||
type=obj_type, subtype=subtype, id=obj_id,
|
||||
level=level,
|
||||
max_nodes=nb_max,
|
||||
hidden=hidden, hidden_str=",".join(hidden),
|
||||
filter=",".join(filter_types)))
|
||||
|
|
|
|||
|
|
@ -164,6 +164,11 @@
|
|||
<i class="fas fa-sync"></i> Resize Graph
|
||||
</button>
|
||||
</span>
|
||||
<span class="float-right mt-2 mx-1">
|
||||
<button class="btn btn-primary py-1" onclick="reset_graph();">
|
||||
<i class="fas fa-undo"></i> Reset Graph
|
||||
</button>
|
||||
</span>
|
||||
<div id="incomplete_graph" class="text-danger mt-3">
|
||||
<i class="fas fa-exclamation-triangle"></i> Graph Incomplete, Max Nodes Reached.
|
||||
</div>
|
||||
|
|
@ -180,6 +185,14 @@
|
|||
</div>
|
||||
</div>
|
||||
|
||||
<p>Press <b>H</b> on an object / node to hide it.</p>
|
||||
{% if dict_object["hidden"] %}
|
||||
<h5>Hidden objects:</h5>
|
||||
{% for obj_hidden in dict_object["hidden"] %}
|
||||
{{ obj_hidden }} <br>
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
|
||||
</div>
|
||||
|
||||
<div class="col-xl-2">
|
||||
|
|
@ -343,6 +356,7 @@
|
|||
<input type="hidden" id="tag_level" name="tag_level" value="{{dict_object["level"]}}">
|
||||
<input type="hidden" id="tag_nb_max" name="tag_nb_max" value="{{dict_object["max_nodes"]}}">
|
||||
<input type="hidden" id="filter" name="tag_filter" value="{{dict_object["filter_str"]}}">
|
||||
<input type="hidden" id="tag_hidden" name="tag_hidden" value="{{dict_object["hidden_str"]}}">
|
||||
{% include 'tags/block_tags_selector.html' %}
|
||||
<button class="btn btn-primary mt-2">
|
||||
<i class="fas fa-tag"></i> Add Tags
|
||||
|
|
@ -362,7 +376,7 @@ $(document).ready(function(){
|
|||
$("#incomplete_graph").hide();
|
||||
$("#page-Decoded").addClass("active");
|
||||
|
||||
all_graph.node_graph = create_graph("{{ url_for('correlation.graph_node_json') }}?id={{ dict_object["correlation_id"] }}&type={{ dict_object["object_type"] }}&mode={{ dict_object["mode"] }}&level={{ dict_object["level"] }}&filter={{ dict_object["filter_str"] }}&max_nodes={{dict_object["max_nodes"]}}{% if 'type_id' in dict_object["metadata"] %}&subtype={{ dict_object["metadata"]["type_id"] }}{% endif %}");
|
||||
all_graph.node_graph = create_graph("{{ url_for('correlation.graph_node_json') }}?id={{ dict_object["correlation_id"] }}&type={{ dict_object["object_type"] }}&mode={{ dict_object["mode"] }}&level={{ dict_object["level"] }}&filter={{ dict_object["filter_str"] }}&max_nodes={{dict_object["max_nodes"]}}{% if 'type_id' in dict_object["metadata"] %}&subtype={{ dict_object["metadata"]["type_id"] }}{% endif %}&hidden={{ dict_object["hidden_str"] }}");
|
||||
{% if dict_object["object_type"] in ["cryptocurrency", "pgp", "username"] %}
|
||||
all_graph.line_chart = create_line_chart('graph_line', "{{ url_for('objects_subtypes.objects_cve_graphline_json') }}?type={{ dict_object["object_type"] }}&subtype={{dict_object["metadata"]["type_id"]}}&id={{dict_object["correlation_id"]}}");
|
||||
{% elif dict_object["object_type"] == "decoded" %}
|
||||
|
|
@ -416,9 +430,18 @@ function SubmitAddTags() {
|
|||
zoom.scaleTo(svg_node, 2);
|
||||
}
|
||||
|
||||
function reset_graph() {
|
||||
window.location.href = "{{ url_for('correlation.show_correlation') }}?type={{ dict_object["type"] }}&subtype={{ dict_object["subtype"] }}&id={{ dict_object["id"] }}"
|
||||
}
|
||||
|
||||
</script>
|
||||
|
||||
<script>
|
||||
var correl_link = "{{ url_for('correlation.show_correlation') }}?type={{ dict_object["type"] }}&subtype={{ dict_object["subtype"] }}&id={{ dict_object["id"] }}&max_nodes={{ dict_object["max_nodes"] }}&level={{ dict_object["level"] }}&filter={{ dict_object["filter_str"] }}"
|
||||
{% if 'hidden_str' in dict_object %}
|
||||
correl_link = correl_link + "&hidden={{ dict_object["hidden_str"] }}"
|
||||
{% endif %}
|
||||
|
||||
var width = 400,
|
||||
height = 400;
|
||||
|
||||
|
|
@ -445,6 +468,8 @@ var simulation = d3.forceSimulation()
|
|||
.force("center", d3.forceCenter(width / 2, height / 2));
|
||||
//.on("tick", ticked);
|
||||
|
||||
var currentObject = null;
|
||||
|
||||
var svg_node = d3.select("#graph").append("svg")
|
||||
.attr("id", "graph_div")
|
||||
.attr("width", width)
|
||||
|
|
@ -540,6 +565,8 @@ d3.json(url)
|
|||
$("#incomplete_graph").show();
|
||||
}
|
||||
|
||||
d3.select("body").on("keypress", keypressed)
|
||||
|
||||
|
||||
|
||||
})
|
||||
|
|
@ -559,6 +586,16 @@ function doubleclick (d) {
|
|||
window.open(d.url, '_blank');
|
||||
}
|
||||
|
||||
function keypressed () {
|
||||
//console.log(d3.event.keyCode)
|
||||
//console.log(currentObject.id)
|
||||
// hide node, H or h key
|
||||
if ((d3.event.keyCode === 72 || d3.event.keyCode === 104) && currentObject) {
|
||||
window.location.href = correl_link + "&hide=" + currentObject.id
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
function click (d) {
|
||||
console.log('clicked')
|
||||
}
|
||||
|
|
@ -582,6 +619,8 @@ d.fy = d.y;
|
|||
|
||||
function mouseovered(d) {
|
||||
|
||||
currentObject = d;
|
||||
|
||||
var d3_pageX = d3.event.pageX;
|
||||
var d3_pageY = d3.event.pageY;
|
||||
|
||||
|
|
@ -666,6 +705,8 @@ if (d.popover) {
|
|||
}
|
||||
|
||||
function mouseouted() {
|
||||
currentObject = null;
|
||||
|
||||
div.transition()
|
||||
.duration(500)
|
||||
.style("opacity", 0);
|
||||
|
|
|
|||
Loading…
Reference in New Issue