chg: [correlation] correlation graph, add an option to hide an object/node by pressing H + reset correlation graph

bin/lib/correlations_engine.py

@@ -170,18 +170,18 @@ def get_obj_str_id(obj_type, subtype, obj_id):
         subtype = ''
     return f'{obj_type}:{subtype}:{obj_id}'
 
-def get_correlations_graph_nodes_links(obj_type, subtype, obj_id, filter_types=[], max_nodes=300, level=1, flask_context=False):
+def get_correlations_graph_nodes_links(obj_type, subtype, obj_id, filter_types=[], max_nodes=300, level=1, objs_hidden=set(), flask_context=False):
     links = set()
     nodes = set()
     meta = {'complete': True, 'objs': set()}
 
     obj_str_id = get_obj_str_id(obj_type, subtype, obj_id)
 
-    _get_correlations_graph_node(links, nodes, meta, obj_type, subtype, obj_id, level, max_nodes, filter_types=filter_types, previous_str_obj='')
+    _get_correlations_graph_node(links, nodes, meta, obj_type, subtype, obj_id, level, max_nodes, filter_types=filter_types, objs_hidden=objs_hidden, previous_str_obj='')
     return obj_str_id, nodes, links, meta
 
 
-def _get_correlations_graph_node(links, nodes, meta, obj_type, subtype, obj_id, level, max_nodes, filter_types=[], previous_str_obj=''):
+def _get_correlations_graph_node(links, nodes, meta, obj_type, subtype, obj_id, level, max_nodes, filter_types=[], objs_hidden=set(), previous_str_obj=''):
     obj_str_id = get_obj_str_id(obj_type, subtype, obj_id)
     meta['objs'].add(obj_str_id)
     nodes.add(obj_str_id)
@@ -192,6 +192,10 @@ def _get_correlations_graph_node(links, nodes, meta, obj_type, subtype, obj_id,
         for str_obj in obj_correlations[correl_type]:
             subtype2, obj2_id = str_obj.split(':', 1)
             obj2_str_id = get_obj_str_id(correl_type, subtype2, obj2_id)
+            # filter objects to hide
+            if obj2_str_id in objs_hidden:
+                continue
+
             meta['objs'].add(obj2_str_id)
 
             if obj2_str_id == previous_str_obj:
@@ -205,5 +209,5 @@ def _get_correlations_graph_node(links, nodes, meta, obj_type, subtype, obj_id,
 
             if level > 0:
                 next_level = level - 1
-                _get_correlations_graph_node(links, nodes, meta, correl_type, subtype2, obj2_id, next_level, max_nodes, filter_types=filter_types, previous_str_obj=obj_str_id)
+                _get_correlations_graph_node(links, nodes, meta, correl_type, subtype2, obj2_id, next_level, max_nodes, filter_types=filter_types, objs_hidden=objs_hidden, previous_str_obj=obj_str_id)
 

bin/lib/objects/ail_objects.py

@@ -338,7 +338,7 @@ def get_obj_correlations(obj_type, subtype, obj_id):
     obj = get_object(obj_type, subtype, obj_id)
     return obj.get_correlations()
 
-def _get_obj_correlations_objs(objs, obj_type, subtype, obj_id, filter_types, lvl, nb_max):
+def _get_obj_correlations_objs(objs, obj_type, subtype, obj_id, filter_types, lvl, nb_max, objs_hidden):
     if len(objs) < nb_max or nb_max == 0:
         if lvl == 0:
             objs.add((obj_type, subtype, obj_id))
@@ -351,16 +351,17 @@ def _get_obj_correlations_objs(objs, obj_type, subtype, obj_id, filter_types, lv
             for obj2_type in correlations:
                 for str_obj in correlations[obj2_type]:
                     obj2_subtype, obj2_id = str_obj.split(':', 1)
-                    _get_obj_correlations_objs(objs, obj2_type, obj2_subtype, obj2_id, filter_types, lvl, nb_max)
+                    if get_obj_global_id(obj2_type, obj2_subtype, obj2_id) in objs_hidden:
+                        continue  # filter object to hide
+                    _get_obj_correlations_objs(objs, obj2_type, obj2_subtype, obj2_id, filter_types, lvl, nb_max, objs_hidden)
 
-def get_obj_correlations_objs(obj_type, subtype, obj_id, filter_types=[], lvl=0, nb_max=300):
+def get_obj_correlations_objs(obj_type, subtype, obj_id, filter_types=[], lvl=0, nb_max=300, objs_hidden=set()):
     objs = set()
-    _get_obj_correlations_objs(objs, obj_type, subtype, obj_id, filter_types, lvl, nb_max)
+    _get_obj_correlations_objs(objs, obj_type, subtype, obj_id, filter_types, lvl, nb_max, objs_hidden)
     return objs
 
-def obj_correlations_objs_add_tags(obj_type, subtype, obj_id, tags, filter_types=[], lvl=0, nb_max=300):
+def obj_correlations_objs_add_tags(obj_type, subtype, obj_id, tags, filter_types=[], lvl=0, nb_max=300, objs_hidden=set()):
-    print(nb_max)
+    objs = get_obj_correlations_objs(obj_type, subtype, obj_id, filter_types=filter_types, lvl=lvl, nb_max=nb_max, objs_hidden=objs_hidden)
-    objs = get_obj_correlations_objs(obj_type, subtype, obj_id, filter_types=filter_types, lvl=lvl, nb_max=nb_max)
     # print(objs)
     for obj_tuple in objs:
         obj1_type, subtype1, id1 = obj_tuple
@@ -422,10 +423,12 @@ def create_correlation_graph_nodes(nodes_set, obj_str_id, flask_context=True):
 
 
 def get_correlations_graph_node(obj_type, subtype, obj_id, filter_types=[], max_nodes=300, level=1,
+                                objs_hidden=set(),
                                 flask_context=False):
     obj_str_id, nodes, links, meta = correlations_engine.get_correlations_graph_nodes_links(obj_type, subtype, obj_id,
                                                                                             filter_types=filter_types,
                                                                                             max_nodes=max_nodes, level=level,
+                                                                                            objs_hidden=objs_hidden,
                                                                                             flask_context=flask_context)
     # print(meta)
     meta['objs'] = list(meta['objs'])

var/www/blueprints/correlation.py

@@ -61,6 +61,13 @@ def sanitise_level(level):
         level = 2
     return level
 
+def sanitise_objs_hidden(objs_hidden):
+    if objs_hidden:
+        objs_hidden = set(objs_hidden.split(','))  # TODO sanitize objects
+    else:
+        objs_hidden = set()
+    return objs_hidden
+
 # ============= ROUTES ==============
 @correlation.route('/correlation/show', methods=['GET', 'POST'])
 @login_required
@@ -130,6 +137,10 @@ def show_correlation():
         max_nodes = sanitise_nb_max_nodes(request.args.get('max_nodes'))
         mode = sanitise_graph_mode(request.args.get('mode'))
         level = sanitise_level(request.args.get('level'))
+        objs_hidden = sanitise_objs_hidden(request.args.get('hidden'))
+        obj_to_hide = request.args.get('hide')
+        if obj_to_hide:
+            objs_hidden.add(obj_to_hide)
 
         related_btc = bool(request.args.get('related_btc', False))
 
@@ -139,17 +150,24 @@ def show_correlation():
         if not ail_objects.exists_obj(obj_type, subtype, obj_id):
             return abort(404)
         # object exist
-        else:
+        else: # TODO remove old dict key
-            dict_object = {"object_type": obj_type,
+            dict_object = {"type": obj_type,
-                           "correlation_id": obj_id,
+                           "id": obj_id,
+                           "object_type": obj_type,
                            "max_nodes": max_nodes, "mode": mode, "level": level,
                            "filter": filter_types, "filter_str": ",".join(filter_types),
+                           "hidden": objs_hidden, "hidden_str": ",".join(objs_hidden),
+
+                           "correlation_id": obj_id,
                            "metadata": ail_objects.get_object_meta(obj_type, subtype, obj_id,
                                                                    options={'tags'}, flask_context=True),
                            "nb_correl": ail_objects.get_obj_nb_correlations(obj_type, subtype, obj_id)
                            }
             if subtype:
+                dict_object["subtype"] = subtype
                 dict_object["metadata"]['type_id'] = subtype
+            else:
+                dict_object["subtype"] = ''
             dict_object["metadata_card"] = ail_objects.get_object_card_meta(obj_type, subtype, obj_id, related_btc=related_btc)
             return render_template("show_correlation.html", dict_object=dict_object, bootstrap_label=bootstrap_label,
                                    tags_selector_data=Tag.get_tags_selector_data())
@@ -194,9 +212,15 @@ def graph_node_json():
     max_nodes = sanitise_nb_max_nodes(request.args.get('max_nodes'))
     level = sanitise_level(request.args.get('level'))
 
+    hidden = request.args.get('hidden')
+    if hidden:
+        hidden = set(hidden.split(','))
+    else:
+        hidden = set()
+
     filter_types = ail_objects.sanitize_objs_types(request.args.get('filter', '').split(','))
 
-    json_graph = ail_objects.get_correlations_graph_node(obj_type, subtype, obj_id, filter_types=filter_types, max_nodes=max_nodes, level=level, flask_context=True)
+    json_graph = ail_objects.get_correlations_graph_node(obj_type, subtype, obj_id, filter_types=filter_types, max_nodes=max_nodes, level=level, objs_hidden=hidden, flask_context=True)
     #json_graph = Correlate_object.get_graph_node_object_correlation(obj_type, obj_id, 'union', correlation_names, correlation_objects, requested_correl_type=subtype, max_nodes=max_nodes)
     return jsonify(json_graph)
 
@@ -224,6 +248,7 @@ def correlation_tags_add():
     nb_max = sanitise_nb_max_nodes(request.form.get('tag_nb_max'))
     level = sanitise_level(request.form.get('tag_level'))
     filter_types = ail_objects.sanitize_objs_types(request.form.get('tag_filter', '').split(','))
+    hidden = sanitise_objs_hidden(request.form.get('tag_hidden'))
 
     if not ail_objects.exists_obj(obj_type, subtype, obj_id):
         return abort(404)
@@ -252,9 +277,11 @@ def correlation_tags_add():
 
     if tags:
         ail_objects.obj_correlations_objs_add_tags(obj_type, subtype, obj_id, tags, filter_types=filter_types,
+                                                   objs_hidden=hidden,
                                                    lvl=level + 1, nb_max=nb_max)
     return redirect(url_for('correlation.show_correlation',
                             type=obj_type, subtype=subtype, id=obj_id,
                             level=level,
                             max_nodes=nb_max,
+                            hidden=hidden, hidden_str=",".join(hidden),
                             filter=",".join(filter_types)))

var/www/templates/correlation/show_correlation.html

@@ -164,6 +164,11 @@
 											<i class="fas fa-sync"></i>&nbsp;Resize Graph
 										</button>
 									</span>
+                                    <span class="float-right mt-2 mx-1">
+										<button class="btn btn-primary py-1" onclick="reset_graph();">
+											<i class="fas fa-undo"></i>&nbsp;Reset Graph
+										</button>
+									</span>
                                     <div id="incomplete_graph" class="text-danger mt-3">
                                         <i class="fas fa-exclamation-triangle"></i>&nbsp;Graph Incomplete, Max Nodes Reached.
                                     </div>
@@ -180,6 +185,14 @@
 								</div>
 							</div>
 
+                            <p>Press <b>H</b> on an object / node to hide it.</p>
+                            {% if dict_object["hidden"] %}
+                                <h5>Hidden objects:</h5>
+                                {% for obj_hidden in dict_object["hidden"] %}
+                                    {{ obj_hidden }} <br>
+                                {% endfor %}
+                            {% endif %}
+
 						</div>
 
 						<div class="col-xl-2">
@@ -343,6 +356,7 @@
                                 <input type="hidden" id="tag_level" name="tag_level" value="{{dict_object["level"]}}">
                                 <input type="hidden" id="tag_nb_max" name="tag_nb_max" value="{{dict_object["max_nodes"]}}">
                                 <input type="hidden" id="filter" name="tag_filter" value="{{dict_object["filter_str"]}}">
+                                <input type="hidden" id="tag_hidden" name="tag_hidden" value="{{dict_object["hidden_str"]}}">
                                 {% include 'tags/block_tags_selector.html' %}
                                 <button class="btn btn-primary mt-2">
 									<i class="fas fa-tag"></i> Add Tags
@@ -362,7 +376,7 @@ $(document).ready(function(){
         $("#incomplete_graph").hide();
 		$("#page-Decoded").addClass("active");
 
-		all_graph.node_graph = create_graph("{{ url_for('correlation.graph_node_json') }}?id={{ dict_object["correlation_id"] }}&type={{ dict_object["object_type"] }}&mode={{ dict_object["mode"] }}&level={{ dict_object["level"] }}&filter={{ dict_object["filter_str"] }}&max_nodes={{dict_object["max_nodes"]}}{% if 'type_id' in dict_object["metadata"] %}&subtype={{ dict_object["metadata"]["type_id"] }}{% endif %}");
+		all_graph.node_graph = create_graph("{{ url_for('correlation.graph_node_json') }}?id={{ dict_object["correlation_id"] }}&type={{ dict_object["object_type"] }}&mode={{ dict_object["mode"] }}&level={{ dict_object["level"] }}&filter={{ dict_object["filter_str"] }}&max_nodes={{dict_object["max_nodes"]}}{% if 'type_id' in dict_object["metadata"] %}&subtype={{ dict_object["metadata"]["type_id"] }}{% endif %}&hidden={{ dict_object["hidden_str"] }}");
 		{% if dict_object["object_type"] in ["cryptocurrency", "pgp", "username"] %}
 			all_graph.line_chart = create_line_chart('graph_line', "{{ url_for('objects_subtypes.objects_cve_graphline_json') }}?type={{ dict_object["object_type"] }}&subtype={{dict_object["metadata"]["type_id"]}}&id={{dict_object["correlation_id"]}}");
 		{% elif dict_object["object_type"] == "decoded" %}
@@ -416,9 +430,18 @@ function SubmitAddTags() {
 		zoom.scaleTo(svg_node, 2);
 	}
 
+    function reset_graph() {
+        window.location.href = "{{ url_for('correlation.show_correlation') }}?type={{ dict_object["type"] }}&subtype={{ dict_object["subtype"] }}&id={{ dict_object["id"] }}"
+    }
+
 </script>
 
 <script>
+var correl_link = "{{ url_for('correlation.show_correlation') }}?type={{ dict_object["type"] }}&subtype={{ dict_object["subtype"] }}&id={{ dict_object["id"] }}&max_nodes={{ dict_object["max_nodes"] }}&level={{ dict_object["level"] }}&filter={{ dict_object["filter_str"] }}"
+{% if 'hidden_str' in dict_object %}
+    correl_link = correl_link + "&hidden={{ dict_object["hidden_str"] }}"
+{% endif %}
+
 var width = 400,
 		height = 400;
 
@@ -445,6 +468,8 @@ var simulation = d3.forceSimulation()
 		.force("center", d3.forceCenter(width / 2, height / 2));
 		//.on("tick", ticked);
 
+var currentObject = null;
+
 var svg_node = d3.select("#graph").append("svg")
 		.attr("id", "graph_div")
 		.attr("width", width)
@@ -540,6 +565,8 @@ d3.json(url)
         $("#incomplete_graph").show();
     }
 
+    d3.select("body").on("keypress", keypressed)
+
 
 
 })
@@ -559,6 +586,16 @@ function doubleclick (d) {
 	window.open(d.url, '_blank');
 }
 
+function keypressed () {
+    //console.log(d3.event.keyCode)
+    //console.log(currentObject.id)
+    // hide node, H or h key
+    if ((d3.event.keyCode === 72 || d3.event.keyCode === 104) && currentObject) {
+        window.location.href = correl_link + "&hide=" + currentObject.id
+    }
+
+}
+
 function click (d) {
 console.log('clicked')
 }
@@ -582,6 +619,8 @@ d.fy = d.y;
 
 function mouseovered(d) {
 
+currentObject = d;
+
 var d3_pageX = d3.event.pageX;
 var d3_pageY = d3.event.pageY;
 
@@ -666,6 +705,8 @@ if (d.popover) {
 }
 
 function mouseouted() {
+    currentObject = null;
+
 	div.transition()
 		.duration(500)
 		.style("opacity", 0);