fix: [UI matches extractor] handle overlapping matches

bin/lib/module_extractor.py

@@ -62,6 +62,24 @@ tools = Tools(queue=False)
 for tool_name in tools.get_tools():
     MODULES[f'infoleak:automatic-detection="{tool_name}-tool"'] = tools
 
+def merge_overlap(extracted):
+    merged = []
+    curr_start, curr_end, curr_string_match, curr_obj_ref = extracted[0]
+    curr_obj_ref = [(curr_obj_ref, curr_string_match)]
+
+    for start, end, mstring, ref in extracted[1:]:
+        # overlap
+        if start <= curr_end:
+            curr_string_match += mstring[curr_end - start:]
+            curr_end = max(curr_end, end)
+            curr_obj_ref.append((ref, mstring))
+        else:
+            merged.append((curr_start, curr_end, curr_string_match, curr_obj_ref))
+            curr_start, curr_end, curr_string_match, curr_obj_ref = start, end, mstring, [(ref, mstring)]
+
+    merged.append((curr_start, curr_end, curr_string_match, curr_obj_ref))
+    return merged
+
 def get_correl_match(extract_type, obj, content):
     extracted = []
     correl = correlations_engine.get_correlation_by_correl_type(obj.type, obj.get_subtype(r_str=True), obj.id, extract_type)
@@ -81,6 +99,8 @@ def get_correl_match(extract_type, obj, content):
         map_value_id[sha256_val] = value
     if to_extract:
         objs = regex_helper.regex_finditer(r_key, '|'.join(to_extract), obj.get_global_id(), content)
+        if extract_type == 'title' and objs:
+            objs = [objs[0]]
         for ob in objs:
             if map_subtype.get(ob[2]):
                 subtype = map_subtype[ob[2]]
@@ -223,7 +243,7 @@ def extract(obj_type, subtype, obj_id, content=None):
 
     # SORT By Start Pos
     extracted = sorted(extracted, key=itemgetter(0))
-    # print(extracted)
+    extracted = merge_overlap(extracted)
 
     # Save In Cache
     if extracted:
@@ -236,43 +256,46 @@ def extract(obj_type, subtype, obj_id, content=None):
 # TODO ADD LINK UI
 def get_extracted_by_match(extracted):
     matches = {}
-    for start, end, value, str_obj in extracted:
+    for start, end, value, raw_objs in extracted:
 
-        if str_obj not in matches:
+        for raw in raw_objs:
-            matches[str_obj] = {}
+            str_obj, str_match = raw
-            ob_type, row_id = str_obj.split(':', 1)
+
-            if ob_type == 'tag':  # TODO put me in object class
+            if str_obj not in matches:
-                matches[str_obj]['subtype'] = 'tag'
+                matches[str_obj] = {}
-                matches[str_obj]['id'] = row_id
+                ob_type, row_id = str_obj.split(':', 1)
-                matches[str_obj]['icon'] = {'style': 'fas', 'icon': '\uf02b', 'color': '#28a745', 'radius': 5}
+                if ob_type == 'tag':  # TODO put me in object class
-                matches[str_obj]['link'] = ''
+                    matches[str_obj]['subtype'] = 'tag'
-            elif ob_type == 'tracker':  # TODO put me in object class
+                    matches[str_obj]['id'] = row_id
-                matches[str_obj]['subtype'] = 'tracker'
+                    matches[str_obj]['icon'] = {'style': 'fas', 'icon': '\uf02b', 'color': '#28a745', 'radius': 5}
-                matches[str_obj]['id'] = row_id
+                    matches[str_obj]['link'] = ''
-                matches[str_obj]['icon'] = {'style': 'fas', 'icon': '\uf05b', 'color': '#ffc107', 'radius': 5}
+                elif ob_type == 'tracker':  # TODO put me in object class
-                matches[str_obj]['link'] = ''
+                    matches[str_obj]['subtype'] = 'tracker'
-            elif ob_type == 'retro_hunt':  # TODO put me in object class
+                    matches[str_obj]['id'] = row_id
-                matches[str_obj]['subtype'] = 'retro_hunt'
+                    matches[str_obj]['icon'] = {'style': 'fas', 'icon': '\uf05b', 'color': '#ffc107', 'radius': 5}
-                matches[str_obj]['id'] = row_id
+                    matches[str_obj]['link'] = ''
-                matches[str_obj]['icon'] = {'style': 'fas', 'icon': '\uf05b', 'color': '#008107', 'radius': 5}
+                elif ob_type == 'retro_hunt':  # TODO put me in object class
-                matches[str_obj]['link'] = ''
+                    matches[str_obj]['subtype'] = 'retro_hunt'
-            else:
+                    matches[str_obj]['id'] = row_id
-                row_id = row_id.split(':', 1)
+                    matches[str_obj]['icon'] = {'style': 'fas', 'icon': '\uf05b', 'color': '#008107', 'radius': 5}
-                if len(row_id) == 2:
+                    matches[str_obj]['link'] = ''
-                    subtype = row_id[0]
-                    obj_id = row_id[1]
                 else:
-                    subtype = ''
+                    row_id = row_id.split(':', 1)
-                    obj_id = row_id[0]
+                    if len(row_id) == 2:
-                matches[str_obj]['subtype'] = subtype
+                        subtype = row_id[0]
-                matches[str_obj]['id'] = obj_id
+                        obj_id = row_id[1]
-                matches[str_obj]['icon'] = ail_objects.get_object_svg(ob_type, subtype, obj_id)
+                    else:
-                matches[str_obj]['link'] = ail_objects.get_object_link(ob_type, subtype, obj_id)
+                        subtype = ''
+                        obj_id = row_id[0]
+                    matches[str_obj]['subtype'] = subtype
+                    matches[str_obj]['id'] = obj_id
+                    matches[str_obj]['icon'] = ail_objects.get_object_svg(ob_type, subtype, obj_id)
+                    matches[str_obj]['link'] = ail_objects.get_object_link(ob_type, subtype, obj_id)
 
-            matches[str_obj]['matches'] = []
+                matches[str_obj]['matches'] = []
 
-        match = [start, end, value]
+            match = [start, end, str_match]
-        matches[str_obj]['matches'].append(match)
+            matches[str_obj]['matches'].append(match)
     return matches
 
 

var/www/templates/chats_explorer/ChatMessage.html

@@ -225,13 +225,12 @@
     $(document).ready(function(){
         $("#page-Decoded").addClass("active");
         $("#nav_chat").addClass("active");
+        $('[data-toggle="popover"]').popover({
+            boundary:'window',
+        });
 
   });
 
-$(function () {
-    $('[data-toggle="popover"]').popover()
-})
-
 function toggle_sidebar(){
 	if($('#nav_menu').is(':visible')){
 		$('#nav_menu').hide();

var/www/templates/chats_explorer/block_message.html

@@ -79,7 +79,7 @@
         {% if not message['extracted'] %}
             <pre class="my-0">{{ message['content'] }}</pre>
         {% else %}
-            <pre class="my-0">{{ message['content'][:message['extracted'][0][0]] }}{% for row in message['extracted'] %}<span class="hg-text" data-toggle="popover" data-trigger="hover" data-html="true" title="<svg height=&quot;26&quot; width=&quot;26&quot;><g class=&quot;nodes&quot;><circle cx=&quot;13&quot; cy=&quot;13&quot; r=&quot;13&quot; fill=&quot;{{ message['extracted_matches'][row[3]]['icon']['color'] }}&quot;></circle><text x=&quot;13&quot; y=&quot;13&quot; text-anchor=&quot;middle&quot; dominant-baseline=&quot;central&quot; class=&quot;graph_node_icon {{ message['extracted_matches'][row[3]]['icon']['style'] }}&quot; font-size=&quot;16px&quot;>{{ message['extracted_matches'][row[3]]['icon']['icon'] }}</text></g></svg> {{ message['extracted_matches'][row[3]]['subtype'] }}" data-content="{{ message['extracted_matches'][row[3]]['id'] }}" id="{{ row[0] }}:{{ row[1] }}">{{ message['content'][row[0]:row[1]] }}</span>{% if loop.index + 1 > message['extracted']|length %}{{ message['content'][message['extracted'][-1][1]:] }}{% else %}{{ message['content'][row[1]:message['extracted'][loop.index][0]] }}{% endif %}{% endfor %}</pre>
+            <pre class="my-0">{{ message['content'][:message['extracted'][0][0]] }}{% for row in message['extracted'] %}<span class="hg-text" data-toggle="popover" data-trigger="hover" data-html="true" title="Extracted:" data-content="<ul class=&quot;list-group&quot;>{% for r in row[3] %}<li class=&quot;list-group-item&quot;><div><svg height=&quot;26&quot; width=&quot;26&quot;><g class=&quot;nodes&quot;><circle cx=&quot;13&quot; cy=&quot;13&quot; r=&quot;13&quot; fill=&quot;{{ message['extracted_matches'][r[0]]['icon']['color'] }}&quot;></circle><text x=&quot;13&quot; y=&quot;13&quot; text-anchor=&quot;middle&quot; dominant-baseline=&quot;central&quot; class=&quot;{{ message['extracted_matches'][r[0]]['icon']['style'] }}&quot; font-size=&quot;16px&quot;>{{ message['extracted_matches'][r[0]]['icon']['icon'] }}</text></g></svg> {{ message['extracted_matches'][r[0]]['subtype'] }}</div>{{ message['extracted_matches'][r[0]]['id'] }} <div><b>{{ r[1] }}</b></div></li>{% endfor %}</ul>" id="{{ row[0] }}:{{ row[1] }}">{{ message['content'][row[0]:row[1]] }}</span>{% if loop.index + 1 > message['extracted']|length %}{{ message['content'][message['extracted'][-1][1]:] }}{% else %}{{ message['content'][row[1]:message['extracted'][loop.index][0]] }}{% endif %}{% endfor %}</pre>
         {% endif %}
         {% if message['translation'] %}
             <hr class="m-1">

var/www/templates/objects/item/show_item.html

@@ -547,7 +547,7 @@
             {% if not extracted %}
                 <p class="my-0"> <pre class="border">{{ meta['content'] }}</pre></p>
             {% else %}
-                <p class="my-0"> <pre class="border">{{ meta['content'][:extracted[0][0]] }}{% for row in extracted %}<span class="hg-text" data-toggle="popover" data-trigger="hover" data-html="true" title="<svg height=&quot;26&quot; width=&quot;26&quot;><g class=&quot;nodes&quot;><circle cx=&quot;13&quot; cy=&quot;13&quot; r=&quot;13&quot; fill=&quot;{{ extracted_matches[row[3]]['icon']['color'] }}&quot;></circle><text x=&quot;13&quot; y=&quot;13&quot; text-anchor=&quot;middle&quot; dominant-baseline=&quot;central&quot; class=&quot;graph_node_icon {{ extracted_matches[row[3]]['icon']['style'] }}&quot; font-size=&quot;16px&quot;>{{ extracted_matches[row[3]]['icon']['icon'] }}</text></g></svg> {{ extracted_matches[row[3]]['subtype'] }}" data-content="{{ extracted_matches[row[3]]['id'] }}" id="{{ row[0] }}:{{ row[1] }}">{{ meta['content'][row[0]:row[1]] }}</span>{% if loop.index + 1 > extracted|length %}{{ meta['content'][extracted[-1][1]:] }}{% else %}{{ meta['content'][row[1]:extracted[loop.index][0]] }}{% endif %}{% endfor %}</pre></p>
+                <p class="my-0"> <pre class="border">{{ meta['content'][:extracted[0][0]] }}{% for row in extracted %}<span class="hg-text" data-toggle="popover" data-trigger="hover" data-html="true" title="Extracted:" data-content="<ul class=&quot;list-group&quot;>{% for r in row[3] %}<li class=&quot;list-group-item&quot;><div><svg height=&quot;26&quot; width=&quot;26&quot;><g class=&quot;nodes&quot;><circle cx=&quot;13&quot; cy=&quot;13&quot; r=&quot;13&quot; fill=&quot;{{ extracted_matches[r[0]]['icon']['color'] }}&quot;></circle><text x=&quot;13&quot; y=&quot;13&quot; text-anchor=&quot;middle&quot; dominant-baseline=&quot;central&quot; class=&quot;{{ extracted_matches[r[0]]['icon']['style'] }}&quot; font-size=&quot;16px&quot;>{{ extracted_matches[r[0]]['icon']['icon'] }}</text></g></svg> {{ extracted_matches[r[0]]['subtype'] }}</div>{{ extracted_matches[r[0]]['id'] }} <div><b>{{ r[1] }}</b></div></li>{% endfor %}</ul>" id="{{ row[0] }}:{{ row[1] }}">{{ meta['content'][row[0]:row[1]] }}</span>{% if loop.index + 1 > extracted|length %}{{ meta['content'][extracted[-1][1]:] }}{% else %}{{ meta['content'][row[1]:extracted[loop.index][0]] }}{% endif %}{% endfor %}</pre></p>
             {% endif %}
         </div>
         <div class="tab-pane fade" id="pills-html2text" role="tabpanel" aria-labelledby="pills-html2text-tab">
@@ -576,7 +576,10 @@
         $(".rotate").click(function(){
             $(this).toggleClass("down");
         })
-      });
+        $('[data-toggle="popover"]').popover({
+            boundary:'window',
+        })
+    });
 
       $('#pills-html2text-tab').on('shown.bs.tab', function (e) {
         if ($('#html2text-container').is(':empty')){
@@ -639,9 +642,6 @@
   }
 
   blocks.addEventListener('change', pixelate, false);
-  $(function () {
-    $('[data-toggle="popover"]').popover()
-  })
   </script>
 {% endif %}