CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

fix: [UI matches extractor] handle overlapping matches

ocr
terrtia 2024-03-27 16:30:29 +01:00
parent 5ec0d7f0cf
commit 5fce682541
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
4 changed files with 66 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
bin/lib/module_extractor.py
View File

@ -62,6 +62,24 @@ tools = Tools(queue=False)
for tool_name in tools.get_tools():
MODULES[f'infoleak:automatic-detection="{tool_name}-tool"'] = tools
def merge_overlap(extracted):
merged = []
curr_start, curr_end, curr_string_match, curr_obj_ref = extracted[0]
curr_obj_ref = [(curr_obj_ref, curr_string_match)]
for start, end, mstring, ref in extracted[1:]:
# overlap
if start <= curr_end:
curr_string_match += mstring[curr_end - start:]
curr_end = max(curr_end, end)
curr_obj_ref.append((ref, mstring))
else:
merged.append((curr_start, curr_end, curr_string_match, curr_obj_ref))
curr_start, curr_end, curr_string_match, curr_obj_ref = start, end, mstring, [(ref, mstring)]
merged.append((curr_start, curr_end, curr_string_match, curr_obj_ref))
return merged
def get_correl_match(extract_type, obj, content):
extracted = []
correl = correlations_engine.get_correlation_by_correl_type(obj.type, obj.get_subtype(r_str=True), obj.id, extract_type)
@ -81,6 +99,8 @@ def get_correl_match(extract_type, obj, content):
map_value_id[sha256_val] = value
if to_extract:
objs = regex_helper.regex_finditer(r_key, '|'.join(to_extract), obj.get_global_id(), content)
if extract_type == 'title' and objs:
objs = [objs[0]]
for ob in objs:
if map_subtype.get(ob[2]):
subtype = map_subtype[ob[2]]
@ -223,7 +243,7 @@ def extract(obj_type, subtype, obj_id, content=None):
# SORT By Start Pos
extracted = sorted(extracted, key=itemgetter(0))
# print(extracted)
extracted = merge_overlap(extracted)
# Save In Cache
if extracted:
@ -236,7 +256,10 @@ def extract(obj_type, subtype, obj_id, content=None):
# TODO ADD LINK UI
def get_extracted_by_match(extracted):
matches = {}
for start, end, value, str_obj in extracted:
for start, end, value, raw_objs in extracted:
for raw in raw_objs:
str_obj, str_match = raw
if str_obj not in matches:
matches[str_obj] = {}
@ -271,7 +294,7 @@ def get_extracted_by_match(extracted):
matches[str_obj]['matches'] = []
match = [start, end, value]
match = [start, end, str_match]
matches[str_obj]['matches'].append(match)
return matches

7
var/www/templates/chats_explorer/ChatMessage.html
View File

@ -225,12 +225,11 @@
$(document).ready(function(){
$("#page-Decoded").addClass("active");
$("#nav_chat").addClass("active");
$('[data-toggle="popover"]').popover({
boundary:'window',
});
$(function () {
$('[data-toggle="popover"]').popover()
})
});
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){

2
var/www/templates/chats_explorer/block_message.html
View File

@ -79,7 +79,7 @@
{% if not message['extracted'] %}
<pre class="my-0">{{ message['content'] }}</pre>
{% else %}
<pre class="my-0">{{ message['content'][:message['extracted'][0][0]] }}{% for row in message['extracted'] %}<span class="hg-text" data-toggle="popover" data-trigger="hover" data-html="true" title="<svg height=&quot;26&quot; width=&quot;26&quot;><g class=&quot;nodes&quot;><circle cx=&quot;13&quot; cy=&quot;13&quot; r=&quot;13&quot; fill=&quot;{{ message['extracted_matches'][row[3]]['icon']['color'] }}&quot;></circle><text x=&quot;13&quot; y=&quot;13&quot; text-anchor=&quot;middle&quot; dominant-baseline=&quot;central&quot; class=&quot;graph_node_icon {{ message['extracted_matches'][row[3]]['icon']['style'] }}&quot; font-size=&quot;16px&quot;>{{ message['extracted_matches'][row[3]]['icon']['icon'] }}</text></g></svg> {{ message['extracted_matches'][row[3]]['subtype'] }}" data-content="{{ message['extracted_matches'][row[3]]['id'] }}" id="{{ row[0] }}:{{ row[1] }}">{{ message['content'][row[0]:row[1]] }}</span>{% if loop.index + 1 > message['extracted']|length %}{{ message['content'][message['extracted'][-1][1]:] }}{% else %}{{ message['content'][row[1]:message['extracted'][loop.index][0]] }}{% endif %}{% endfor %}</pre>
<pre class="my-0">{{ message['content'][:message['extracted'][0][0]] }}{% for row in message['extracted'] %}<span class="hg-text" data-toggle="popover" data-trigger="hover" data-html="true" title="Extracted:" data-content="<ul class=&quot;list-group&quot;>{% for r in row[3] %}<li class=&quot;list-group-item&quot;><div><svg height=&quot;26&quot; width=&quot;26&quot;><g class=&quot;nodes&quot;><circle cx=&quot;13&quot; cy=&quot;13&quot; r=&quot;13&quot; fill=&quot;{{ message['extracted_matches'][r[0]]['icon']['color'] }}&quot;></circle><text x=&quot;13&quot; y=&quot;13&quot; text-anchor=&quot;middle&quot; dominant-baseline=&quot;central&quot; class=&quot;{{ message['extracted_matches'][r[0]]['icon']['style'] }}&quot; font-size=&quot;16px&quot;>{{ message['extracted_matches'][r[0]]['icon']['icon'] }}</text></g></svg> {{ message['extracted_matches'][r[0]]['subtype'] }}</div>{{ message['extracted_matches'][r[0]]['id'] }} <div><b>{{ r[1] }}</b></div></li>{% endfor %}</ul>" id="{{ row[0] }}:{{ row[1] }}">{{ message['content'][row[0]:row[1]] }}</span>{% if loop.index + 1 > message['extracted']|length %}{{ message['content'][message['extracted'][-1][1]:] }}{% else %}{{ message['content'][row[1]:message['extracted'][loop.index][0]] }}{% endif %}{% endfor %}</pre>
{% endif %}
{% if message['translation'] %}
<hr class="m-1">

8
var/www/templates/objects/item/show_item.html
View File

@ -547,7 +547,7 @@
{% if not extracted %}
<p class="my-0"> <pre class="border">{{ meta['content'] }}</pre></p>
{% else %}
<p class="my-0"> <pre class="border">{{ meta['content'][:extracted[0][0]] }}{% for row in extracted %}<span class="hg-text" data-toggle="popover" data-trigger="hover" data-html="true" title="<svg height=&quot;26&quot; width=&quot;26&quot;><g class=&quot;nodes&quot;><circle cx=&quot;13&quot; cy=&quot;13&quot; r=&quot;13&quot; fill=&quot;{{ extracted_matches[row[3]]['icon']['color'] }}&quot;></circle><text x=&quot;13&quot; y=&quot;13&quot; text-anchor=&quot;middle&quot; dominant-baseline=&quot;central&quot; class=&quot;graph_node_icon {{ extracted_matches[row[3]]['icon']['style'] }}&quot; font-size=&quot;16px&quot;>{{ extracted_matches[row[3]]['icon']['icon'] }}</text></g></svg> {{ extracted_matches[row[3]]['subtype'] }}" data-content="{{ extracted_matches[row[3]]['id'] }}" id="{{ row[0] }}:{{ row[1] }}">{{ meta['content'][row[0]:row[1]] }}</span>{% if loop.index + 1 > extracted|length %}{{ meta['content'][extracted[-1][1]:] }}{% else %}{{ meta['content'][row[1]:extracted[loop.index][0]] }}{% endif %}{% endfor %}</pre></p>
<p class="my-0"> <pre class="border">{{ meta['content'][:extracted[0][0]] }}{% for row in extracted %}<span class="hg-text" data-toggle="popover" data-trigger="hover" data-html="true" title="Extracted:" data-content="<ul class=&quot;list-group&quot;>{% for r in row[3] %}<li class=&quot;list-group-item&quot;><div><svg height=&quot;26&quot; width=&quot;26&quot;><g class=&quot;nodes&quot;><circle cx=&quot;13&quot; cy=&quot;13&quot; r=&quot;13&quot; fill=&quot;{{ extracted_matches[r[0]]['icon']['color'] }}&quot;></circle><text x=&quot;13&quot; y=&quot;13&quot; text-anchor=&quot;middle&quot; dominant-baseline=&quot;central&quot; class=&quot;{{ extracted_matches[r[0]]['icon']['style'] }}&quot; font-size=&quot;16px&quot;>{{ extracted_matches[r[0]]['icon']['icon'] }}</text></g></svg> {{ extracted_matches[r[0]]['subtype'] }}</div>{{ extracted_matches[r[0]]['id'] }} <div><b>{{ r[1] }}</b></div></li>{% endfor %}</ul>" id="{{ row[0] }}:{{ row[1] }}">{{ meta['content'][row[0]:row[1]] }}</span>{% if loop.index + 1 > extracted|length %}{{ meta['content'][extracted[-1][1]:] }}{% else %}{{ meta['content'][row[1]:extracted[loop.index][0]] }}{% endif %}{% endfor %}</pre></p>
{% endif %}
</div>
<div class="tab-pane fade" id="pills-html2text" role="tabpanel" aria-labelledby="pills-html2text-tab">
@ -576,6 +576,9 @@
$(".rotate").click(function(){
$(this).toggleClass("down");
})
$('[data-toggle="popover"]').popover({
boundary:'window',
})
});
$('#pills-html2text-tab').on('shown.bs.tab', function (e) {
@ -639,9 +642,6 @@
}
blocks.addEventListener('change', pixelate, false);
$(function () {
$('[data-toggle="popover"]').popover()
})
</script>
{% endif %}