chg: [message] show trackers + modules matches

ocr
terrtia 2024-03-27 13:42:15 +01:00
parent a3a664b7f1
commit 5ec0d7f0cf
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
10 changed files with 121 additions and 50 deletions

View File

@ -40,6 +40,11 @@ r_key = regex_helper.generate_redis_cache_key('extractor')
# TODO UI Link # TODO UI Link
CORRELATION_TO_EXTRACT = {
'item': ['cve', 'cryptocurrency', 'title', 'username'],
'message': ['cve', 'cryptocurrency', 'username']
}
MODULES = { MODULES = {
'infoleak:automatic-detection="credit-card"': CreditCards(queue=False), 'infoleak:automatic-detection="credit-card"': CreditCards(queue=False),
'infoleak:automatic-detection="iban"': Iban(queue=False), 'infoleak:automatic-detection="iban"': Iban(queue=False),
@ -57,9 +62,9 @@ tools = Tools(queue=False)
for tool_name in tools.get_tools(): for tool_name in tools.get_tools():
MODULES[f'infoleak:automatic-detection="{tool_name}-tool"'] = tools MODULES[f'infoleak:automatic-detection="{tool_name}-tool"'] = tools
def get_correl_match(extract_type, obj_id, content): def get_correl_match(extract_type, obj, content):
extracted = [] extracted = []
correl = correlations_engine.get_correlation_by_correl_type('item', '', obj_id, extract_type) correl = correlations_engine.get_correlation_by_correl_type(obj.type, obj.get_subtype(r_str=True), obj.id, extract_type)
to_extract = [] to_extract = []
map_subtype = {} map_subtype = {}
map_value_id = {} map_value_id = {}
@ -75,18 +80,18 @@ def get_correl_match(extract_type, obj_id, content):
sha256_val = sha256(value.encode()).hexdigest() sha256_val = sha256(value.encode()).hexdigest()
map_value_id[sha256_val] = value map_value_id[sha256_val] = value
if to_extract: if to_extract:
objs = regex_helper.regex_finditer(r_key, '|'.join(to_extract), obj_id, content) objs = regex_helper.regex_finditer(r_key, '|'.join(to_extract), obj.get_global_id(), content)
for obj in objs: for ob in objs:
if map_subtype.get(obj[2]): if map_subtype.get(ob[2]):
subtype = map_subtype[obj[2]] subtype = map_subtype[ob[2]]
else: else:
subtype = '' subtype = ''
sha256_val = sha256(obj[2].encode()).hexdigest() sha256_val = sha256(ob[2].encode()).hexdigest()
value_id = map_value_id.get(sha256_val) value_id = map_value_id.get(sha256_val)
if not value_id: if not value_id:
logger.critical(f'Error module extractor: {sha256_val}\n{extract_type}\n{subtype}\n{value_id}\n{map_value_id}\n{objs}') logger.critical(f'Error module extractor: {sha256_val}\n{extract_type}\n{subtype}\n{value_id}\n{map_value_id}\n{objs}')
value_id = 'ERROR' value_id = 'ERROR'
extracted.append([obj[0], obj[1], obj[2], f'{extract_type}:{subtype}:{value_id}']) extracted.append([ob[0], ob[1], ob[2], f'{extract_type}:{subtype}:{value_id}'])
return extracted return extracted
def _get_yara_match(data): def _get_yara_match(data):
@ -100,7 +105,7 @@ def _get_yara_match(data):
return yara.CALLBACK_CONTINUE return yara.CALLBACK_CONTINUE
def _get_word_regex(word): def _get_word_regex(word):
return '(?:^|(?<=[\&\~\:\;\,\.\(\)\{\}\|\[\]\\\\/\-/\=\'\"\%\$\?\@\+\#\_\^\<\>\!\*\n\r\t\s]))' + word + '(?:$|(?=[\&\~\:\;\,\.\(\)\{\}\|\[\]\\\\/\-/\=\'\"\%\$\?\@\+\#\_\^\<\>\!\*\n\r\t\s]))' return '(?i)(?:^|(?<=[\&\~\:\;\,\.\(\)\{\}\|\[\]\\\\/\-/\=\'\\"\%\$\?\@\+\#\_\^\<\>\!\*\n\r\t\s]))' + word + '(?:$|(?=[\&\~\:\;\,\.\(\)\{\}\|\[\]\\\\/\-/\=\'\\"\%\$\?\@\+\#\_\^\<\>\!\*\n\r\t\s]))'
def convert_byte_offset_to_string(b_content, offset): def convert_byte_offset_to_string(b_content, offset):
byte_chunk = b_content[:offset + 1] byte_chunk = b_content[:offset + 1]
@ -115,17 +120,18 @@ def convert_byte_offset_to_string(b_content, offset):
# TODO RETRO HUNTS # TODO RETRO HUNTS
# TODO TRACKER TYPE IN UI # TODO TRACKER TYPE IN UI
def get_tracker_match(obj_id, content): def get_tracker_match(obj, content):
extracted = [] extracted = []
extracted_yara = [] extracted_yara = []
trackers = Tracker.get_obj_trackers('item', '', obj_id) obj_gid = obj.get_global_id()
trackers = Tracker.get_obj_trackers(obj.type, obj.get_subtype(r_str=True), obj.id)
for tracker_uuid in trackers: for tracker_uuid in trackers:
tracker = Tracker.Tracker(tracker_uuid) tracker = Tracker.Tracker(tracker_uuid)
tracker_type = tracker.get_type() tracker_type = tracker.get_type()
# print(tracker_type) # print(tracker_type)
tracked = tracker.get_tracked() tracked = tracker.get_tracked()
if tracker_type == 'regex': # TODO Improve word detection -> word delimiter if tracker_type == 'regex': # TODO Improve word detection -> word delimiter
regex_match = regex_helper.regex_finditer(r_key, tracked, obj_id, content) regex_match = regex_helper.regex_finditer(r_key, tracked, obj_gid, content)
for match in regex_match: for match in regex_match:
extracted.append([int(match[0]), int(match[1]), match[2], f'tracker:{tracker.uuid}']) extracted.append([int(match[0]), int(match[1]), match[2], f'tracker:{tracker.uuid}'])
elif tracker_type == 'yara': elif tracker_type == 'yara':
@ -147,13 +153,13 @@ def get_tracker_match(obj_id, content):
words = [tracked] words = [tracked]
for word in words: for word in words:
regex = _get_word_regex(word) regex = _get_word_regex(word)
regex_match = regex_helper.regex_finditer(r_key, regex, obj_id, content) regex_match = regex_helper.regex_finditer(r_key, regex, obj_gid, content)
# print(regex_match) # print(regex_match)
for match in regex_match: for match in regex_match:
extracted.append([int(match[0]), int(match[1]), match[2], f'tracker:{tracker.uuid}']) extracted.append([int(match[0]), int(match[1]), match[2], f'tracker:{tracker.uuid}'])
# Retro Hunt # Retro Hunt
retro_hunts = Tracker.get_obj_retro_hunts('item', '', obj_id) retro_hunts = Tracker.get_obj_retro_hunts(obj.type, obj.get_subtype(r_str=True), obj.id)
for retro_uuid in retro_hunts: for retro_uuid in retro_hunts:
retro_hunt = Tracker.RetroHunt(retro_uuid) retro_hunt = Tracker.RetroHunt(retro_uuid)
rule = retro_hunt.get_rule(r_compile=True) rule = retro_hunt.get_rule(r_compile=True)
@ -182,35 +188,36 @@ def get_tracker_match(obj_id, content):
# Type:subtype:id # Type:subtype:id
# tag:iban # tag:iban
# tracker:uuid # tracker:uuid
# def extract(obj_id, content=None):
def extract(obj_id, content=None): def extract(obj_type, subtype, obj_id, content=None):
item = Item(obj_id) obj = ail_objects.get_object(obj_type, subtype, obj_id)
if not item.exists(): if not obj.exists():
return [] return []
obj_gid = obj.get_global_id()
# CHECK CACHE # CHECK CACHE
cached = r_cache.get(f'extractor:cache:{obj_id}') cached = r_cache.get(f'extractor:cache:{obj_gid}')
# cached = None # cached = None
if cached: if cached:
r_cache.expire(f'extractor:cache:{obj_id}', 300) r_cache.expire(f'extractor:cache:{obj_gid}', 300)
return json.loads(cached) return json.loads(cached)
if not content: if not content:
content = item.get_content() content = obj.get_content()
extracted = get_tracker_match(obj_id, content) extracted = get_tracker_match(obj, content)
# print(item.get_tags()) # print(item.get_tags())
for tag in item.get_tags(): for tag in obj.get_tags():
if MODULES.get(tag): if MODULES.get(tag):
# print(tag) # print(tag)
module = MODULES.get(tag) module = MODULES.get(tag)
matches = module.extract(obj_id, content, tag) matches = module.extract(obj, content, tag)
if matches: if matches:
extracted = extracted + matches extracted = extracted + matches
for obj_t in ['cve', 'cryptocurrency', 'title', 'username']: # Decoded, PGP->extract bloc for obj_t in CORRELATION_TO_EXTRACT[obj.type]:
matches = get_correl_match(obj_t, obj_id, content) matches = get_correl_match(obj_t, obj, content)
if matches: if matches:
extracted = extracted + matches extracted = extracted + matches
@ -221,8 +228,8 @@ def extract(obj_id, content=None):
# Save In Cache # Save In Cache
if extracted: if extracted:
extracted_dump = json.dumps(extracted) extracted_dump = json.dumps(extracted)
r_cache.set(f'extractor:cache:{obj_id}', extracted_dump) r_cache.set(f'extractor:cache:{obj_gid}', extracted_dump)
r_cache.expire(f'extractor:cache:{obj_id}', 300) # TODO Reduce CACHE ??????????????? r_cache.expire(f'extractor:cache:{obj_gid}', 300) # TODO Reduce CACHE ???????????????
return extracted return extracted
@ -271,15 +278,7 @@ def get_extracted_by_match(extracted):
# if __name__ == '__main__': # if __name__ == '__main__':
# t0 = time.time() # t0 = time.time()
# obj_id = 'crawled/2022/09/15/circl.lu179c7903-5b21-452e-9f25-4b61d9934e2b'
# obj_id = 'crawled/2022/09/15/circl.lu1e4f9721-06dc-404f-aabf-3c3bd0b533bd'
# obj_id = 'submitted/2022/09/13/submitted_ba3ee771-c91c-4f50-9d6a-8558cdac7aeb.gz'
# # obj_id = 'tests/2021/01/01/credit_cards.gz'
# # obj_id = 'crawled/2020/07/20/circl.luc9301321-f1b1-4d91-9082-5eb452b946c5'
# obj_id = 'submitted/2019/09/22/97172282-e4c2-4a1e-b82c-c4fb9490a56e.gz'
# obj_id = 'submitted/2019/09/20/4fb7f02d-1241-4ef4-b17e-80ae76038835.gz'
# obj_id = 'crawled/2023/02/21/circl.lu1c300acb-0cbe-480f-917e-9afe3ec958e8' # obj_id = 'crawled/2023/02/21/circl.lu1c300acb-0cbe-480f-917e-9afe3ec958e8'
#
# extract(obj_id) # extract(obj_id)
# #
# # get_obj_correl('cve', obj_id, content) # # get_obj_correl('cve', obj_id, content)

View File

@ -58,9 +58,9 @@ class CreditCards(AbstractModule):
if lib_refine.is_luhn_valid(clean_card): if lib_refine.is_luhn_valid(clean_card):
return clean_card return clean_card
def extract(self, obj_id, content, tag): def extract(self, obj, content, tag):
extracted = [] extracted = []
cards = self.regex_finditer(self.regex, obj_id, content) cards = self.regex_finditer(self.regex, obj.get_global_id(), content)
for card in cards: for card in cards:
start, end, value = card start, end, value = card
if self.get_valid_card(value): if self.get_valid_card(value):

View File

@ -62,9 +62,9 @@ class Iban(AbstractModule):
return True return True
return False return False
def extract(self, obj_id, content, tag): def extract(self, obj, content, tag):
extracted = [] extracted = []
ibans = self.regex_finditer(self.iban_regex, obj_id, content) ibans = self.regex_finditer(self.iban_regex, obj.get_global_id(), content)
for iban in ibans: for iban in ibans:
start, end, value = iban start, end, value = iban
value = ''.join(e for e in value if e.isalnum()) value = ''.join(e for e in value if e.isalnum())

View File

@ -118,10 +118,10 @@ class Mail(AbstractModule):
print(e) print(e)
return valid_mxdomain return valid_mxdomain
def extract(self, obj_id, content, tag): def extract(self, obj, content, tag):
extracted = [] extracted = []
mxdomains = {} mxdomains = {}
mails = self.regex_finditer(self.email_regex, obj_id, content) mails = self.regex_finditer(self.email_regex, obj.get_global_id(), content)
for mail in mails: for mail in mails:
start, end, value = mail start, end, value = mail
mxdomain = value.rsplit('@', 1)[1].lower() mxdomain = value.rsplit('@', 1)[1].lower()

View File

@ -55,9 +55,9 @@ class Onion(AbstractModule):
# TEMP var: SAVE I2P Domain (future I2P crawler) # TEMP var: SAVE I2P Domain (future I2P crawler)
# self.save_i2p = config_loader.get_config_boolean("Onion", "save_i2p") # self.save_i2p = config_loader.get_config_boolean("Onion", "save_i2p")
def extract(self, obj_id, content, tag): def extract(self, obj, content, tag):
extracted = [] extracted = []
onions = self.regex_finditer(self.onion_regex, obj_id, content) onions = self.regex_finditer(self.onion_regex, obj.get_global_id(), content)
for onion in onions: for onion in onions:
start, end, value = onion start, end, value = onion
url_unpack = crawlers.unpack_url(value) url_unpack = crawlers.unpack_url(value)

View File

@ -41,9 +41,9 @@ class Phone(AbstractModule):
# Waiting time in seconds between to message processed # Waiting time in seconds between to message processed
self.pending_seconds = 1 self.pending_seconds = 1
def extract(self, obj_id, content, tag): def extract(self, obj, content, tag):
extracted = [] extracted = []
phones = self.regex_phone_iter('ZZ', obj_id, content) phones = self.regex_phone_iter('ZZ', obj.get_global_id(), content)
for phone in phones: for phone in phones:
extracted.append([phone[0], phone[1], phone[2], f'tag:{tag}']) extracted.append([phone[0], phone[1], phone[2], f'tag:{tag}'])
return extracted return extracted

View File

@ -23,6 +23,7 @@ from lib import ail_core
from lib import chats_viewer from lib import chats_viewer
from lib import Language from lib import Language
from lib import Tag from lib import Tag
from lib import module_extractor
# ============ BLUEPRINT ============ # ============ BLUEPRINT ============
chats_explorer = Blueprint('chats_explorer', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/chats_explorer')) chats_explorer = Blueprint('chats_explorer', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/chats_explorer'))
@ -235,6 +236,10 @@ def objects_message():
else: else:
message = message[0] message = message[0]
languages = Language.get_translation_languages() languages = Language.get_translation_languages()
extracted = module_extractor.extract('message', '', message['id'], content=message['content'])
extracted_matches = module_extractor.get_extracted_by_match(extracted)
message['extracted'] = extracted
message['extracted_matches'] = extracted_matches
return render_template('ChatMessage.html', meta=message, bootstrap_label=bootstrap_label, return render_template('ChatMessage.html', meta=message, bootstrap_label=bootstrap_label,
translation_languages=languages, translation_target=target, translation_languages=languages, translation_target=target,
modal_add_tags=Tag.get_modal_add_tags(message['id'], object_type='message')) modal_add_tags=Tag.get_modal_add_tags(message['id'], object_type='message'))

View File

@ -85,7 +85,7 @@ def showItem(): # # TODO: support post
else: else:
meta['investigations'] = [] meta['investigations'] = []
extracted = module_extractor.extract(item.id, content=meta['content']) extracted = module_extractor.extract('item', '', item.id, content=meta['content'])
extracted_matches = module_extractor.get_extracted_by_match(extracted) extracted_matches = module_extractor.get_extracted_by_match(extracted)
return render_template("show_item.html", bootstrap_label=bootstrap_label, return render_template("show_item.html", bootstrap_label=bootstrap_label,

View File

@ -14,8 +14,8 @@
<!-- JS --> <!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script> <script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script> <script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script> <script src="{{ url_for('static', filename='js/bootstrap4.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script> <script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script> <script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/tags.js') }}"></script> <script src="{{ url_for('static', filename='js/tags.js') }}"></script>
@ -134,6 +134,65 @@
</div> </div>
{% if meta['extracted_matches'] %}
<div id="accordion_extracted" class="mb-3 mx-3">
<div class="card">
<div class="card-header py-1" id="heading_extracted">
<div class="row">
<div class="col-11">
<div class="mt-2">
<img id="misp-logo" src="{{ url_for('static', filename='image/ail-icon.png')}}" height="32"> Extracted&nbsp;&nbsp;
<div class="badge badge-warning">{{meta['extracted_matches']|length}}</div>
</div>
</div>
<div class="col-1">
<button class="btn btn-link btn-lg py-2 float-right rotate down" data-toggle="collapse" data-target="#collapse_extracted" aria-expanded="true" aria-controls="collapseDecoded">
<i class="fas fa-chevron-circle-down"></i>
</button>
</div>
</div>
</div>
<div id="collapse_extracted" class="collapse" aria-labelledby="heading_extracted" data-parent="#accordion_extracted">
<div class="card-body">
<table id="table_extracted" class="table table-striped">
<thead class="thead-dark">
<tr>
<th>Type</th>
<th>ID</th>
<th>Extracted</th>
</tr>
</thead>
<tbody>
{% for match in meta['extracted_matches'] %}
<tr>
<td>
<svg height="26" width="26">
<g class="nodes">
<circle cx="13" cy="13" r="13" fill="{{ meta['extracted_matches'][match]['icon']['color'] }}"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ meta['extracted_matches'][match]['icon']['style'] }}" font-size="16px">{{ meta['extracted_matches'][match]['icon']['icon'] }}</text>
</g>
</svg>
{{ meta['extracted_matches'][match]['subtype'] }}
</td>
<td>{{ meta['extracted_matches'][match]['id'] }}</td>
<td>
{% for row in meta['extracted_matches'][match]['matches'] %}
<a href="#{{ row[0] }}:{{row[1] }}">{{ row[2] }}</a><br>
{% endfor %}
</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
</div>
</div>
{% endif %}
{% include 'objects/image/block_blur_img_slider.html' %} {% include 'objects/image/block_blur_img_slider.html' %}
{% with translate_url=url_for('chats_explorer.objects_message', id=meta['id']), obj_id=meta['id'] %} {% with translate_url=url_for('chats_explorer.objects_message', id=meta['id']), obj_id=meta['id'] %}
{% include 'chats_explorer/block_translation.html' %} {% include 'chats_explorer/block_translation.html' %}
@ -169,6 +228,10 @@
}); });
$(function () {
$('[data-toggle="popover"]').popover()
})
function toggle_sidebar(){ function toggle_sidebar(){
if($('#nav_menu').is(':visible')){ if($('#nav_menu').is(':visible')){
$('#nav_menu').hide(); $('#nav_menu').hide();

View File

@ -76,7 +76,11 @@
</div> </div>
{% endfor %} {% endfor %}
{% endif %} {% endif %}
<pre class="my-0">{{ message['content'] }}</pre> {% if not message['extracted'] %}
<pre class="my-0">{{ message['content'] }}</pre>
{% else %}
<pre class="my-0">{{ message['content'][:message['extracted'][0][0]] }}{% for row in message['extracted'] %}<span class="hg-text" data-toggle="popover" data-trigger="hover" data-html="true" title="<svg height=&quot;26&quot; width=&quot;26&quot;><g class=&quot;nodes&quot;><circle cx=&quot;13&quot; cy=&quot;13&quot; r=&quot;13&quot; fill=&quot;{{ message['extracted_matches'][row[3]]['icon']['color'] }}&quot;></circle><text x=&quot;13&quot; y=&quot;13&quot; text-anchor=&quot;middle&quot; dominant-baseline=&quot;central&quot; class=&quot;graph_node_icon {{ message['extracted_matches'][row[3]]['icon']['style'] }}&quot; font-size=&quot;16px&quot;>{{ message['extracted_matches'][row[3]]['icon']['icon'] }}</text></g></svg> {{ message['extracted_matches'][row[3]]['subtype'] }}" data-content="{{ message['extracted_matches'][row[3]]['id'] }}" id="{{ row[0] }}:{{ row[1] }}">{{ message['content'][row[0]:row[1]] }}</span>{% if loop.index + 1 > message['extracted']|length %}{{ message['content'][message['extracted'][-1][1]:] }}{% else %}{{ message['content'][row[1]:message['extracted'][loop.index][0]] }}{% endif %}{% endfor %}</pre>
{% endif %}
{% if message['translation'] %} {% if message['translation'] %}
<hr class="m-1"> <hr class="m-1">
<pre class="my-0 text-secondary">{{ message['translation'] }}</pre> <pre class="my-0 text-secondary">{{ message['translation'] }}</pre>