diff --git a/bin/lib/Tracker.py b/bin/lib/Tracker.py index 227213db..1bc7ea2f 100755 --- a/bin/lib/Tracker.py +++ b/bin/lib/Tracker.py @@ -1055,6 +1055,23 @@ def api_delete_tracker(data, user_id): tracker = Tracker(tracker_uuid) return tracker.delete(), 200 +def api_tracker_add_object(data, user_id): + tracker_uuid = data.get('uuid') + res = api_check_tracker_acl(tracker_uuid, user_id) + if res: + return res + tracker = Tracker(tracker_uuid) + object_gid = data.get('gid') + date = data.get('date') + if date: + if not Date.validate_str_date(date): + date = None + try: + obj_type, subtype, obj_id = object_gid.split(':', 2) + except (AttributeError, IndexError): + return {"status": "error", "reason": "Invalid Object"}, 400 + return tracker.add(obj_type, subtype, obj_id, date=date), 200 + def api_tracker_remove_object(data, user_id): tracker_uuid = data.get('uuid') res = api_check_tracker_acl(tracker_uuid, user_id) diff --git a/bin/lib/chats_viewer.py b/bin/lib/chats_viewer.py index f6f56449..3554b09f 100755 --- a/bin/lib/chats_viewer.py +++ b/bin/lib/chats_viewer.py @@ -322,7 +322,7 @@ def get_threads_metas(threads): def get_username_meta_from_global_id(username_global_id): _, instance_uuid, username_id = username_global_id.split(':', 2) username = Usernames.Username(username_id, instance_uuid) - return username.get_meta() + return username.get_meta(options={'icon'}) # TODO Filter ## Instance type @@ -386,6 +386,8 @@ def get_user_account_chats_meta(user_id, chats, subchannels): c_subtype, c_id = chat_g_id.split(':', 1) chat = Chats.Chat(c_id, c_subtype) chat_meta = chat.get_meta(options={'icon', 'info', 'nb_participants', 'tags_safe', 'username'}) + if chat_meta['username']: + chat_meta['username'] = get_username_meta_from_global_id(chat_meta['username']) chat_meta['nb_messages'] = len(chat.get_user_messages(user_id)) chat_meta['subchannels'] = [] for subchannel_gid in chat.get_subchannels(): @@ -425,6 +427,39 @@ def get_user_account_nb_all_week_messages(user_id, chats, subchannels): nb_day += 1 return stats +def _get_chat_card_meta_options(): + return {'created_at', 'icon', 'info', 'nb_participants', 'origin_link', 'subchannels', 'tags_safe', 'threads', 'translation', 'username'} + +def _get_message_bloc_meta_options(): + return {'chat', 'content', 'files-names', 'icon', 'images', 'language', 'link', 'parent', 'parent_meta', 'reactions','thread', 'translation', 'user-account'} + +def get_message_report(l_mess): # TODO Force language + translation + translation_target = 'en' + chats = {} + messages = [] + mess_options = _get_message_bloc_meta_options() + + l_mess = sorted(l_mess, key=lambda x: x[2]) + + for m in l_mess: + message = Messages.Message(m[2]) + meta = message.get_meta(options=mess_options, translation_target=translation_target) + if meta['chat'] not in chats: + chat = Chats.Chat(meta['chat'], message.get_chat_instance()) + meta_chat = chat.get_meta(options=_get_chat_card_meta_options(), translation_target=translation_target) + if meta_chat['username']: + meta_chat['username'] = get_username_meta_from_global_id(meta_chat['username']) + chats[chat.id] = meta_chat + + # stats + chats[chat.id]['t_messages'] = 1 + else: + chats[meta['chat']]['t_messages'] += 1 + + messages.append(meta) + + return chats, messages + #### FIX #### def fix_correlations_subchannel_message(): diff --git a/bin/lib/objects/Chats.py b/bin/lib/objects/Chats.py index e883f4a7..f7c82d5f 100755 --- a/bin/lib/objects/Chats.py +++ b/bin/lib/objects/Chats.py @@ -56,6 +56,13 @@ class Chat(AbstractChatObject): url = f'{baseurl}/correlation/show?type={self.type}&subtype={self.subtype}&id={self.id}' return url + def get_origin_link(self): + if self.subtype == '00098785-7e70-5d12-a120-c5cdc1252b2b': + username = self.get_username() + if username: + username = username.split(':', 2)[2] + return f'https://t.me/{username}' + def get_svg_icon(self): # TODO # if self.subtype == 'telegram': # style = 'fab' @@ -100,6 +107,8 @@ class Chat(AbstractChatObject): meta['threads'] = self.get_threads() if 'tags_safe' in options: meta['tags_safe'] = self.is_tags_safe(meta['tags']) + if 'origin_link' in options: + meta['origin_link'] = self.get_origin_link() return meta def get_misp_object(self): diff --git a/bin/lib/objects/Messages.py b/bin/lib/objects/Messages.py index 86521611..b3b30457 100755 --- a/bin/lib/objects/Messages.py +++ b/bin/lib/objects/Messages.py @@ -71,6 +71,10 @@ class Message(AbstractObject): def get_basename(self): return os.path.basename(self.id) + def get_chat_instance(self): + c_id = self.id.split('/') + return c_id[0] + def get_content(self, r_type='str'): # TODO ADD cache # TODO Compress content ??????? """ Returns content @@ -259,7 +263,7 @@ class Message(AbstractObject): else: timestamp = float(timestamp) timestamp = datetime.utcfromtimestamp(float(timestamp)) - meta['date'] = timestamp.strftime('%Y/%m/%d') + meta['date'] = timestamp.strftime('%Y-%m-%d') meta['hour'] = timestamp.strftime('%H:%M:%S') meta['full_date'] = timestamp.isoformat(' ') if 'last_full_date' in options: diff --git a/bin/lib/objects/UsersAccount.py b/bin/lib/objects/UsersAccount.py index 73ae7525..f9b09b48 100755 --- a/bin/lib/objects/UsersAccount.py +++ b/bin/lib/objects/UsersAccount.py @@ -150,7 +150,7 @@ class UserAccount(AbstractSubtypeObject): if meta['username']: _, username_account_subtype, username_account_id = meta['username'].split(':', 3) if 'username_meta' in options: - meta['username'] = Usernames.Username(username_account_id, username_account_subtype).get_meta() + meta['username'] = Usernames.Username(username_account_id, username_account_subtype).get_meta(options={'icon'}) else: meta['username'] = {'type': 'username', 'subtype': username_account_subtype, 'id': username_account_id} if 'usernames' in options: diff --git a/bin/packages/Date.py b/bin/packages/Date.py index 73ba1e2a..77d71188 100644 --- a/bin/packages/Date.py +++ b/bin/packages/Date.py @@ -1,6 +1,7 @@ #!/usr/bin/python3 import datetime +import time from calendar import monthrange from dateutil.rrule import rrule, MONTHLY @@ -91,6 +92,10 @@ def get_current_week_day(): start = dt - datetime.timedelta(days=dt.weekday()) return start.strftime("%Y%m%d") +def get_current_utc_full_time(): + timestamp = datetime.datetime.fromtimestamp(time.time()) + return timestamp.strftime('%Y-%m-%d %H:%M:%S') + def get_month_dates(date=None): if date: date = convert_date_str_to_datetime(date) diff --git a/var/www/blueprints/correlation.py b/var/www/blueprints/correlation.py index 1222755d..e23fbda4 100644 --- a/var/www/blueprints/correlation.py +++ b/var/www/blueprints/correlation.py @@ -359,6 +359,10 @@ def show_relationship(): dict_object["metadata"]['type_id'] = subtype else: dict_object["subtype"] = '' - dict_object["metadata_card"] = ail_objects.get_object_card_meta(obj_type, subtype, obj_id) + dict_object["metadata_card"] = ail_objects.get_object_card_meta(obj_type, subtype, obj_id) + dict_object["metadata_card"]['tags_safe'] = True return render_template("show_relationship.html", dict_object=dict_object, bootstrap_label=bootstrap_label, - tags_selector_data=Tag.get_tags_selector_data()) + tags_selector_data=Tag.get_tags_selector_data(), + meta=dict_object["metadata_card"], + ail_tags=dict_object["metadata_card"]["add_tags_modal"]) + diff --git a/var/www/blueprints/hunters.py b/var/www/blueprints/hunters.py index ac4d9a7b..3dd709db 100644 --- a/var/www/blueprints/hunters.py +++ b/var/www/blueprints/hunters.py @@ -24,6 +24,7 @@ sys.path.append(os.environ['AIL_BIN']) ################################## from lib import ail_core from lib.objects import ail_objects +from lib import chats_viewer from lib import item_basic from lib import Tracker from lib import Tag @@ -372,6 +373,27 @@ def get_json_tracker_graph(): res = Tracker.get_trackers_graph_by_day([tracker_uuid]) return jsonify(res) +@hunters.route('/tracker/object/add', methods=['GET']) +@login_required +@login_admin +def tracker_object_add(): + user_id = current_user.get_id() + tracker_uuid = request.args.get('uuid') + object_global_id = request.args.get('gid') + if object_global_id.startswith('messages::'): + obj = ail_objects.get_obj_from_global_id(object_global_id) + date = obj.get_date() + else: + date = request.args.get('date') # TODO check daterange + res = Tracker.api_tracker_add_object({'uuid': tracker_uuid, 'gid': object_global_id, 'date': date}, user_id) + if res[1] != 200: + return create_json_response(res[0], res[1]) + else: + if request.referrer: + return redirect(request.referrer) + else: + return redirect(url_for('hunters.show_tracker', uuid=tracker_uuid)) + @hunters.route('/tracker/object/remove', methods=['GET']) @login_required @login_analyst @@ -389,6 +411,41 @@ def tracker_object_remove(): return redirect(url_for('hunters.show_tracker', uuid=tracker_uuid)) +@hunters.route('/tracker/objects', methods=['GET']) +@login_required +@login_admin +def tracker_objects(): + user_id = current_user.get_id() + tracker_uuid = request.args.get('uuid', None) + res = Tracker.api_is_allowed_to_edit_tracker(tracker_uuid, user_id) + if res[1] != 200: # invalid access + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + + tracker = Tracker.Tracker(tracker_uuid) + meta = tracker.get_meta(options={'description', 'sparkline', 'tags', 'nb_objs'}) + if meta['type'] == 'yara': + yara_rule_content = Tracker.get_yara_rule_content(meta['tracked']) + else: + yara_rule_content = None + + chats, messages = chats_viewer.get_message_report(tracker.get_objs()) + + meta['date'] = Date.get_current_utc_full_time() + + return render_template("messages_report.html", meta=meta, yara_rule_content=yara_rule_content, + chats=chats, messages=messages, bootstrap_label=bootstrap_label) + + # TODO + + # Manual - Title + # - Summary + + # Messages table + + # Timeline messages by chats - line + # pie charts NB messages all chats + # Barchart NB messages by days + #################### # RETRO HUNT # #################### diff --git a/var/www/templates/chats_explorer/basic_card_chat.html b/var/www/templates/chats_explorer/basic_card_chat.html index bc4273bb..d747ff8e 100644 --- a/var/www/templates/chats_explorer/basic_card_chat.html +++ b/var/www/templates/chats_explorer/basic_card_chat.html @@ -13,58 +13,87 @@ {{ meta["svg_icon"]["icon"] }} - {% if meta['username'] %}{{ meta["username"]["id"] }} {% else %} {{ meta['name'] }}{% endif %} : {{ meta['id'] }} + {% if meta['name'] %}{{ meta['name'] }}{% endif %}{% if not report_mode %} : {{ meta['id'] }}{% endif %}
- - {% if meta["tags_safe"] %} - {% if meta['icon'] %} - {{ meta['id'] }} + +
+
+ {% if meta["tags_safe"] %} + {% if meta['icon'] %} + {{ meta['id'] }} + {% endif %} + {% else %} + + + + {% endif %} - {% else %} - - - - - {% endif %} - +
- - - - - - {{meta["first_seen"]}} - - - - {{meta["last_seen"]}} - - - - - - - - - {{meta["nb_subchannels"]}}   - - - - {{meta["nb_participants"]}} - - - - - - - {{meta["nb_messages"]}}   - - -
- {{ meta['info'] }} +
+ {% if meta['username'] %} +
+ + + + {{ meta["username"]["icon"]["icon"] }} + + + {{ meta['username']['id'] }} +
+ {% endif %} + + +
+ + + + {{meta["first_seen"][0:4]}}-{{meta["first_seen"][4:6]}}-{{meta["first_seen"][6:8]}} + + + + {{meta["last_seen"][0:4]}}-{{meta["last_seen"][4:6]}}-{{meta["last_seen"][6:8]}} + + + +
+{#
#} +{# #} +{# #} +{# #} +{# #} +{# {{meta["created_at"]}}#} +{# #} +{#
#} +
+ + + Subchannels + + {{meta["nb_subchannels"]}}   + + Participants + + {{meta["nb_participants"]}} + + {% if "nb_messages" in meta %} + + + + + + {{ meta["nb_messages"] }}   + + {% endif %} +
+
+
+ +
+
{{ meta['info'] }}
@@ -120,5 +149,7 @@ {% endif %}
- {% include 'objects/block_object_footer_small.html' %} + {% if not report_mode %} + {% include 'objects/block_object_footer_small.html' %} + {% endif %}
\ No newline at end of file diff --git a/var/www/templates/chats_explorer/block_message.html b/var/www/templates/chats_explorer/block_message.html index 4cbdfc9d..1d7b4d75 100644 --- a/var/www/templates/chats_explorer/block_message.html +++ b/var/www/templates/chats_explorer/block_message.html @@ -22,15 +22,16 @@ } - -
{{ message['user-account']['id'] }} + class="rounded-circle mr-1" alt="{{ message['user-account']['id'] }}" width="60" height="60"> -
{{ message['hour'] }}
+
+
{{ message['date'] }}
+
{{ message['hour'] }}
+
@@ -77,13 +78,13 @@ {% endfor %} {% endif %} {% if not message['extracted'] %} -
{{ message['content'] }}
+
{{ message['content'] }}
{% else %} -
{{ message['content'][:message['extracted'][0][0]] }}{% for row in message['extracted'] %}{{ message['content'][row[0]:row[1]] }}{% if loop.index + 1 > message['extracted']|length %}{{ message['content'][message['extracted'][-1][1]:] }}{% else %}{{ message['content'][row[1]:message['extracted'][loop.index][0]] }}{% endif %}{% endfor %}
+
{{ message['content'][:message['extracted'][0][0]] }}{% for row in message['extracted'] %}{{ message['content'][row[0]:row[1]] }}{% if loop.index + 1 > message['extracted']|length %}{{ message['content'][message['extracted'][-1][1]:] }}{% else %}{{ message['content'][row[1]:message['extracted'][loop.index][0]] }}{% endif %}{% endfor %}
{% endif %} {% if message['translation'] %}
-
{{ message['translation'] }}
+
{{ message['translation'] }}
{% endif %} {% for reaction in message['reactions'] %} diff --git a/var/www/templates/hunter/messages_report.html b/var/www/templates/hunter/messages_report.html new file mode 100644 index 00000000..b0636fb0 --- /dev/null +++ b/var/www/templates/hunter/messages_report.html @@ -0,0 +1,274 @@ + + + + + Report - AIL + + + + + + + + + + + + + + + + + + + + + + {% include 'nav_bar.html' %} + +
+
+ +
+ +

Tracker Report:

+ +
+
+ +
+
+ + + +

+ {% if meta['description'] %} + {{ meta['description'] }} + {% endif %} +

+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + {% if meta['type'] != 'yara' %} + + + + + {% endif %} +
Type + {% if meta['type'] == 'word' %} +   + {% elif meta['type'] == 'set' %} +   + {% elif meta['type'] == 'regex' %} +   + {% elif meta['type'] == 'typosquatting' %} +   + {% elif meta['type'] == 'yara' %} +  {   + {% endif %} + {{ meta['type'] }} +
Generation Date + {{meta['date']}} +
First Seen   + {% if meta['first_seen'] %} + {{ meta['first_seen'][0:4] }} - {{ meta['first_seen'][4:6] }} - {{ meta['first_seen'][6:8] }} + {% endif %} +
Last Seen   + {% if meta['last_seen'] %} + {{ meta['last_seen'][0:4] }} - {{ meta['last_seen'][4:6] }} - {{ meta['last_seen'][6:8] }} + {% endif %} +
Tags + {%for tag in meta['tags']%} + {{ tag }} + {%endfor%} +
Objects Match + {%for obj_type in meta['nb_objs']%} +

+ {{ obj_type }} + {{ meta['nb_objs'][obj_type] }} +

+ {%endfor%} +
Tracker + {{ meta['tracked'] }} +
+ + {% if yara_rule_content %} +
Yara Rule:
+

+

{{ yara_rule_content }}
+

+ {% endif %} + +
+
+ +
+
+
+ +
+
+ +
+
+
+ +

Messages:

+ {% for message in messages %} +
+
+ + + + {{ chats[message['chat']]["svg_icon"]["icon"] }} + + + {% if chats[message['chat']]['name'] %}{{ chats[message['chat']]['name'] }}{% endif %} +
+
+ {% if chats[message['chat']]['origin_link'] %} + {{ chats[message['chat']]['origin_link'] }} + {% endif %} +
+
+ + {% with message=message,show_full_message=True %} + {% include 'chats_explorer/block_message.html' %} + {% endwith %} + + {% endfor %} + +

Chats Metadata:

+ {% for chat in chats %} +
+ {% with meta=chats[chat],report_mode=True %} + {% include 'chats_explorer/basic_card_chat.html' %} + {% endwith %} +
+ {% endfor %} + +
+
+
+ +{#
All Messages:
#} +{#
#} + +{# {% with translate_url=url_for('chats_explorer.chats_explorer_chat', subtype=chat['subtype']), obj_id=chat['id'], pagination=chat['pagination'] %}#} +{# {% include 'chats_explorer/block_translation.html' %}#} +{# {% endwith %}#} + +{# {% if chat['messages'] %}#} +{##} +{#
#} +{#
#} +{##} +{# {% for date in chat['messages'] %}#} +{##} +{#
#} +{#

#} +{# {{ date }}#} +{#

#} +{#
#} +{##} +{# {% for mess in chat['messages'][date] %}#} +{##} +{# {% with message=mess %}#} +{# {% include 'chats_explorer/block_message.html' %}#} +{# {% endwith %}#} +{##} +{# {% endfor %}#} +{#
#} +{# {% endfor %}#} +{##} +{#
#} +{#
#} +{##} +{# {% endif %}#} + + + + + + +{##} + + + + +