chg: [core] add telegram importer + Chat object + message Object + add timeline engine

bin/importer/FeederImporter.py

@@ -87,13 +87,16 @@ class FeederImporter(AbstractImporter):
         feeder_name = feeder.get_name()
         print(f'importing: {feeder_name} feeder')
 
-        item_id = feeder.get_item_id()
+        item_id = feeder.get_item_id() # TODO replace me with object global id
         # process meta
         if feeder.get_json_meta():
             feeder.process_meta()
-        gzip64_content = feeder.get_gzip64_content()
 
-        return f'{feeder_name} {item_id} {gzip64_content}'
+        if feeder_name == 'telegram':
+            return item_id  # TODO support UI dashboard
+        else:
+            gzip64_content = feeder.get_gzip64_content()
+            return f'{feeder_name} {item_id} {gzip64_content}'
 
 
 class FeederModuleImporter(AbstractModule):

bin/importer/feeders/Telegram.py

@@ -16,9 +16,30 @@ sys.path.append(os.environ['AIL_BIN'])
 # Import Project packages
 ##################################
 from importer.feeders.Default import DefaultFeeder
+from lib.ConfigLoader import ConfigLoader
+from lib.objects.Chats import Chat
+from lib.objects import Messages
+from lib.objects import UsersAccount
 from lib.objects.Usernames import Username
 from lib import item_basic
 
+import base64
+import io
+import gzip
+def gunzip_bytes_obj(bytes_obj):
+    gunzipped_bytes_obj = None
+    try:
+        in_ = io.BytesIO()
+        in_.write(bytes_obj)
+        in_.seek(0)
+
+        with gzip.GzipFile(fileobj=in_, mode='rb') as fo:
+            gunzipped_bytes_obj = fo.read()
+    except Exception as e:
+        print(f'Global; Invalid Gzip file: {e}')
+
+    return gunzipped_bytes_obj
+
 class TelegramFeeder(DefaultFeeder):
 
     def __init__(self, json_data):
@@ -26,14 +47,17 @@ class TelegramFeeder(DefaultFeeder):
         self.name = 'telegram'
 
     # define item id
-    def get_item_id(self):
+    def get_item_id(self): # TODO rename self.item_id
-        # TODO use telegram message date
+        # Get message date
-        date = datetime.date.today().strftime("%Y/%m/%d")
+        timestamp = self.json_data['meta']['date']['timestamp']  # TODO CREATE DEFAULT TIMESTAMP
-        channel_id = str(self.json_data['meta']['chat']['id'])
+        # if self.json_data['meta'].get('date'):
+        #     date = datetime.datetime.fromtimestamp( self.json_data['meta']['date']['timestamp'])
+        #     date = date.strftime('%Y/%m/%d')
+        # else:
+        #     date = datetime.date.today().strftime("%Y/%m/%d")
+        chat_id = str(self.json_data['meta']['chat']['id'])
         message_id = str(self.json_data['meta']['id'])
-        item_id = f'{channel_id}_{message_id}'
+        self.item_id = Messages.create_obj_id('telegram', chat_id, message_id, timestamp)
-        item_id = os.path.join('telegram', date, item_id)
-        self.item_id = f'{item_id}.gz'
         return self.item_id
 
     def process_meta(self):
@@ -42,19 +66,67 @@ class TelegramFeeder(DefaultFeeder):
         """
         # message chat
         meta = self.json_data['meta']
+        mess_id = self.json_data['meta']['id']
+        if meta.get('reply_to'):
+            reply_to_id = meta['reply_to']
+        else:
+            reply_to_id = None
+
+        timestamp = meta['date']['timestamp']
+        date = datetime.datetime.fromtimestamp(timestamp)
+        date = date.strftime('%Y%m%d')
+
         if meta.get('chat'):
-            if meta['chat'].get('username'):
+            chat = Chat(meta['chat']['id'], 'telegram')
-                user = meta['chat']['username']
+
-                if user:
+            if meta['chat'].get('username'):  # TODO USE ID AND SAVE USERNAME
-                    date = item_basic.get_item_date(self.item_id)
+                chat_username = meta['chat']['username']
-                    username = Username(user, 'telegram')
+
-                    username.add(date, self.item_id)
+            # Chat---Message
+            chat.add(date, self.item_id)  # TODO modify to accept file objects
+            # message meta ????? who is the user if two user ????
+
+            if self.json_data.get('translation'):
+                translation = self.json_data['translation']
+            else:
+                translation = None
+            decoded = base64.standard_b64decode(self.json_data['data'])
+            content = gunzip_bytes_obj(decoded)
+            Messages.create(self.item_id, content, translation=translation)
+
+            chat.add_message(self.item_id, timestamp, mess_id, reply_id=reply_to_id)
+        else:
+            chat = None
+
         # message sender
-        if meta.get('sender'):
+        if meta.get('sender'):  # TODO handle message channel forward
+            user_id = meta['sender']['id']
+            user_account = UsersAccount.UserAccount(user_id, 'telegram')
+            # UserAccount---Message
+            user_account.add(date, self.item_id)
+            # UserAccount---Chat
+            user_account.add_correlation(chat.type, chat.get_subtype(r_str=True), chat.id)
+
+            if meta['sender'].get('firstname'):
+                user_account.set_first_name(meta['sender']['firstname'])
+            if meta['sender'].get('lastname'):
+                user_account.set_last_name(meta['sender']['lastname'])
+            if meta['sender'].get('phone'):
+                user_account.set_phone(meta['sender']['phone'])
+
             if meta['sender'].get('username'):
-                user = meta['sender']['username']
+                username = Username(meta['sender']['username'], 'telegram')
-                if user:
+                user_account.add_correlation(username.type, username.get_subtype(r_str=True), username.id)
-                    date = item_basic.get_item_date(self.item_id)
+
-                    username = Username(user, 'telegram')
+                # Username---Message
-                    username.add(date, self.item_id)
+                username.add(date, self.item_id) # TODO ####################################################################
+                if chat:
+                    chat.add_correlation(username.type, username.get_subtype(r_str=True), username.id)
+
+        # if meta.get('fwd_from'):
+            # if meta['fwd_from'].get('post_author') # user first name
+
+        # TODO reply threads ????
+
+
         return None

bin/lib/ail_core.py

@@ -15,8 +15,8 @@ config_loader = ConfigLoader()
 r_serv_db = config_loader.get_db_conn("Kvrocks_DB")
 config_loader = None
 
-AIL_OBJECTS = sorted({'cookie-name', 'cve', 'cryptocurrency', 'decoded', 'domain', 'etag', 'favicon', 'hhhash', 'item',
+AIL_OBJECTS = sorted({'chat', 'cookie-name', 'cve', 'cryptocurrency', 'decoded', 'domain', 'etag', 'favicon', 'hhhash', 'item',
-                      'pgp', 'screenshot', 'title', 'username'})
+                      'pgp', 'screenshot', 'title', 'user-account', 'username'})
 
 def get_ail_uuid():
     ail_uuid = r_serv_db.get('ail:uuid')
@@ -38,9 +38,11 @@ def get_all_objects():
     return AIL_OBJECTS
 
 def get_objects_with_subtypes():
-    return ['cryptocurrency', 'pgp', 'username']
+    return ['chat', 'cryptocurrency', 'pgp', 'username']
 
 def get_object_all_subtypes(obj_type):
+    if obj_type == 'chat':
+        return ['discord', 'jabber', 'telegram']
     if obj_type == 'cryptocurrency':
         return ['bitcoin', 'bitcoin-cash', 'dash', 'ethereum', 'litecoin', 'monero', 'zcash']
     if obj_type == 'pgp':
@@ -66,6 +68,14 @@ def get_all_objects_with_subtypes_tuple():
             str_objs.append((obj_type, ''))
     return str_objs
 
+def unpack_obj_global_id(global_id, r_type='tuple'):
+    if r_type == 'dict':
+        obj = global_id.split(':', 2)
+        return {'type': obj[0], 'subtype': obj[1], 'id': obj['2']}
+    else:  # tuple(type, subtype, id)
+        return global_id.split(':', 2)
+
+
 ##-- AIL OBJECTS --##
 
 ####    Redis     ####

bin/lib/correlations_engine.py

@@ -41,6 +41,7 @@ config_loader = None
 ##################################
 
 CORRELATION_TYPES_BY_OBJ = {
+    "chat": ["item", "username"],  # item ???
     "cookie-name": ["domain"],
     "cryptocurrency": ["domain", "item"],
     "cve": ["domain", "item"],
@@ -49,11 +50,11 @@ CORRELATION_TYPES_BY_OBJ = {
     "etag": ["domain"],
     "favicon": ["domain", "item"],  # TODO Decoded
     "hhhash": ["domain"],
-    "item": ["cve", "cryptocurrency", "decoded", "domain", "favicon", "pgp", "screenshot", "title", "username"],
+    "item": ["chat", "cve", "cryptocurrency", "decoded", "domain", "favicon", "pgp", "screenshot", "title", "username"],
     "pgp": ["domain", "item"],
     "screenshot": ["domain", "item"],
     "title": ["domain", "item"],
-    "username": ["domain", "item"],
+    "username": ["chat", "domain", "item"],
 }
 
 def get_obj_correl_types(obj_type):
@@ -65,6 +66,8 @@ def sanityze_obj_correl_types(obj_type, correl_types):
         correl_types = set(correl_types).intersection(obj_correl_types)
     if not correl_types:
         correl_types = obj_correl_types
+        if not correl_types:
+            return []
     return correl_types
 
 def get_nb_correlation_by_correl_type(obj_type, subtype, obj_id, correl_type):

bin/lib/objects/Chats.py

@@ -0,0 +1,288 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import os
+import sys
+
+from datetime import datetime
+
+from flask import url_for
+# from pymisp import MISPObject
+
+sys.path.append(os.environ['AIL_BIN'])
+##################################
+# Import Project packages
+##################################
+from lib import ail_core
+from lib.ConfigLoader import ConfigLoader
+from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id
+from lib.data_retention_engine import update_obj_date
+from lib.objects import ail_objects
+from lib import item_basic
+
+from lib.correlations_engine import get_correlation_by_correl_type
+
+config_loader = ConfigLoader()
+baseurl = config_loader.get_config_str("Notifications", "ail_domain")
+r_object = config_loader.get_db_conn("Kvrocks_Objects")
+r_cache = config_loader.get_redis_conn("Redis_Cache")
+config_loader = None
+
+
+################################################################################
+################################################################################
+################################################################################
+
+class Chat(AbstractSubtypeObject):  # TODO # ID == username ?????
+    """
+    AIL Chat Object. (strings)
+    """
+
+    def __init__(self, id, subtype):
+        super(Chat, self).__init__('chat', id, subtype)
+
+    # def get_ail_2_ail_payload(self):
+    #     payload = {'raw': self.get_gzip_content(b64=True),
+    #                 'compress': 'gzip'}
+    #     return payload
+
+    # # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\
+    def delete(self):
+        # # TODO:
+        pass
+
+    def get_link(self, flask_context=False):
+        if flask_context:
+            url = url_for('correlation.show_correlation', type=self.type, subtype=self.subtype, id=self.id)
+        else:
+            url = f'{baseurl}/correlation/show?type={self.type}&subtype={self.subtype}&id={self.id}'
+        return url
+
+    def get_svg_icon(self):  # TODO
+        # if self.subtype == 'telegram':
+        #     style = 'fab'
+        #     icon = '\uf2c6'
+        # elif self.subtype == 'discord':
+        #     style = 'fab'
+        #     icon = '\uf099'
+        # else:
+        #     style = 'fas'
+        #     icon = '\uf007'
+        style = 'fas'
+        icon = '\uf086'
+        return {'style': style, 'icon': icon, 'color': '#4dffff', 'radius': 5}
+
+    def get_meta(self, options=set()):
+        meta = self._get_meta(options=options)
+        meta['id'] = self.id
+        meta['subtype'] = self.subtype
+        meta['tags'] = self.get_tags(r_list=True)
+        return meta
+
+    def get_misp_object(self):
+        # obj_attrs = []
+        # if self.subtype == 'telegram':
+        #     obj = MISPObject('telegram-account', standalone=True)
+        #     obj_attrs.append(obj.add_attribute('username', value=self.id))
+        #
+        # elif self.subtype == 'twitter':
+        #     obj = MISPObject('twitter-account', standalone=True)
+        #     obj_attrs.append(obj.add_attribute('name', value=self.id))
+        #
+        # else:
+        #     obj = MISPObject('user-account', standalone=True)
+        #     obj_attrs.append(obj.add_attribute('username', value=self.id))
+        #
+        # first_seen = self.get_first_seen()
+        # last_seen = self.get_last_seen()
+        # if first_seen:
+        #     obj.first_seen = first_seen
+        # if last_seen:
+        #     obj.last_seen = last_seen
+        # if not first_seen or not last_seen:
+        #     self.logger.warning(
+        #         f'Export error, None seen {self.type}:{self.subtype}:{self.id}, first={first_seen}, last={last_seen}')
+        #
+        # for obj_attr in obj_attrs:
+        #     for tag in self.get_tags():
+        #         obj_attr.add_tag(tag)
+        # return obj
+        return
+
+    ############################################################################
+    ############################################################################
+
+    # others optional metas, ... -> # TODO ALL meta in hset
+
+    def get_name(self):  # get username ????
+        pass
+
+    # return username correlation
+    def get_users(self):  # get participants ??? -> passive users ???
+        pass
+
+    # def get_last_message_id(self):
+    #
+    #     return r_object.hget(f'meta:{self.type}:{self.subtype}:{self.id}', 'last:message:id')
+
+    def get_obj_message_id(self, obj_id):
+        if obj_id.endswith('.gz'):
+            obj_id = obj_id[:-3]
+        return int(obj_id.split('_')[-1])
+
+    def _get_message_timestamp(self, obj_global_id):
+        return r_object.zscore(f'messages:{self.type}:{self.subtype}:{self.id}', obj_global_id)
+
+    def _get_messages(self):
+        return r_object.zrange(f'messages:{self.type}:{self.subtype}:{self.id}', 0, -1, withscores=True)
+
+    def get_message_meta(self, obj_global_id, parent=True, mess_datetime=None):
+        obj = ail_objects.get_obj_from_global_id(obj_global_id)
+        mess_dict = obj.get_meta(options={'content', 'link', 'parent'})
+        if mess_dict.get('parent') and parent:
+            mess_dict['reply_to'] = self.get_message_meta(mess_dict['parent'], parent=False)
+        mess_dict['username'] = {}
+        user = obj.get_correlation('username').get('username')
+        if user:
+            subtype, user = user.pop().split(':', 1)
+            mess_dict['username']['type'] = 'telegram'
+            mess_dict['username']['subtype'] = subtype
+            mess_dict['username']['id'] = user
+        else:
+            mess_dict['username']['id'] = 'UNKNOWN'
+
+        if not mess_datetime:
+            obj_mess_id = self._get_message_timestamp(obj_global_id)
+            mess_datetime = datetime.fromtimestamp(obj_mess_id)
+        mess_dict['date'] = mess_datetime.isoformat(' ')
+        mess_dict['hour'] = mess_datetime.strftime('%H:%M:%S')
+        return mess_dict
+
+
+    def get_messages(self, start=0, page=1, nb=500):  # TODO limit nb returned, # TODO add replies
+        start = 0
+        stop = -1
+        # r_object.delete(f'messages:{self.type}:{self.subtype}:{self.id}')
+
+        # TODO chat without username ???? -> chat ID ????
+
+        messages = {}
+        curr_date = None
+        for message in self._get_messages():
+            date = datetime.fromtimestamp(message[1])
+            date_day = date.strftime('%Y/%m/%d')
+            if date_day != curr_date:
+                messages[date_day] = []
+                curr_date = date_day
+            mess_dict = self.get_message_meta(message[0], parent=True, mess_datetime=date)
+            messages[date_day].append(mess_dict)
+        return messages
+
+        # Zset with ID ???  id -> item id ??? multiple id == media + text
+        #                   id -> media id
+        # How do we handle reply/thread ??? -> separate with new chats name/id ZSET ???
+        # Handle media ???
+
+        # list of message id -> obj_id
+        # list of obj_id ->
+        # abuse parent children ???
+
+    # def add(self, timestamp, obj_id, mess_id=0, username=None, user_id=None):
+    #     date = # TODO get date from object
+    #     self.update_daterange(date)
+    #     update_obj_date(date, self.type, self.subtype)
+    #
+    #
+    #     # daily
+    #     r_object.hincrby(f'{self.type}:{self.subtype}:{date}', self.id, 1)
+    #     # all subtypes
+    #     r_object.zincrby(f'{self.type}_all:{self.subtype}', 1, self.id)
+    #
+    #     #######################################################################
+    #     #######################################################################
+    #
+    #     # Correlations
+    #     self.add_correlation('item', '', item_id)
+    #     # domain
+    #     if is_crawled(item_id):
+    #         domain = get_item_domain(item_id)
+    #         self.add_correlation('domain', '', domain)
+
+    # TODO kvrocks exception if key don't exists
+    def get_obj_by_message_id(self, mess_id):
+        return r_object.hget(f'messages:ids:{self.type}:{self.subtype}:{self.id}', mess_id)
+
+    # importer -> use cache for previous reply SET to_add_id: previously_imported : expire SET key -> 30 mn
+    def add_message(self, obj_global_id, timestamp, mess_id, reply_id=None):
+        r_object.hset(f'messages:ids:{self.type}:{self.subtype}:{self.id}', mess_id, obj_global_id)
+        r_object.zadd(f'messages:{self.type}:{self.subtype}:{self.id}', {obj_global_id: timestamp})
+
+        if reply_id:
+            reply_obj = self.get_obj_by_message_id(reply_id)
+            if reply_obj:
+                self.add_obj_children(reply_obj, obj_global_id)
+            else:
+                self.add_message_cached_reply(reply_id, mess_id)
+
+        # ADD cached replies
+        for reply_obj in self.get_cached_message_reply(mess_id):
+            self.add_obj_children(obj_global_id, reply_obj)
+
+    def _get_message_cached_reply(self, message_id):
+        return r_cache.smembers(f'messages:ids:{self.type}:{self.subtype}:{self.id}:{message_id}')
+
+    def get_cached_message_reply(self, message_id):
+        objs_global_id = []
+        for mess_id in self._get_message_cached_reply(message_id):
+            obj_global_id = self.get_obj_by_message_id(mess_id)
+            if obj_global_id:
+                objs_global_id.append(obj_global_id)
+        return objs_global_id
+
+    def add_message_cached_reply(self, reply_to_id, message_id):
+        r_cache.sadd(f'messages:ids:{self.type}:{self.subtype}:{self.id}:{reply_to_id}', message_id)
+        r_cache.expire(f'messages:ids:{self.type}:{self.subtype}:{self.id}:{reply_to_id}', 600)
+
+    # TODO nb replies = nb son ???? what if it create a onion item ??? -> need source filtering
+
+
+# TODO factorize
+def get_all_subtypes():
+    return ail_core.get_object_all_subtypes('chat')
+
+def get_all():
+    objs = {}
+    for subtype in get_all_subtypes():
+        objs[subtype] = get_all_by_subtype(subtype)
+    return objs
+
+def get_all_by_subtype(subtype):
+    return get_all_id('chat', subtype)
+
+# # TODO FILTER NAME + Key + mail
+# def sanitize_username_name_to_search(name_to_search, subtype): # TODO FILTER NAME
+#
+#     return name_to_search
+#
+# def search_usernames_by_name(name_to_search, subtype, r_pos=False):
+#     usernames = {}
+#     # for subtype in subtypes:
+#     r_name = sanitize_username_name_to_search(name_to_search, subtype)
+#     if not name_to_search or isinstance(r_name, dict):
+#         # break
+#         return usernames
+#     r_name = re.compile(r_name)
+#     for user_name in get_all_usernames_by_subtype(subtype):
+#         res = re.search(r_name, user_name)
+#         if res:
+#             usernames[user_name] = {}
+#             if r_pos:
+#                 usernames[user_name]['hl-start'] = res.start()
+#                 usernames[user_name]['hl-end'] = res.end()
+#     return usernames
+
+
+if __name__ == '__main__':
+    chat = Chat('test', 'telegram')
+    r = chat.get_messages()
+    print(r)

bin/lib/objects/Items.py

@@ -288,6 +288,8 @@ class Item(AbstractObject):
             meta['mimetype'] = self.get_mimetype(content=content)
         if 'investigations' in options:
             meta['investigations'] = self.get_investigations()
+        if 'link' in options:
+            meta['link'] = self.get_link(flask_context=True)
 
         # meta['encoding'] = None
         return meta

bin/lib/objects/Messages.py

@@ -0,0 +1,268 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import os
+import re
+import sys
+import cld3
+import html2text
+
+from datetime import datetime
+
+from pymisp import MISPObject
+
+sys.path.append(os.environ['AIL_BIN'])
+##################################
+# Import Project packages
+##################################
+from lib.ail_core import get_ail_uuid
+from lib.objects.abstract_object import AbstractObject
+from lib.ConfigLoader import ConfigLoader
+from lib.data_retention_engine import update_obj_date, get_obj_date_first
+# TODO Set all messages ???
+
+
+from flask import url_for
+
+config_loader = ConfigLoader()
+r_cache = config_loader.get_redis_conn("Redis_Cache")
+r_object = config_loader.get_db_conn("Kvrocks_Objects")
+r_content = config_loader.get_db_conn("Kvrocks_Content")
+baseurl = config_loader.get_config_str("Notifications", "ail_domain")
+config_loader = None
+
+
+# TODO SAVE OR EXTRACT MESSAGE SOURCE FOR ICON ?????????
+# TODO iterate on all objects
+# TODO also add support for small objects ????
+
+# CAN Message exists without CHAT -> no convert it to object
+
+# ID:   source:chat_id:message_id ????
+#
+# /!\ handle null chat and message id -> chat = uuid and message = timestamp ???
+
+
+class Message(AbstractObject):
+    """
+    AIL Message Object. (strings)
+    """
+
+    def __init__(self, id):  # TODO subtype or use source ????
+        super(Message, self).__init__('message', id)  # message::< telegram/1692189934.380827/ChatID_MessageID >
+
+    def exists(self):
+        if self.subtype is None:
+            return r_object.exists(f'meta:{self.type}:{self.id}')
+        else:
+            return r_object.exists(f'meta:{self.type}:{self.get_subtype(r_str=True)}:{self.id}')
+
+    def get_source(self):
+        """
+        Returns source/feeder name
+        """
+        l_source = self.id.split('/')[:-4]
+        return os.path.join(*l_source)
+
+    def get_basename(self):
+        return os.path.basename(self.id)
+
+    def get_content(self, r_type='str'): # TODO ADD cache # TODO Compress content ???????
+        """
+        Returns content
+        """
+        content = self._get_field('content')
+        if r_type == 'str':
+            return content
+        elif r_type == 'bytes':
+            return content.encode()
+
+    def get_date(self):
+        timestamp = self.get_timestamp()
+        return datetime.fromtimestamp(timestamp).strftime('%Y%m%d')
+
+    def get_timestamp(self):
+        dirs = self.id.split('/')
+        return dirs[-2]
+
+    def get_message_id(self):  # TODO optimize
+        message_id = self.get_basename().rsplit('_', 1)[1]
+        # if message_id.endswith('.gz'):
+        #     message_id = message_id[:-3]
+        return message_id
+
+    def get_chat_id(self):  # TODO optimize
+        chat_id =  self.get_basename().rsplit('_', 1)[0]
+        # if chat_id.endswith('.gz'):
+        #     chat_id = chat_id[:-3]
+        return chat_id
+
+    # Update value on import
+    # reply to -> parent ?
+    # reply/comment - > children ?
+    # nb views
+    # reactions
+    # nb fowards
+    # room ???
+    # message from channel ???
+    # message media
+
+    def get_translation(self):  # TODO support multiple translated languages ?????
+        """
+        Returns translated content
+        """
+        return self._get_field('translated')  # TODO multiples translation ... -> use set
+
+    def _set_translation(self, translation):
+        """
+        Set translated content
+        """
+        return self._set_field('translated', translation)  # translation by hash ??? -> avoid translating multiple time
+
+    def get_html2text_content(self, content=None, ignore_links=False):
+        if not content:
+            content = self.get_content()
+        h = html2text.HTML2Text()
+        h.ignore_links = ignore_links
+        h.ignore_images = ignore_links
+        return h.handle(content)
+
+    # def get_ail_2_ail_payload(self):
+    #     payload = {'raw': self.get_gzip_content(b64=True)}
+    #     return payload
+
+    def get_link(self, flask_context=False):
+        if flask_context:
+            url = url_for('correlation.show_correlation', type=self.type, id=self.id)
+        else:
+            url = f'{baseurl}/correlation/show?type={self.type}&id={self.id}'
+        return url
+
+    def get_svg_icon(self):
+        return {'style': 'fas', 'icon': 'fa-comment-dots', 'color': '#4dffff', 'radius': 5}
+
+    def get_misp_object(self):  # TODO
+        obj = MISPObject('instant-message', standalone=True)
+        obj_date = self.get_date()
+        if obj_date:
+            obj.first_seen = obj_date
+        else:
+            self.logger.warning(
+                f'Export error, None seen {self.type}:{self.subtype}:{self.id}, first={obj_date}')
+
+        # obj_attrs = [obj.add_attribute('first-seen', value=obj_date),
+        #              obj.add_attribute('raw-data', value=self.id, data=self.get_raw_content()),
+        #              obj.add_attribute('sensor', value=get_ail_uuid())]
+        obj_attrs = []
+        for obj_attr in obj_attrs:
+            for tag in self.get_tags():
+                obj_attr.add_tag(tag)
+        return obj
+
+    # def get_url(self):
+    #     return r_object.hget(f'meta:item::{self.id}', 'url')
+
+    # options: set of optional meta fields
+    def get_meta(self, options=None):
+        """
+        :type options: set
+        """
+        if options is None:
+            options = set()
+        meta = self.get_default_meta(tags=True)
+        meta['date'] = self.get_date() # TODO replace me by timestamp ??????
+        meta['source'] = self.get_source()
+        # optional meta fields
+        if 'content' in options:
+            meta['content'] = self.get_content()
+        if 'parent' in options:
+            meta['parent'] = self.get_parent()
+        if 'investigations' in options:
+            meta['investigations'] = self.get_investigations()
+        if 'link' in options:
+            meta['link'] = self.get_link(flask_context=True)
+
+        # meta['encoding'] = None
+        return meta
+
+    def _languages_cleaner(self, content=None):
+        if not content:
+            content = self.get_content()
+        # REMOVE URLS
+        regex = r'\b(?:http://|https://)?(?:[a-zA-Z\d-]{,63}(?:\.[a-zA-Z\d-]{,63})+)(?:\:[0-9]+)*(?:/(?:$|[a-zA-Z0-9\.\,\?\'\\\+&%\$#\=~_\-]+))*\b'
+        url_regex = re.compile(regex)
+        urls = url_regex.findall(content)
+        urls = sorted(urls, key=len, reverse=True)
+        for url in urls:
+            content = content.replace(url, '')
+        # REMOVE PGP Blocks
+        regex_pgp_public_blocs = r'-----BEGIN PGP PUBLIC KEY BLOCK-----[\s\S]+?-----END PGP PUBLIC KEY BLOCK-----'
+        regex_pgp_signature = r'-----BEGIN PGP SIGNATURE-----[\s\S]+?-----END PGP SIGNATURE-----'
+        regex_pgp_message = r'-----BEGIN PGP MESSAGE-----[\s\S]+?-----END PGP MESSAGE-----'
+        re.compile(regex_pgp_public_blocs)
+        re.compile(regex_pgp_signature)
+        re.compile(regex_pgp_message)
+        res = re.findall(regex_pgp_public_blocs, content)
+        for it in res:
+            content = content.replace(it, '')
+        res = re.findall(regex_pgp_signature, content)
+        for it in res:
+            content = content.replace(it, '')
+        res = re.findall(regex_pgp_message, content)
+        for it in res:
+            content = content.replace(it, '')
+        return content
+
+    def detect_languages(self, min_len=600, num_langs=3, min_proportion=0.2, min_probability=0.7):
+        languages = []
+        ## CLEAN CONTENT ##
+        content = self.get_html2text_content(ignore_links=True)
+        content = self._languages_cleaner(content=content)
+        # REMOVE USELESS SPACE
+        content = ' '.join(content.split())
+        # - CLEAN CONTENT - #
+        if len(content) >= min_len:
+            for lang in cld3.get_frequent_languages(content, num_langs=num_langs):
+                if lang.proportion >= min_proportion and lang.probability >= min_probability and lang.is_reliable:
+                    languages.append(lang)
+        return languages
+
+    # def translate(self, content=None): # TODO translation plugin
+    #     # TODO get text language
+    #     if not content:
+    #         content = self.get_content()
+    #     translated = argostranslate.translate.translate(content, 'ru', 'en')
+    #     # Save translation
+    #     self._set_translation(translated)
+    #     return translated
+
+    def create(self, content, translation, tags):
+        self._set_field('content', content)
+        r_content.get(f'content:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', content)
+        if translation:
+            self._set_translation(translation)
+        for tag in tags:
+            self.add_tag(tag)
+
+    # # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\
+    def delete(self):
+        pass
+
+def create_obj_id(source, chat_id, message_id, timestamp):
+    return f'{source}/{timestamp}/{chat_id}_{message_id}'
+
+# TODO Check if already exists
+# def create(source, chat_id, message_id, timestamp, content, tags=[]):
+def create(obj_id, content, translation=None, tags=[]):
+    message = Message(obj_id)
+    if not message.exists():
+        message.create(content, translation, tags)
+    return message
+
+
+# TODO Encode translation
+
+
+if __name__ == '__main__':
+    r = 'test'
+    print(r)

bin/lib/objects/UsersAccount.py

@@ -0,0 +1,154 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import os
+import sys
+import re
+
+from flask import url_for
+from pymisp import MISPObject
+
+sys.path.append(os.environ['AIL_BIN'])
+##################################
+# Import Project packages
+##################################
+from lib import ail_core
+from lib.ConfigLoader import ConfigLoader
+from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id
+
+config_loader = ConfigLoader()
+baseurl = config_loader.get_config_str("Notifications", "ail_domain")
+config_loader = None
+
+
+################################################################################
+################################################################################
+################################################################################
+
+class UserAccount(AbstractSubtypeObject):
+    """
+    AIL User Object. (strings)
+    """
+
+    def __init__(self, id, subtype):
+        super(UserAccount, self).__init__('user-account', id, subtype)
+
+    # def get_ail_2_ail_payload(self):
+    #     payload = {'raw': self.get_gzip_content(b64=True),
+    #                 'compress': 'gzip'}
+    #     return payload
+
+    # # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\
+    def delete(self):
+        # # TODO:
+        pass
+
+    def get_link(self, flask_context=False):
+        if flask_context:
+            url = url_for('correlation.show_correlation', type=self.type, subtype=self.subtype, id=self.id)
+        else:
+            url = f'{baseurl}/correlation/show?type={self.type}&subtype={self.subtype}&id={self.id}'
+        return url
+
+    def get_svg_icon(self): # TODO change icon/color
+        if self.subtype == 'telegram':
+            style = 'fab'
+            icon = '\uf2c6'
+        elif self.subtype == 'twitter':
+            style = 'fab'
+            icon = '\uf099'
+        else:
+            style = 'fas'
+            icon = '\uf007'
+        return {'style': style, 'icon': icon, 'color': '#4dffff', 'radius': 5}
+
+    def get_first_name(self):
+        return self._get_field('firstname')
+
+    def get_last_name(self):
+        return self._get_field('lastname')
+
+    def get_phone(self):
+        return self._get_field('phone')
+
+    def set_first_name(self, firstname):
+        return self._set_field('firstname', firstname)
+
+    def set_last_name(self, lastname):
+        return self._set_field('lastname', lastname)
+
+    def set_phone(self, phone):
+        return self._set_field('phone', phone)
+
+    # TODO REWRITE ADD FUNCTION
+
+    def get_username(self):
+        return ''
+
+    def get_usernames(self):
+        usernames = []
+        correl = self.get_correlation('username')
+        for partial_id in correl.get('username', []):
+            usernames.append(f'username:{partial_id}')
+        return usernames
+
+    def get_meta(self, options=set()):
+        meta = self._get_meta(options=options)
+        meta['id'] = self.id
+        meta['subtype'] = self.subtype
+        meta['tags'] = self.get_tags(r_list=True)
+        if 'username' in options:
+            meta['username'] = self.get_username()
+        if 'usernames' in options:
+            meta['usernames'] = self.get_usernames()
+        return meta
+
+    def get_misp_object(self):
+        obj_attrs = []
+        if self.subtype == 'telegram':
+            obj = MISPObject('telegram-account', standalone=True)
+            obj_attrs.append(obj.add_attribute('username', value=self.id))
+
+        elif self.subtype == 'twitter':
+            obj = MISPObject('twitter-account', standalone=True)
+            obj_attrs.append(obj.add_attribute('name', value=self.id))
+
+        else:
+            obj = MISPObject('user-account', standalone=True)
+            obj_attrs.append(obj.add_attribute('username', value=self.id))
+
+        first_seen = self.get_first_seen()
+        last_seen = self.get_last_seen()
+        if first_seen:
+            obj.first_seen = first_seen
+        if last_seen:
+            obj.last_seen = last_seen
+        if not first_seen or not last_seen:
+            self.logger.warning(
+                f'Export error, None seen {self.type}:{self.subtype}:{self.id}, first={first_seen}, last={last_seen}')
+
+        for obj_attr in obj_attrs:
+            for tag in self.get_tags():
+                obj_attr.add_tag(tag)
+        return obj
+
+def get_user_by_username():
+    pass
+
+def get_all_subtypes():
+    return ail_core.get_object_all_subtypes('user-account')
+
+def get_all():
+    users = {}
+    for subtype in get_all_subtypes():
+        users[subtype] = get_all_by_subtype(subtype)
+    return users
+
+def get_all_by_subtype(subtype):
+    return get_all_id('user-account', subtype)
+
+
+# if __name__ == '__main__':
+#     name_to_search = 'co'
+#     subtype = 'telegram'
+#     print(search_usernames_by_name(name_to_search, subtype))

bin/lib/objects/abstract_daterange_object.py

@@ -45,10 +45,10 @@ class AbstractDaterangeObject(AbstractObject, ABC):
     def exists(self):
         return r_object.exists(f'meta:{self.type}:{self.id}')
 
-    def _get_field(self, field):
+    def _get_field(self, field): # TODO remove me (NEW in abstract)
         return r_object.hget(f'meta:{self.type}:{self.id}', field)
 
-    def _set_field(self, field, value):
+    def _set_field(self, field, value): # TODO remove me (NEW in abstract)
         return r_object.hset(f'meta:{self.type}:{self.id}', field, value)
 
     def get_first_seen(self, r_int=False):

bin/lib/objects/abstract_object.py

@@ -20,6 +20,7 @@ sys.path.append(os.environ['AIL_BIN'])
 ##################################
 from lib import ail_logger
 from lib import Tag
+from lib.ConfigLoader import ConfigLoader
 from lib import Duplicate
 from lib.correlations_engine import get_nb_correlations, get_correlations, add_obj_correlation, delete_obj_correlation, delete_obj_correlations, exists_obj_correlation, is_obj_correlated, get_nb_correlation_by_correl_type
 from lib.Investigations import is_object_investigated, get_obj_investigations, delete_obj_investigations
@@ -27,6 +28,11 @@ from lib.Tracker import is_obj_tracked, get_obj_trackers, delete_obj_trackers
 
 logging.config.dictConfig(ail_logger.get_config(name='ail'))
 
+config_loader = ConfigLoader()
+# r_cache = config_loader.get_redis_conn("Redis_Cache")
+r_object = config_loader.get_db_conn("Kvrocks_Objects")
+config_loader = None
+
 class AbstractObject(ABC):
     """
     Abstract Object
@@ -67,6 +73,18 @@ class AbstractObject(ABC):
             dict_meta['tags'] = self.get_tags()
         return dict_meta
 
+    def _get_field(self, field):
+        if self.subtype is None:
+            return r_object.hget(f'meta:{self.type}:{self.id}', field)
+        else:
+            return r_object.hget(f'meta:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', field)
+
+    def _set_field(self, field, value):
+        if self.subtype is None:
+            return r_object.hset(f'meta:{self.type}:{self.id}', field, value)
+        else:
+            return r_object.hset(f'meta:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', field, value)
+
     ## Tags ##
     def get_tags(self, r_list=False):
         tags = Tag.get_object_tags(self.type, self.id, self.get_subtype(r_str=True))
@@ -198,6 +216,8 @@ class AbstractObject(ABC):
         else:
             return []
 
+    ## Correlation ##
+
     def _get_external_correlation(self, req_type, req_subtype, req_id, obj_type):
         """
         Get object correlation
@@ -253,3 +273,39 @@ class AbstractObject(ABC):
         Get object correlations
         """
         delete_obj_correlation(self.type, self.subtype, self.id, type2, subtype2, id2)
+
+    ## -Correlation- ##
+
+    ## Parent ##
+
+    def is_parent(self):
+        return r_object.exists(f'child:{self.type}:{self.get_subtype(r_str=True)}:{self.id}')
+
+    def is_children(self):
+        return r_object.hexists(f'meta:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', 'parent')
+
+    def get_parent(self):
+        return r_object.hget(f'meta:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', 'parent')
+
+    def get_children(self):
+        return r_object.smembers(f'child:{self.type}:{self.get_subtype(r_str=True)}:{self.id}')
+
+    def set_parent(self, obj_type=None, obj_subtype=None, obj_id=None, obj_global_id=None): # TODO ######################
+        if not obj_global_id:
+            if obj_subtype is None:
+                obj_subtype = ''
+            obj_global_id = f'{obj_type}:{obj_subtype}:{obj_id}'
+        r_object.hset(f'meta:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', 'parent', obj_global_id)
+
+    def add_children(self, obj_type=None, obj_subtype=None, obj_id=None, obj_global_id=None): # TODO ######################
+        if not obj_global_id:
+            if obj_subtype is None:
+                obj_subtype = ''
+            obj_global_id = f'{obj_type}:{obj_subtype}:{obj_id}'
+        r_object.sadd(f'child:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', obj_global_id)
+
+    def add_obj_children(self, parent_global_id, son_global_id):
+        r_object.sadd(f'child:{parent_global_id}', son_global_id)
+        r_object.hset(f'meta:{son_global_id}', 'parent', parent_global_id)
+
+    ## Parent ##

bin/lib/objects/ail_objects.py

@@ -13,6 +13,7 @@ from lib import correlations_engine
 from lib import btc_ail
 from lib import Tag
 
+from lib.objects import Chats
 from lib.objects import CryptoCurrencies
 from lib.objects import CookiesNames
 from lib.objects.Cves import Cve
@@ -55,6 +56,8 @@ def get_object(obj_type, subtype, id):
         return Domain(id)
     elif obj_type == 'decoded':
         return Decoded(id)
+    elif obj_type == 'chat':
+        return Chats.Chat(id, subtype)
     elif obj_type == 'cookie-name':
         return CookiesNames.CookieName(id)
     elif obj_type == 'cve':

bin/lib/timeline_engine.py

@@ -0,0 +1,157 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import os
+import sys
+
+from uuid import uuid4
+
+sys.path.append(os.environ['AIL_BIN'])
+##################################
+# Import Project packages
+##################################
+from lib.ConfigLoader import ConfigLoader
+
+config_loader = ConfigLoader()
+r_meta = config_loader.get_db_conn("Kvrocks_Timeline")
+config_loader = None
+
+# CORRELATION_TYPES_BY_OBJ = {
+#     "chat": ["item", "username"],  # item ???
+#     "cookie-name": ["domain"],
+#     "cryptocurrency": ["domain", "item"],
+#     "cve": ["domain", "item"],
+#     "decoded": ["domain", "item"],
+#     "domain": ["cve", "cookie-name", "cryptocurrency", "decoded", "etag", "favicon", "hhhash", "item", "pgp", "title", "screenshot", "username"],
+#     "etag": ["domain"],
+#     "favicon": ["domain", "item"],
+#     "hhhash": ["domain"],
+#     "item": ["chat", "cve", "cryptocurrency", "decoded", "domain", "favicon", "pgp", "screenshot", "title", "username"],
+#     "pgp": ["domain", "item"],
+#     "screenshot": ["domain", "item"],
+#     "title": ["domain", "item"],
+#     "username": ["chat", "domain", "item"],
+# }
+#
+# def get_obj_correl_types(obj_type):
+#     return CORRELATION_TYPES_BY_OBJ.get(obj_type)
+
+# def sanityze_obj_correl_types(obj_type, correl_types):
+#     obj_correl_types = get_obj_correl_types(obj_type)
+#     if correl_types:
+#         correl_types = set(correl_types).intersection(obj_correl_types)
+#     if not correl_types:
+#         correl_types = obj_correl_types
+#         if not correl_types:
+#             return []
+#     return correl_types
+
+# TODO rename all function + add missing parameters
+
+def get_bloc_obj_global_id(bloc):
+    return r_meta.hget('hset:key', bloc)
+
+def set_bloc_obj_global_id(bloc, global_id):
+    return r_meta.hset('hset:key', bloc, global_id)
+
+def get_bloc_timestamp(bloc, position):
+    return r_meta.zscore('key', f'{position}:{bloc}')
+
+def add_bloc(global_id, timestamp, end=None):
+    if end:
+        timestamp_end = end
+    else:
+        timestamp_end = timestamp
+    new_bloc = str(uuid4())
+    r_meta.zadd('key', {f'start:{new_bloc}': timestamp, f'end:{new_bloc}': timestamp_end})
+    set_bloc_obj_global_id(new_bloc, global_id)
+    return new_bloc
+
+def _update_bloc(bloc, position, timestamp):
+    r_meta.zadd('key', {f'{position}:{bloc}': timestamp})
+
+# score = timestamp
+def get_nearest_bloc_inf(timestamp):
+    return r_meta.zrevrangebyscore('key', timestamp, 0, num=1)
+
+def get_nearest_bloc_sup(timestamp):
+    return r_meta.zrangebyscore('key', timestamp, 0, num=1)
+
+#######################################################################################
+
+def add_timestamp(timestamp, obj_global_id):
+    inf = get_nearest_bloc_inf(timestamp)
+    sup = get_nearest_bloc_sup(timestamp)
+    if not inf and not sup:
+        # create new bloc
+        new_bloc = add_bloc(obj_global_id, timestamp)
+        return new_bloc
+    # timestamp < first_seen
+    elif not inf:
+        sup_pos, sup_id = inf.split(':')
+        sup_obj = get_bloc_obj_global_id(sup_pos)
+        if sup_obj == obj_global_id:
+            _update_bloc(sup_id, 'start', timestamp)
+        # create new bloc
+        else:
+            new_bloc = add_bloc(obj_global_id, timestamp)
+            return new_bloc
+
+    # timestamp > first_seen
+    elif not sup:
+        inf_pos, inf_id = inf.split(':')
+        inf_obj = get_bloc_obj_global_id(inf_id)
+        if inf_obj == obj_global_id:
+            _update_bloc(inf_id, 'end', timestamp)
+        # create new bloc
+        else:
+            new_bloc = add_bloc(obj_global_id, timestamp)
+            return new_bloc
+
+    else:
+        inf_pos, inf_id = inf.split(':')
+        sup_pos, sup_id = inf.split(':')
+        inf_obj = get_bloc_obj_global_id(inf_id)
+
+        if inf_id == sup_id:
+            # reduce bloc + create two new bloc
+            if obj_global_id != inf_obj:
+                # get end timestamp
+                sup_timestamp = get_bloc_timestamp(sup_id, 'end')
+                # reduce original bloc
+                _update_bloc(inf_id, 'end', timestamp - 1)
+                # Insert new bloc
+                new_bloc = add_bloc(obj_global_id, timestamp)
+                # Recreate end of the first bloc by a new bloc
+                add_bloc(inf_obj, timestamp + 1, end=sup_timestamp)
+                return new_bloc
+
+            # timestamp in existing bloc
+            else:
+                return inf_id
+
+        # different blocs: expend sup/inf bloc or create a new bloc if
+        elif inf_pos == 'end' and sup_pos == 'start':
+            # Extend inf bloc
+            if obj_global_id == inf_obj:
+                _update_bloc(inf_id, 'end', timestamp)
+                return inf_id
+
+            sup_obj = get_bloc_obj_global_id(sup_pos)
+            # Extend sup bloc
+            if obj_global_id == sup_obj:
+                _update_bloc(sup_id, 'start', timestamp)
+                return sup_id
+
+            # create new bloc
+            new_bloc = add_bloc(obj_global_id, timestamp)
+            return new_bloc
+
+        # inf_pos == 'start' and sup_pos == 'end'
+        # else raise error ???
+
+
+
+
+
+

configs/6383.conf

@@ -663,6 +663,7 @@ namespace.crawl ail_crawlers
 namespace.db ail_datas
 namespace.dup ail_dups
 namespace.obj ail_objs
+namespace.tl ail_tls
 namespace.stat ail_stats
 namespace.tag ail_tags
 namespace.track ail_trackers

configs/core.cfg.sample

@@ -190,6 +190,11 @@ host = localhost
 port = 6383
 password = ail_objs
 
+[Kvrocks_Timeline]
+host = localhost
+port = 6383
+password = ail_tls
+
 [Kvrocks_Stats]
 host = localhost
 port = 6383

var/www/Flask_server.py

@@ -50,6 +50,7 @@ from blueprints.objects_title import objects_title
 from blueprints.objects_cookie_name import objects_cookie_name
 from blueprints.objects_etag import objects_etag
 from blueprints.objects_hhhash import objects_hhhash
+from blueprints.objects_chat import objects_chat
 
 Flask_dir = os.environ['AIL_FLASK']
 
@@ -107,6 +108,7 @@ app.register_blueprint(objects_title, url_prefix=baseUrl)
 app.register_blueprint(objects_cookie_name, url_prefix=baseUrl)
 app.register_blueprint(objects_etag, url_prefix=baseUrl)
 app.register_blueprint(objects_hhhash, url_prefix=baseUrl)
+app.register_blueprint(objects_chat, url_prefix=baseUrl)
 
 # =========       =========#
 

var/www/blueprints/objects_chat.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+'''
+    Blueprint Flask: crawler splash endpoints: dashboard, onion crawler ...
+'''
+
+import os
+import sys
+import json
+
+from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, abort, send_file
+from flask_login import login_required, current_user
+
+# Import Role_Manager
+from Role_Manager import login_admin, login_analyst, login_read_only
+
+sys.path.append(os.environ['AIL_BIN'])
+##################################
+# Import Project packages
+##################################
+from lib import ail_core
+from lib.objects import abstract_subtype_object
+from lib.objects import ail_objects
+from lib.objects import Chats
+from packages import Date
+
+# ============ BLUEPRINT ============
+objects_chat = Blueprint('objects_chat', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/objects/chat'))
+
+# ============ VARIABLES ============
+bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info']
+
+def create_json_response(data, status_code):
+    return Response(json.dumps(data, indent=2, sort_keys=True), mimetype='application/json'), status_code
+
+# ============ FUNCTIONS ============
+
+# ============= ROUTES ==============
+
+
+@objects_chat.route("/objects/chat/messages", methods=['GET'])
+@login_required
+@login_read_only
+def objects_dashboard_chat():
+    chat = request.args.get('id')
+    subtype = request.args.get('subtype')
+    chat = Chats.Chat(chat, subtype)
+    if chat.exists():
+        messages = chat.get_messages()
+        meta = chat.get_meta({'icon'})
+        print(meta)
+        return render_template('ChatMessages.html', meta=meta, messages=messages, bootstrap_label=bootstrap_label)
+    else:
+        return abort(404)
+
+
+

var/www/blueprints/objects_subtypes.py

@@ -91,6 +91,12 @@ def subtypes_objects_dashboard(obj_type, f_request):
 
 # ============= ROUTES ==============
 
+@objects_subtypes.route("/objects/chats", methods=['GET'])
+@login_required
+@login_read_only
+def objects_dashboard_chat():
+    return subtypes_objects_dashboard('chat', request)
+
 @objects_subtypes.route("/objects/cryptocurrencies", methods=['GET'])
 @login_required
 @login_read_only

var/www/templates/objects/chat/ChatMessages.html

@@ -0,0 +1,190 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+  <title>Chat Messages - AIL</title>
+	<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
+
+  <!-- Core CSS -->
+	<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
+{#	<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet">#}
+
+  <!-- JS -->
+	<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
+{#	<script src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
+	<script src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>#}
+	<script src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
+    <script src="{{ url_for('static', filename='js/d3/sparklines.js')}}"></script>
+
+	<style>
+        .chat-message-left,
+        .chat-message-right {
+            display: flex;
+            flex-shrink: 0;
+        }
+        .chat-message-right {
+            flex-direction: row-reverse;
+            margin-left: auto
+        }
+	</style>
+</head>
+<body>
+
+	{% include 'nav_bar.html' %}
+
+	<div class="container-fluid">
+		<div class="row">
+
+			{% include 'sidebars/sidebar_objects.html' %}
+
+			<div class="col-12 col-lg-10" id="core_content">
+
+                <div class="card my-3">
+                    <div class="card-header" style="background-color:#d9edf7;font-size: 15px">
+                        <h3 class="text-secondary">{{ meta["id"] }} :</h3>
+                        <ul class="list-group mb-2">
+                            <li class="list-group-item py-0">
+                                <div class="row">
+                                    <div class="col-md-10">
+                                        <table class="table">
+                                            <thead>
+                                                <tr>
+                                                    <th>Object subtype</th>
+                                                    <th>First seen</th>
+                                                    <th>Last seen</th>
+                                                    <th>Nb seen</th>
+                                                </tr>
+                                            </thead>
+                                            <tbody>
+                                                <tr>
+                                                    <td>
+                                                        <svg height="26" width="26">
+                                                            <g class="nodes">
+                                                                <circle cx="13" cy="13" r="13" fill="{{ meta["icon"]["color"] }}"></circle>
+                                                                <text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ meta["icon"]["style"] }}" font-size="16px">{{ meta["icon"]["icon"] }}</text>
+                                                            </g>
+                                                        </svg>
+                                                        {{ meta["subtype"] }}
+                                                    </td>
+                                                    <td>{{ meta['first_seen'] }}</td>
+                                                    <td>{{ meta['last_seen'] }}</td>
+                                                    <td>{{ meta['nb_seen'] }}</td>
+                                                </tr>
+                                            </tbody>
+                                        </table>
+                                    </div>
+                                    <div class="col-md-1">
+                                        <div id="sparkline"></div>
+                                    </div>
+                                </div>
+                            </li>
+                            <li class="list-group-item py-0">
+                                <br>
+                                <div class="mb-3">
+                                    Tags:
+                                    {% for tag in meta['tags'] %}
+                                        <button class="btn btn-{{ bootstrap_label[loop.index0 % 5] }}" data-toggle="modal" data-target="#edit_tags_modal"
+                                        data-tagid="{{ tag }}" data-objtype="chat" data-objsubtype="{{ meta["subtype"] }}" data-objid="{{ meta["id"] }}">
+                                            {{ tag }}
+                                        </button>
+                                    {%  endfor %}
+                                    <button type="button" class="btn btn-light" data-toggle="modal" data-target="#add_tags_modal">
+                                        <i class="far fa-plus-square"></i>
+                                    </button>
+                                </div>
+                            </li>
+                        </ul>
+
+                        {% with obj_type='chat', obj_id=meta['id'], obj_subtype=meta['subtype'] %}
+                            {% include 'modals/investigations_register_obj.html' %}
+                        {% endwith %}
+                        <button type="button" class="btn btn-primary" data-toggle="modal" data-target="#investigations_register_obj_modal">
+                            <i class="fas fa-microscope"></i> Investigations
+                        </button>
+
+                    </div>
+                </div>
+
+                <div class="position-relative">
+                    <div class="chat-messages p-4">
+
+                        {% for date in messages %}
+                            <h2><span class="badge badge-secondary mb-2">{{ date }}</span></h2>
+                            {% for mess in messages[date] %}
+
+                                <div class="chat-message-left pb-0">
+                                    <div>
+                                        <img src="{{ url_for('static', filename='image/ail-icon.png') }}" class="rounded-circle mr-1" alt="{{ mess['username']['id'] }}" width="40" height="40">
+                                        <div class="text-muted small text-nowrap mt-2">{{ mess['hour'] }}</div>
+                                    </div>
+                                    <div class="flex-shrink-1 bg-light rounded py-2 px-3 ml-4 pb-4" style="overflow-x: auto">
+                                        <div class="font-weight-bold mb-1">{{ mess['username']['id'] }}</div>
+                                        {% if mess['reply_to'] %}
+                                            <div class="flex-shrink-1 border rounded py-2 px-3 ml-4 mb-3" style="overflow-x: auto">
+                                                <div class="font-weight-bold mb-1">{{ mess['reply_to']['username']['id'] }}</div>
+                                                <pre class="my-0">{{ mess['reply_to']['content'] }}</pre>
+                                                {% for tag in mess['reply_to']['tags'] %}
+                                                    <span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
+                                                {%  endfor %}
+                                                <div class="text-muted small text-nowrap">{{ mess['reply_to']['date'] }}</div>
+{#                                                <div class="">#}
+{#                                                    <a class="btn btn-light btn-sm text-secondary py-0" href="{{ url_for('correlation.show_correlation')}}?type={{ mess['reply_to']['type'] }}&subtype={{ mess['reply_to']['subtype'] }}&id={{ mess['reply_to']['id'] }}"><i class="fas fa-project-diagram"></i></a>#}
+{#                                                    <a class="btn btn-light btn-sm text-secondary py-0" href="{{ mess['reply_to']['link'] }}"><i class="fas fa-eye"></i></a>#}
+{#                                                </div>#}
+                                            </div>
+                                        {% endif %}
+                                        <pre class="my-0">{{ mess['content'] }}</pre>
+                                        {% for tag in mess['tags'] %}
+                                            <span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
+                                        {%  endfor %}
+                                        <div class="">
+                                            <a class="btn btn-light btn-sm text-secondary px-1" href="{{ url_for('correlation.show_correlation')}}?type={{ mess['type'] }}&subtype={{ mess['subtype'] }}&id={{ mess['id'] }}"><i class="fas fa-project-diagram"></i></a>
+                                            <a class="btn btn-light btn-sm text-secondary px-1" href="{{ mess['link'] }}"><i class="fas fa-eye"></i></a>
+                                        </div>
+                                    </div>
+                                </div>
+
+                            {% endfor %}
+                            <br>
+                        {% endfor %}
+
+                    </div>
+                </div>
+
+            </div>
+
+        </div>
+    </div>
+
+<script>
+    var chart = {};
+    $(document).ready(function(){
+        $("#page-Decoded").addClass("active");
+        $("#nav_chat").addClass("active");
+
+  });
+
+function toggle_sidebar(){
+	if($('#nav_menu').is(':visible')){
+		$('#nav_menu').hide();
+		$('#side_menu').removeClass('border-right')
+		$('#side_menu').removeClass('col-lg-2')
+		$('#core_content').removeClass('col-lg-10')
+	}else{
+		$('#nav_menu').show();
+		$('#side_menu').addClass('border-right')
+		$('#side_menu').addClass('col-lg-2')
+		$('#core_content').addClass('col-lg-10')
+	}
+}
+</script>
+
+</body>
+
+</html>