chg: [crawler] add function to delete schedules

HOWTO.md

@@ -29,6 +29,8 @@ Contributions are welcome! Fork the repository, experiment with the code, and su
 
 AIL supports crawling of websites and Tor hidden services. Ensure your Tor client's proxy configuration is correct, especially the SOCKS5 proxy settings.
 
+![Crawler](./doc/screenshots/ail-lacus.png?raw=true "AIL framework Crawler")
+
 ### Installation
 
 [Install Lacus](https://github.com/ail-project/lacus)

README.md

@@ -29,6 +29,8 @@ AIL framework - Framework for Analysis of Information Leaks
 
 AIL is a modular framework to analyse potential information leaks from unstructured data sources like pastes from Pastebin or similar services or unstructured data streams. AIL framework is flexible and can be extended to support other functionalities to mine or process sensitive information (e.g. data leak prevention).
 
+![Overview](./doc/screenshots/ail-overview.png?raw=true "AIL framework Overview")
+
 ![Dashboard](./doc/screenshots/dashboard0.png?raw=true "AIL framework dashboard")
 
 
@@ -55,6 +57,8 @@ Allow easy creation and customization by extending an abstract class.
 
 ## Features
 
+![Internal](./doc/screenshots/ail-internal.png?raw=true "AIL framework Internal")
+
 - Modular architecture to handle streams of unstructured or structured information
 - Default support for external ZMQ feeds, such as provided by CIRCL or other providers
 - Multiple Importers and feeds support

bin/LAUNCH.sh

@@ -275,8 +275,11 @@ function launching_scripts {
     screen -S "Script_AIL" -X screen -t "MISP_Thehive_Auto_Push" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./MISP_Thehive_Auto_Push.py; read x"
     sleep 0.1
 
+    # IMAGES
     screen -S "Script_AIL" -X screen -t "Exif" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Exif.py; read x"
     sleep 0.1
+    screen -S "Script_AIL" -X screen -t "OcrExtractor" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./OcrExtractor.py; read x"
+    sleep 0.1
 
     ##################################
     #       TRACKERS MODULES         #

bin/importer/FeederImporter.py

@@ -109,6 +109,9 @@ class FeederImporter(AbstractImporter):
                 gzip64_content = feeder.get_gzip64_content()
                 relay_message = f'{feeder_name} {gzip64_content}'
                 objs_messages.append({'obj': obj, 'message': relay_message})
+            elif obj.type == 'image':
+                date = feeder.get_date()
+                objs_messages.append({'obj': obj, 'message': f'{feeder_name} {date}'})
             else:  # Messages save on DB
                 if obj.exists() and obj.type != 'chat':
                     objs_messages.append({'obj': obj, 'message': feeder_name})

bin/importer/feeders/Default.py

@@ -41,6 +41,9 @@ class DefaultFeeder:
     def get_source(self):
         return self.json_data.get('source')
 
+    def get_date(self):
+        return datetime.date.today().strftime("%Y%m%d")
+
     def get_json_data(self):
         """
         Return the JSON data,

bin/importer/feeders/abstract_chats_feeder.py

@@ -92,6 +92,14 @@ class AbstractChatFeeder(DefaultFeeder, ABC):
     def get_reactions(self):
         return self.json_data['meta'].get('reactions', [])
 
+    def get_date(self):
+        if self.json_data['meta'].get('date'):
+            date = datetime.datetime.fromtimestamp( self.json_data['meta']['date']['timestamp'])
+            date = date.strftime('%Y%m%d')
+        else:
+            date = datetime.date.today().strftime("%Y%m%d")
+        return date
+
     def get_message_timestamp(self):
         if not self.json_data['meta'].get('date'):
             return None

bin/lib/ConfigLoader.py

@@ -9,7 +9,6 @@ The ``Domain``
 
 import os
 import sys
-import time
 import redis
 import configparser
 

bin/lib/Language.py

@@ -330,6 +330,11 @@ def get_obj_languages(obj_type, obj_subtype, obj_id):
 def get_obj_language_stats(obj_type, obj_subtype, obj_id):
     return r_lang.zrange(f'obj:langs:stat:{obj_type}:{obj_subtype}:{obj_id}', 0, -1, withscores=True)
 
+def get_obj_main_language(obj_type, obj_subtype, obj_id):
+    language = r_lang.zrevrange(f'obj:langs:stat:{obj_type}:{obj_subtype}:{obj_id}', 0, 0)
+    if language:
+        return language[0]
+
 # TODO ADD language to CHAT GLOBAL SET
 def add_obj_language(language, obj_type, obj_subtype, obj_id, objs_containers=set()):  # (s)
     if not obj_subtype:
@@ -434,8 +439,12 @@ def delete_obj_translation(obj_global_id, language, field=''):
 
 class LanguagesDetector:
 
-    def __init__(self, nb_langs=3, min_proportion=0.2, min_probability=0.7, min_len=0):
-        self.lt = LibreTranslateAPI(get_translator_instance())
+    def __init__(self, nb_langs=3, min_proportion=0.2, min_probability=-1, min_len=0):
+        lt_url = get_translator_instance()
+        if not lt_url:
+            self.lt = None
+        else:
+            self.lt = LibreTranslateAPI(get_translator_instance())
         try:
             self.lt.languages()
         except Exception:
@@ -461,9 +470,11 @@ class LanguagesDetector:
                 languages.append(lang.language)
         return languages
 
-    def detect_lexilang(self, content):  # TODO clean text ??? - TODO REMOVE SEPARATOR
+    def detect_lexilang(self, content):
         language, prob = lexilang_detect(content)
-        if prob > 0:
+        if prob > 0 and self.min_probability == -1:
+            return [language]
+        elif prob > 0.4:
             return [language]
         else:
             return []
@@ -486,10 +497,10 @@ class LanguagesDetector:
 
     def detect(self, content, force_gcld3=False):  # TODO detect length between 20-200 ????
         if not content:
-            return None
+            return []
         content = _clean_text_to_translate(content, html=True)
         if not content:
-            return None
+            return []
         # DEBUG
         # print('-------------------------------------------------------')
         # print(content)
@@ -506,6 +517,8 @@ class LanguagesDetector:
         # libretranslate
         # else:
         #     languages = self.detect_libretranslate(content)
+        if not languages:
+            languages = []
         return languages
 
 class LanguageTranslator:

bin/lib/Tag.py

@@ -32,6 +32,9 @@ config_loader = None
 
 # # # # UNSAFE TAGS # # # #
 
+# set of unsafe tags
+UNSAFE_TAGS = None
+
 def build_unsafe_tags():
     tags = set()
     # CE content
@@ -52,12 +55,12 @@ def is_tags_safe(ltags):
     :return: is a tag in the set unsafe
     :rtype: boolean
     """
-    return unsafe_tags.isdisjoint(ltags)
+    global UNSAFE_TAGS
+    if UNSAFE_TAGS is None:
+        UNSAFE_TAGS = build_unsafe_tags()
+    return UNSAFE_TAGS.isdisjoint(ltags)
 
 
-# set of unsafe tags
-unsafe_tags = build_unsafe_tags()
-
 # - - - UNSAFE TAGS - - - #
 
 # # TODO: verify tags + object_type
@@ -80,16 +83,15 @@ def get_obj_by_tag(key_tag):
 
 #### Taxonomies ####
 
-TAXONOMIES = {}
+TAXONOMIES = None
 def load_taxonomies():
     global TAXONOMIES
     manifest = os.path.join(os.environ['AIL_HOME'], 'files/misp-taxonomies/MANIFEST.json')
     TAXONOMIES = Taxonomies(manifest_path=manifest)
 
-
-load_taxonomies()
-
 def get_taxonomies():
+    if TAXONOMIES is None:
+        load_taxonomies()
     return TAXONOMIES.keys()
 
 # TODO rename me to get enabled_taxonomies
@@ -111,12 +113,18 @@ def disable_taxonomy(taxonomy):
     r_tags.srem('taxonomies:enabled', taxonomy)
 
 def exists_taxonomy(taxonomy):
+    if TAXONOMIES is None:
+        load_taxonomies()
     return TAXONOMIES.get(taxonomy) is not None
 
 def get_taxonomy_description(taxonomy):
+    if TAXONOMIES is None:
+        load_taxonomies()
     return TAXONOMIES.get(taxonomy).description
 
 def get_taxonomy_name(taxonomy):
+    if TAXONOMIES is None:
+        load_taxonomies()
     return TAXONOMIES.get(taxonomy).name
 
 def get_taxonomy_predicates(taxonomy):
@@ -133,12 +141,18 @@ def get_taxonomy_predicates(taxonomy):
     return meta
 
 def get_taxonomy_refs(taxonomy):
+    if TAXONOMIES is None:
+        load_taxonomies()
     return TAXONOMIES.get(taxonomy).refs
 
 def get_taxonomy_version(taxonomy):
+    if TAXONOMIES is None:
+        load_taxonomies()
     return TAXONOMIES.get(taxonomy).version
 
 def get_taxonomy_tags(taxonomy, enabled=False):
+    if TAXONOMIES is None:
+        load_taxonomies()
     taxonomy_obj = TAXONOMIES.get(taxonomy)
     tags = []
     for p, content in taxonomy_obj.items():
@@ -165,6 +179,8 @@ def get_taxonomy_meta(taxonomy_name, enabled=False, enabled_tags=False, nb_activ
     meta = {}
     if not exists_taxonomy(taxonomy_name):
         return meta
+    if TAXONOMIES is None:
+        load_taxonomies()
     taxonomy = TAXONOMIES.get(taxonomy_name)
     meta['description'] = taxonomy.description
     meta['name'] = taxonomy.name
@@ -241,6 +257,8 @@ def api_update_taxonomy_tag_enabled(data):
     if not exists_taxonomy(taxonomy):
         return {'error': f'taxonomy {taxonomy} not found'}, 404
     tags = data.get('tags', [])
+    if TAXONOMIES is None:
+        load_taxonomies()
     taxonomy_tags = set(TAXONOMIES.get(taxonomy).machinetags())
     for tag in tags:
         if tag not in taxonomy_tags:
@@ -249,6 +267,8 @@ def api_update_taxonomy_tag_enabled(data):
 
 def enable_taxonomy_tags(taxonomy):
     enable_taxonomy(taxonomy)
+    if TAXONOMIES is None:
+        load_taxonomies()
     for tag in TAXONOMIES.get(taxonomy).machinetags():
         add_taxonomy_tag_enabled(taxonomy, tag)
 
@@ -279,9 +299,8 @@ def api_disable_taxonomy_tags(data):
 #
 
 # TODO Synonyms
-
-GALAXIES = {}
-CLUSTERS = {}
+GALAXIES = None
+CLUSTERS = None
 def load_galaxies():
     global GALAXIES
     galaxies = []
@@ -298,11 +317,10 @@ def load_galaxies():
             clusters.append(json.load(f))
     CLUSTERS = Clusters(clusters)
 
-
-# LOAD GALAXY + CLUSTERS
-load_galaxies()
-
 def get_galaxies():
+    if GALAXIES is None:
+        # LOAD GALAXY + CLUSTERS
+        load_galaxies()
     return GALAXIES.keys()
 
 # TODO RENAME ME
@@ -310,9 +328,15 @@ def get_active_galaxies():
     return r_tags.smembers('galaxies:enabled')
 
 def get_galaxy(galaxy_name):
+    if GALAXIES is None:
+        # LOAD GALAXY + CLUSTERS
+        load_galaxies()
     return GALAXIES.get(galaxy_name)
 
 def exists_galaxy(galaxy):
+    if CLUSTERS is None:
+        # LOAD GALAXY + CLUSTERS
+        load_galaxies()
     return CLUSTERS.get(galaxy) is not None
 
 def is_galaxy_enabled(galaxy):
@@ -369,9 +393,15 @@ def get_galaxy_tag_meta(galaxy_type, tag):
 
 
 def get_clusters():
+    if CLUSTERS is None:
+        # LOAD GALAXY + CLUSTERS
+        load_galaxies()
     return CLUSTERS.keys()
 
 def get_cluster(cluster_type):
+    if CLUSTERS is None:
+        # LOAD GALAXY + CLUSTERS
+        load_galaxies()
     return CLUSTERS.get(cluster_type)
 
 def get_galaxy_tags(galaxy_type):

bin/lib/Tracker.py

@@ -12,7 +12,6 @@ import yara
 import datetime
 import base64
 
-from ail_typo_squatting import runAll
 import math
 
 from collections import defaultdict
@@ -38,24 +37,22 @@ logger = logging.getLogger()
 
 config_loader = ConfigLoader.ConfigLoader()
 r_cache = config_loader.get_redis_conn("Redis_Cache")
-
 r_tracker = config_loader.get_db_conn("Kvrocks_Trackers")
-
-items_dir = config_loader.get_config_str("Directories", "pastes")
-if items_dir[-1] == '/':
-    items_dir = items_dir[:-1]
 config_loader = None
 
-email_regex = r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}'
-email_regex = re.compile(email_regex)
-
-special_characters = set('[<>~!?@#$%^&*|()_-+={}":;,.\'\n\r\t]/\\')
-special_characters.add('\\s')
-
 # NLTK tokenizer
-tokenizer = RegexpTokenizer('[\&\~\:\;\,\.\(\)\{\}\|\[\]\\\\/\-/\=\'\"\%\$\?\@\+\#\_\^\<\>\!\*\n\r\t\s]+',
+TOKENIZER = None
+
+def init_tokenizer():
+    global TOKENIZER
+    TOKENIZER = RegexpTokenizer('[\&\~\:\;\,\.\(\)\{\}\|\[\]\\\\/\-/\=\'\"\%\$\?\@\+\#\_\^\<\>\!\*\n\r\t\s]+',
                             gaps=True, discard_empty=True)
 
+def get_special_characters():
+    special_characters = set('[<>~!?@#$%^&*|()_-+={}":;,.\'\n\r\t]/\\')
+    special_characters.add('\\s')
+    return special_characters
+
 ###############
 #### UTILS ####
 def is_valid_uuid_v4(curr_uuid):
@@ -76,6 +73,8 @@ def is_valid_regex(tracker_regex):
         return False
 
 def is_valid_mail(email):
+    email_regex = r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}'
+    email_regex = re.compile(email_regex)
     result = email_regex.match(email)
     if result:
         return True
@@ -385,7 +384,7 @@ class Tracker:
             r_tracker.srem(f'obj:tracker:{obj_type}:{subtype}:{obj_id}:{self.uuid}', date)
 
         r_tracker.srem(f'obj:trackers:{obj_type}:{subtype}:{obj_id}', self.uuid)
-        r_tracker.srem(f'tracker:objs:{self.uuid}', f'{obj_type}:{subtype}:{obj_id}')
+        r_tracker.srem(f'tracker:objs:{self.uuid}:{obj_type}', f'{subtype}:{obj_id}')
         self.update_daterange()
 
     # TODO escape custom tags
@@ -400,6 +399,9 @@ class Tracker:
             tracker_type = 'yara'
 
         elif tracker_type == 'typosquatting':
+
+            from ail_typo_squatting import runAll
+
             domain = to_track.split(" ")[0]
             typo_generation = runAll(domain=domain, limit=math.inf, formatoutput="text", pathOutput="-", verbose=False) # TODO REPLACE LIMIT BY -1
             for typo in typo_generation:
@@ -857,7 +859,7 @@ def api_validate_tracker_to_add(to_track, tracker_type, nb_words=1):
         # force lowercase
         to_track = to_track.lower()
         word_set = set(to_track)
-        set_inter = word_set.intersection(special_characters)
+        set_inter = word_set.intersection(get_special_characters())
         if set_inter:
             return {"status": "error",
                     "reason": f'special character(s) not allowed: {set_inter}',
@@ -1055,6 +1057,37 @@ def api_delete_tracker(data, user_id):
     tracker = Tracker(tracker_uuid)
     return tracker.delete(), 200
 
+def api_tracker_add_object(data, user_id):
+    tracker_uuid = data.get('uuid')
+    res = api_check_tracker_acl(tracker_uuid, user_id)
+    if res:
+        return res
+    tracker = Tracker(tracker_uuid)
+    object_gid = data.get('gid')
+    date = data.get('date')
+    if date:
+        if not Date.validate_str_date(date):
+            date = None
+    try:
+        obj_type, subtype, obj_id = object_gid.split(':', 2)
+    except (AttributeError, IndexError):
+        return {"status": "error", "reason": "Invalid Object"}, 400
+    return tracker.add(obj_type, subtype, obj_id, date=date), 200
+
+def api_tracker_remove_object(data, user_id):
+    tracker_uuid = data.get('uuid')
+    res = api_check_tracker_acl(tracker_uuid, user_id)
+    if res:
+        return res
+
+    tracker = Tracker(tracker_uuid)
+    object_gid = data.get('gid')
+    try:
+        obj_type, subtype, obj_id = object_gid.split(':', 2)
+    except (AttributeError, IndexError):
+        return {"status": "error", "reason": "Invalid Object"}, 400
+    return tracker.remove(obj_type, subtype, obj_id), 200
+
 ## -- CREATE TRACKER -- ##
 
 ####################
@@ -1082,7 +1115,9 @@ def get_text_word_frequency(content, filtering=True):
     words_dict = defaultdict(int)
 
     if filtering:
-        blob = TextBlob(content, tokenizer=tokenizer)
+        if TOKENIZER is None:
+            init_tokenizer()
+        blob = TextBlob(content, tokenizer=TOKENIZER)
     else:
         blob = TextBlob(content)
     for word in blob.tokens:
@@ -1769,9 +1804,9 @@ def _fix_db_custom_tags():
 #### -- ####
 
 
-if __name__ == '__main__':
+# if __name__ == '__main__':
 
-    _fix_db_custom_tags()
+    # _fix_db_custom_tags()
     # fix_all_tracker_uuid_list()
     # res = get_all_tracker_uuid()
     # print(len(res))

bin/lib/ail_core.py

@@ -18,14 +18,14 @@ config_loader = None
 
 AIL_OBJECTS = sorted({'chat', 'chat-subchannel', 'chat-thread', 'cookie-name', 'cve', 'cryptocurrency', 'decoded',
                       'domain', 'etag', 'favicon', 'file-name', 'hhhash',
-                      'item', 'image', 'message', 'pgp', 'screenshot', 'title', 'user-account', 'username'})
+                      'item', 'image', 'message', 'ocr', 'pgp', 'screenshot', 'title', 'user-account', 'username'})
 
 AIL_OBJECTS_WITH_SUBTYPES = {'chat', 'chat-subchannel', 'cryptocurrency', 'pgp', 'username', 'user-account'}
 
 # TODO by object TYPE ????
 AIL_OBJECTS_CORRELATIONS_DEFAULT = sorted({'chat', 'chat-subchannel', 'chat-thread', 'cve', 'cryptocurrency', 'decoded',
                                            'domain', 'favicon', 'file-name',
-                                           'item', 'image', 'message', 'pgp', 'screenshot', 'title', 'user-account', 'username'})
+                                           'item', 'image', 'message', 'ocr', 'pgp', 'screenshot', 'title', 'user-account', 'username'})
 
 def get_ail_uuid():
     ail_uuid = r_serv_db.get('ail:uuid')
@@ -68,7 +68,7 @@ def get_object_all_subtypes(obj_type):  # TODO Dynamic subtype
     if obj_type == 'chat-thread':
         return r_object.smembers(f'all_chat-thread:subtypes')
     if obj_type == 'cryptocurrency':
-        return ['bitcoin', 'bitcoin-cash', 'dash', 'ethereum', 'litecoin', 'monero', 'zcash']
+        return ['bitcoin', 'bitcoin-cash', 'dash', 'ethereum', 'litecoin', 'monero', 'tron', 'zcash']
     if obj_type == 'pgp':
         return ['key', 'mail', 'name']
     if obj_type == 'username':
@@ -81,10 +81,10 @@ def get_default_correlation_objects():
     return AIL_OBJECTS_CORRELATIONS_DEFAULT
 
 def get_obj_queued():
-    return ['item', 'image']
+    return ['item', 'image', 'message', 'ocr']
 
 def get_objects_tracked():
-    return ['decoded', 'item', 'pgp', 'message', 'title']
+    return ['decoded', 'item', 'pgp', 'message', 'ocr', 'title']
 
 def get_objects_retro_hunted():
     return ['decoded', 'item', 'message']
@@ -105,7 +105,7 @@ def unpack_obj_global_id(global_id, r_type='tuple'):
         obj = global_id.split(':', 2)
         return {'type': obj[0], 'subtype': obj[1], 'id': obj[2]}
     else:  # tuple(type, subtype, id)
-        return global_id.split(':', 2)
+        return global_id.split(':', 2)    # TODO REPLACE get_obj_type_subtype_id_from_global_id(global_id)
 
 def unpack_objs_global_id(objs_global_id, r_type='tuple'):
     objs = []

bin/lib/btc_ail.py

@@ -8,7 +8,6 @@ import sys
 import requests
 
 sys.path.append(os.environ['AIL_BIN'])
-from lib.objects.CryptoCurrencies import CryptoCurrency
 
 logger = logging.getLogger()
 
@@ -53,9 +52,11 @@ def get_bitcoin_info(bitcoin_address, nb_transaction=50):
 
 # filter btc seen in ail
 def filter_btc_seen(btc_addr_set):
+    from lib.objects import CryptoCurrencies
+
     list_seen_btc = []
     for btc_addr in btc_addr_set:
-        cryptocurrency = CryptoCurrency(btc_addr, 'bitcoin')
+        cryptocurrency = CryptoCurrencies.CryptoCurrency(btc_addr, 'bitcoin')
         if cryptocurrency.exists():
             list_seen_btc.append(btc_addr)
     return list_seen_btc

bin/lib/chats_viewer.py

@@ -11,6 +11,7 @@ import sys
 import time
 import uuid
 
+from datetime import datetime
 
 sys.path.append(os.environ['AIL_BIN'])
 ##################################
@@ -287,6 +288,10 @@ def get_obj_chat(chat_type, chat_subtype, chat_id):
     elif chat_type == 'chat-thread':
         return ChatThreads.ChatThread(chat_id, chat_subtype)
 
+def get_obj_chat_from_global_id(chat_gid):
+    chat_type, chat_subtype, chat_id = chat_gid.split(':', 2)
+    return get_obj_chat(chat_type, chat_subtype, chat_id)
+
 def get_obj_chat_meta(obj_chat, new_options=set()):
     options = {}
     if obj_chat.type == 'chat':
@@ -321,7 +326,32 @@ def get_threads_metas(threads):
 def get_username_meta_from_global_id(username_global_id):
     _, instance_uuid, username_id = username_global_id.split(':', 2)
     username = Usernames.Username(username_id, instance_uuid)
-    return username.get_meta()
+    return username.get_meta(options={'icon'})
+
+###############################################################################
+# TODO Pagination
+def list_messages_to_dict(l_messages_id, translation_target=None):
+    options = {'content', 'files-names', 'images', 'language', 'link', 'parent', 'parent_meta', 'reactions', 'thread', 'translation', 'user-account'}
+    meta = {}
+    curr_date = None
+    for mess_id in l_messages_id:
+        message = Messages.Message(mess_id[1:])
+        timestamp = message.get_timestamp()
+        date_day = message.get_date()
+        date_day = f'{date_day[0:4]}/{date_day[4:6]}/{date_day[6:8]}'
+        if date_day != curr_date:
+            meta[date_day] = []
+            curr_date = date_day
+        meta_mess = message.get_meta(options=options, timestamp=timestamp, translation_target=translation_target)
+        meta[date_day].append(meta_mess)
+
+        # if mess_dict.get('tags'):
+        #     for tag in mess_dict['tags']:
+        #         if tag not in tags:
+        #             tags[tag] = 0
+        #         tags[tag] += 1
+    # return messages, pagination, tags
+    return meta
 
 # TODO Filter
 ## Instance type
@@ -349,7 +379,7 @@ def get_messages_iterator(filters={}):
             # threads
             for threads in chat.get_threads():
                 thread = ChatThreads.ChatThread(threads['id'], instance_uuid)
-                _, _ = thread._get_messages(nb=-1)
+                messages, _ = thread._get_messages(nb=-1)
                 for mess in messages:
                     message_id, _, message_id = mess[0].split(':', )
                     yield Messages.Message(message_id)
@@ -379,6 +409,94 @@ def get_nb_messages_iterator(filters={}):
             nb_messages += chat.get_nb_messages()
     return nb_messages
 
+def get_user_account_chats_meta(user_id, chats, subchannels):
+    meta = []
+    for chat_g_id in chats:
+        c_subtype, c_id = chat_g_id.split(':', 1)
+        chat = Chats.Chat(c_id, c_subtype)
+        chat_meta = chat.get_meta(options={'icon', 'info', 'nb_participants', 'tags_safe', 'username'})
+        if chat_meta['username']:
+            chat_meta['username'] = get_username_meta_from_global_id(chat_meta['username'])
+        chat_meta['nb_messages'] = len(chat.get_user_messages(user_id))
+        chat_meta['subchannels'] = []
+        for subchannel_gid in chat.get_subchannels():
+            if subchannel_gid[16:] in subchannels:
+                _, s_subtype, s_id = subchannel_gid.split(':', 2)
+                subchannel = ChatSubChannels.ChatSubChannel(s_id, s_subtype)
+                subchannel_meta = subchannel.get_meta(options={'created_at'})
+                subchannel_meta['nb_messages'] = len(subchannel.get_user_messages(user_id))
+                chat_meta['subchannels'].append(subchannel_meta)
+        meta.append(chat_meta)
+    return meta
+
+def get_user_account_chat_message(user_id, subtype, chat_id): # TODO subchannel + threads ...
+    meta = {}
+    chat = Chats.Chat(chat_id, subtype)
+    chat_meta = chat.get_meta(options={'icon', 'info', 'nb_participants', 'tags_safe', 'username'})
+    if chat_meta['username']:
+        chat_meta['username'] = get_username_meta_from_global_id(chat_meta['username'])
+
+    meta['messages'] = list_messages_to_dict(chat.get_user_messages(user_id), translation_target=None)
+    return meta
+
+def get_user_account_nb_all_week_messages(user_id, chats, subchannels):
+    week = {}
+    # Init
+    for day in ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']:
+        week[day] = {}
+        for i in range(24):
+            week[day][i] = 0
+
+    # chats
+    for chat_g_id in chats:
+        c_subtype, c_id = chat_g_id.split(':', 1)
+        chat = Chats.Chat(c_id, c_subtype)
+        for message in chat.get_user_messages(user_id):
+            timestamp = message.split('/', 2)[1]
+            timestamp = datetime.utcfromtimestamp(float(timestamp))
+            date_name = timestamp.strftime('%a')
+            week[date_name][timestamp.hour] += 1
+
+    stats = []
+    nb_day = 0
+    for day in week:
+        for hour in week[day]:
+            stats.append({'date': day, 'day': nb_day, 'hour': hour, 'count': week[day][hour]})
+        nb_day += 1
+    return stats
+
+def _get_chat_card_meta_options():
+    return {'created_at', 'icon', 'info', 'nb_participants', 'origin_link', 'subchannels', 'tags_safe', 'threads', 'translation', 'username'}
+
+def _get_message_bloc_meta_options():
+    return {'chat', 'content', 'files-names', 'icon', 'images', 'language', 'link', 'parent', 'parent_meta', 'reactions','thread', 'translation', 'user-account'}
+
+def get_message_report(l_mess): # TODO Force language + translation
+    translation_target = 'en'
+    chats = {}
+    messages = []
+    mess_options = _get_message_bloc_meta_options()
+
+    l_mess = sorted(l_mess, key=lambda x: x[2])
+
+    for m in l_mess:
+        message = Messages.Message(m[2])
+        meta = message.get_meta(options=mess_options, translation_target=translation_target)
+        if meta['chat'] not in chats:
+            chat = Chats.Chat(meta['chat'], message.get_chat_instance())
+            meta_chat = chat.get_meta(options=_get_chat_card_meta_options(), translation_target=translation_target)
+            if meta_chat['username']:
+                meta_chat['username'] = get_username_meta_from_global_id(meta_chat['username'])
+            chats[chat.id] = meta_chat
+
+            # stats
+            chats[chat.id]['t_messages'] = 1
+        else:
+            chats[meta['chat']]['t_messages'] += 1
+
+        messages.append(meta)
+
+    return chats, messages
 
 #### FIX ####
 
@@ -403,12 +521,12 @@ def api_get_chat_service_instance(chat_instance_uuid):
         return {"status": "error", "reason": "Unknown uuid"}, 404
     return chat_instance.get_meta({'chats'}), 200
 
-def api_get_chat(chat_id, chat_instance_uuid, translation_target=None, nb=-1, page=-1):
+def api_get_chat(chat_id, chat_instance_uuid, translation_target=None, nb=-1, page=-1, messages=True):
     chat = Chats.Chat(chat_id, chat_instance_uuid)
     if not chat.exists():
         return {"status": "error", "reason": "Unknown chat"}, 404
     # print(chat.get_obj_language_stats())
-    meta = chat.get_meta({'created_at', 'icon', 'info', 'nb_participants', 'subchannels', 'threads', 'translation', 'username'}, translation_target=translation_target)
+    meta = chat.get_meta({'created_at', 'icon', 'info', 'nb_participants', 'subchannels', 'tags_safe', 'threads', 'translation', 'username'}, translation_target=translation_target)
     if meta['username']:
         meta['username'] = get_username_meta_from_global_id(meta['username'])
     if meta['subchannels']:
@@ -416,7 +534,8 @@ def api_get_chat(chat_id, chat_instance_uuid, translation_target=None, nb=-1, pa
     else:
         if translation_target not in Language.get_translation_languages():
             translation_target = None
-        meta['messages'], meta['pagination'], meta['tags_messages'] = chat.get_messages(translation_target=translation_target, nb=nb, page=page)
+        if messages:
+            meta['messages'], meta['pagination'], meta['tags_messages'] = chat.get_messages(translation_target=translation_target, nb=nb, page=page)
     return meta, 200
 
 def api_get_nb_message_by_week(chat_type, chat_instance_uuid, chat_id):
@@ -514,8 +633,29 @@ def api_get_user_account(user_id, instance_uuid, translation_target=None):
     if not user_account.exists():
         return {"status": "error", "reason": "Unknown user-account"}, 404
     meta = user_account.get_meta({'chats', 'icon', 'info', 'subchannels', 'threads', 'translation', 'username', 'username_meta'}, translation_target=translation_target)
+    if meta['chats']:
+        meta['chats'] = get_user_account_chats_meta(user_id, meta['chats'], meta['subchannels'])
     return meta, 200
 
+def api_get_user_account_chat_messages(user_id, instance_uuid, chat_id, translation_target=None):
+    user_account = UsersAccount.UserAccount(user_id, instance_uuid)
+    if not user_account.exists():
+        return {"status": "error", "reason": "Unknown user-account"}, 404
+    meta = get_user_account_chat_message(user_id, instance_uuid, chat_id)
+    meta['user-account'] = user_account.get_meta({'icon', 'info', 'translation', 'username', 'username_meta'}, translation_target=translation_target)
+    resp = api_get_chat(chat_id, instance_uuid, translation_target=translation_target, messages=False)
+    if resp[1] != 200:
+        return resp
+    meta['chat'] = resp[0]
+    return meta, 200
+
+def api_get_user_account_nb_all_week_messages(user_id, instance_uuid):
+    user_account = UsersAccount.UserAccount(user_id, instance_uuid)
+    if not user_account.exists():
+        return {"status": "error", "reason": "Unknown user-account"}, 404
+    week = get_user_account_nb_all_week_messages(user_account.id, user_account.get_chats(), user_account.get_chat_subchannels())
+    return week, 200
+
 def api_chat_messages(subtype, chat_id):
     chat = Chats.Chat(chat_id, subtype)
     if not chat.exists():

bin/lib/correlations_engine.py

@@ -41,25 +41,26 @@ config_loader = None
 ##################################
 
 CORRELATION_TYPES_BY_OBJ = {
-    "chat": ["chat-subchannel", "chat-thread", "image", "user-account"],  # message or direct correlation like cve, bitcoin, ... ???
-    "chat-subchannel": ["chat", "chat-thread", "image", "message", "user-account"],
-    "chat-thread": ["chat", "chat-subchannel", "image", "message", "user-account"], # TODO user account
+    "chat": ["chat-subchannel", "chat-thread", "image", "message", "ocr", "user-account"],  # message or direct correlation like cve, bitcoin, ... ???
+    "chat-subchannel": ["chat", "chat-thread", "image", "message", "ocr", "user-account"],
+    "chat-thread": ["chat", "chat-subchannel", "image", "message", "ocr", "user-account"], # TODO user account
     "cookie-name": ["domain"],
-    "cryptocurrency": ["domain", "item", "message"],
-    "cve": ["domain", "item", "message"],
-    "decoded": ["domain", "item", "message"],
+    "cryptocurrency": ["domain", "item", "message", "ocr"],
+    "cve": ["domain", "item", "message", "ocr"],
+    "decoded": ["domain", "item", "message", "ocr"],
     "domain": ["cve", "cookie-name", "cryptocurrency", "decoded", "etag", "favicon", "hhhash", "item", "pgp", "title", "screenshot", "username"],
     "etag": ["domain"],
     "favicon": ["domain", "item"],  # TODO Decoded
     "file-name": ["chat", "message"],
     "hhhash": ["domain"],
-    "image": ["chat", "message", "user-account"],
+    "image": ["chat", "chat-subchannel", "chat-thread", "message", "ocr", "user-account"],  # TODO subchannel + threads ????
     "item": ["cve", "cryptocurrency", "decoded", "domain", "favicon", "pgp", "screenshot", "title", "username"],  # chat ???
-    "message": ["chat", "chat-subchannel", "chat-thread", "cve", "cryptocurrency", "decoded", "file-name", "image", "pgp", "user-account"],  # chat ??
-    "pgp": ["domain", "item", "message"],
+    "message": ["chat", "chat-subchannel", "chat-thread", "cve", "cryptocurrency", "decoded", "file-name", "image", "ocr", "pgp", "user-account"],
+    "ocr": ["chat", "chat-subchannel", "chat-thread", "cve", "cryptocurrency", "decoded", "image", "message", "pgp", "user-account"],
+    "pgp": ["domain", "item", "message", "ocr"],
     "screenshot": ["domain", "item"],
     "title": ["domain", "item"],
-    "user-account": ["chat", "chat-subchannel", "chat-thread", "image", "message", "username"],
+    "user-account": ["chat", "chat-subchannel", "chat-thread", "image", "message", "ocr", "username"],
     "username": ["domain", "item", "message", "user-account"],
 }
 

bin/lib/crawlers.py

@@ -1286,6 +1286,11 @@ def create_schedule(frequency, user, url, depth=1, har=True, screenshot=True, he
     schedule.create(frequency, user, url, depth=depth, har=har, screenshot=screenshot, header=header, cookiejar=cookiejar, proxy=proxy, user_agent=user_agent, tags=tags)
     return schedule_uuid
 
+def _delete_schedules():
+    for schedule_uuid in get_schedulers_uuid():
+        schedule = CrawlerSchedule(schedule_uuid)
+        schedule.delete()
+
 # TODO sanityze UUID
 def api_delete_schedule(data):
     schedule_uuid = data.get('uuid')
@@ -1673,7 +1678,6 @@ def create_task(url, depth=1, har=True, screenshot=True, header=None, cookiejar=
                             external=external)
     return task_uuid
 
-
 ## -- CRAWLER TASK -- ##
 
 #### CRAWLER TASK API ####

bin/lib/module_extractor.py

@@ -40,6 +40,11 @@ r_key = regex_helper.generate_redis_cache_key('extractor')
 
 # TODO UI Link
 
+CORRELATION_TO_EXTRACT = {
+    'item': ['cve', 'cryptocurrency', 'title', 'username'],
+    'message': ['cve', 'cryptocurrency', 'username']
+}
+
 MODULES = {
     'infoleak:automatic-detection="credit-card"': CreditCards(queue=False),
     'infoleak:automatic-detection="iban"': Iban(queue=False),
@@ -57,9 +62,27 @@ tools = Tools(queue=False)
 for tool_name in tools.get_tools():
     MODULES[f'infoleak:automatic-detection="{tool_name}-tool"'] = tools
 
-def get_correl_match(extract_type, obj_id, content):
+def merge_overlap(extracted):
+    merged = []
+    curr_start, curr_end, curr_string_match, curr_obj_ref = extracted[0]
+    curr_obj_ref = [(curr_obj_ref, curr_string_match)]
+
+    for start, end, mstring, ref in extracted[1:]:
+        # overlap
+        if start <= curr_end:
+            curr_string_match += mstring[curr_end - start:]
+            curr_end = max(curr_end, end)
+            curr_obj_ref.append((ref, mstring))
+        else:
+            merged.append((curr_start, curr_end, curr_string_match, curr_obj_ref))
+            curr_start, curr_end, curr_string_match, curr_obj_ref = start, end, mstring, [(ref, mstring)]
+
+    merged.append((curr_start, curr_end, curr_string_match, curr_obj_ref))
+    return merged
+
+def get_correl_match(extract_type, obj, content):
     extracted = []
-    correl = correlations_engine.get_correlation_by_correl_type('item', '', obj_id, extract_type)
+    correl = correlations_engine.get_correlation_by_correl_type(obj.type, obj.get_subtype(r_str=True), obj.id, extract_type)
     to_extract = []
     map_subtype = {}
     map_value_id = {}
@@ -75,18 +98,20 @@ def get_correl_match(extract_type, obj_id, content):
             sha256_val = sha256(value.encode()).hexdigest()
         map_value_id[sha256_val] = value
     if to_extract:
-        objs = regex_helper.regex_finditer(r_key, '|'.join(to_extract), obj_id, content)
-        for obj in objs:
-            if map_subtype.get(obj[2]):
-                subtype = map_subtype[obj[2]]
+        objs = regex_helper.regex_finditer(r_key, '|'.join(to_extract), obj.get_global_id(), content)
+        if extract_type == 'title' and objs:
+            objs = [objs[0]]
+        for ob in objs:
+            if map_subtype.get(ob[2]):
+                subtype = map_subtype[ob[2]]
             else:
                 subtype = ''
-                sha256_val = sha256(obj[2].encode()).hexdigest()
+                sha256_val = sha256(ob[2].encode()).hexdigest()
             value_id = map_value_id.get(sha256_val)
             if not value_id:
                 logger.critical(f'Error module extractor: {sha256_val}\n{extract_type}\n{subtype}\n{value_id}\n{map_value_id}\n{objs}')
                 value_id = 'ERROR'
-            extracted.append([obj[0], obj[1], obj[2], f'{extract_type}:{subtype}:{value_id}'])
+            extracted.append([ob[0], ob[1], ob[2], f'{extract_type}:{subtype}:{value_id}'])
     return extracted
 
 def _get_yara_match(data):
@@ -100,7 +125,7 @@ def _get_yara_match(data):
     return yara.CALLBACK_CONTINUE
 
 def _get_word_regex(word):
-    return '(?:^|(?<=[\&\~\:\;\,\.\(\)\{\}\|\[\]\\\\/\-/\=\'\"\%\$\?\@\+\#\_\^\<\>\!\*\n\r\t\s]))' + word + '(?:$|(?=[\&\~\:\;\,\.\(\)\{\}\|\[\]\\\\/\-/\=\'\"\%\$\?\@\+\#\_\^\<\>\!\*\n\r\t\s]))'
+    return '(?i)(?:^|(?<=[\&\~\:\;\,\.\(\)\{\}\|\[\]\\\\/\-/\=\'\\"\%\$\?\@\+\#\_\^\<\>\!\*\n\r\t\s]))' + word + '(?:$|(?=[\&\~\:\;\,\.\(\)\{\}\|\[\]\\\\/\-/\=\'\\"\%\$\?\@\+\#\_\^\<\>\!\*\n\r\t\s]))'
 
 def convert_byte_offset_to_string(b_content, offset):
     byte_chunk = b_content[:offset + 1]
@@ -115,17 +140,18 @@ def convert_byte_offset_to_string(b_content, offset):
 
 # TODO RETRO HUNTS
 # TODO TRACKER TYPE IN UI
-def get_tracker_match(obj_id, content):
+def get_tracker_match(obj, content):
     extracted = []
     extracted_yara = []
-    trackers = Tracker.get_obj_trackers('item', '', obj_id)
+    obj_gid = obj.get_global_id()
+    trackers = Tracker.get_obj_trackers(obj.type, obj.get_subtype(r_str=True), obj.id)
     for tracker_uuid in trackers:
         tracker = Tracker.Tracker(tracker_uuid)
         tracker_type = tracker.get_type()
         # print(tracker_type)
         tracked = tracker.get_tracked()
         if tracker_type == 'regex':  # TODO Improve word detection -> word delimiter
-            regex_match = regex_helper.regex_finditer(r_key, tracked, obj_id, content)
+            regex_match = regex_helper.regex_finditer(r_key, tracked, obj_gid, content)
             for match in regex_match:
                 extracted.append([int(match[0]), int(match[1]), match[2], f'tracker:{tracker.uuid}'])
         elif tracker_type == 'yara':
@@ -147,13 +173,13 @@ def get_tracker_match(obj_id, content):
                 words = [tracked]
             for word in words:
                 regex = _get_word_regex(word)
-                regex_match = regex_helper.regex_finditer(r_key, regex, obj_id, content)
+                regex_match = regex_helper.regex_finditer(r_key, regex, obj_gid, content)
                 # print(regex_match)
                 for match in regex_match:
                     extracted.append([int(match[0]), int(match[1]), match[2], f'tracker:{tracker.uuid}'])
 
     # Retro Hunt
-    retro_hunts = Tracker.get_obj_retro_hunts('item', '', obj_id)
+    retro_hunts = Tracker.get_obj_retro_hunts(obj.type, obj.get_subtype(r_str=True), obj.id)
     for retro_uuid in retro_hunts:
         retro_hunt = Tracker.RetroHunt(retro_uuid)
         rule = retro_hunt.get_rule(r_compile=True)
@@ -182,104 +208,101 @@ def get_tracker_match(obj_id, content):
 # Type:subtype:id
 # tag:iban
 # tracker:uuid
-
-def extract(obj_id, content=None):
-    item = Item(obj_id)
-    if not item.exists():
+# def extract(obj_id, content=None):
+def extract(obj_type, subtype, obj_id, content=None):
+    obj = ail_objects.get_object(obj_type, subtype, obj_id)
+    if not obj.exists():
         return []
+    obj_gid = obj.get_global_id()
 
     # CHECK CACHE
-    cached = r_cache.get(f'extractor:cache:{obj_id}')
+    cached = r_cache.get(f'extractor:cache:{obj_gid}')
     # cached = None
     if cached:
-        r_cache.expire(f'extractor:cache:{obj_id}', 300)
+        r_cache.expire(f'extractor:cache:{obj_gid}', 300)
         return json.loads(cached)
 
     if not content:
-        content = item.get_content()
+        content = obj.get_content()
 
-    extracted = get_tracker_match(obj_id, content)
+    extracted = get_tracker_match(obj, content)
 
     # print(item.get_tags())
-    for tag in item.get_tags():
+    for tag in obj.get_tags():
         if MODULES.get(tag):
             # print(tag)
             module = MODULES.get(tag)
-            matches = module.extract(obj_id, content, tag)
+            matches = module.extract(obj, content, tag)
             if matches:
                 extracted = extracted + matches
 
-    for obj_t in ['cve', 'cryptocurrency', 'title', 'username']:  # Decoded, PGP->extract bloc
-        matches = get_correl_match(obj_t, obj_id, content)
+    for obj_t in CORRELATION_TO_EXTRACT[obj.type]:
+        matches = get_correl_match(obj_t, obj, content)
         if matches:
             extracted = extracted + matches
 
     # SORT By Start Pos
-    extracted = sorted(extracted, key=itemgetter(0))
-    # print(extracted)
+    if extracted:
+        extracted = sorted(extracted, key=itemgetter(0))
+        extracted = merge_overlap(extracted)
 
     # Save In Cache
     if extracted:
         extracted_dump = json.dumps(extracted)
-        r_cache.set(f'extractor:cache:{obj_id}', extracted_dump)
-        r_cache.expire(f'extractor:cache:{obj_id}', 300)  # TODO Reduce CACHE ???????????????
+        r_cache.set(f'extractor:cache:{obj_gid}', extracted_dump)
+        r_cache.expire(f'extractor:cache:{obj_gid}', 300)  # TODO Reduce CACHE ???????????????
 
     return extracted
 
 # TODO ADD LINK UI
 def get_extracted_by_match(extracted):
     matches = {}
-    for start, end, value, str_obj in extracted:
+    for start, end, value, raw_objs in extracted:
 
-        if str_obj not in matches:
-            matches[str_obj] = {}
-            ob_type, row_id = str_obj.split(':', 1)
-            if ob_type == 'tag':  # TODO put me in object class
-                matches[str_obj]['subtype'] = 'tag'
-                matches[str_obj]['id'] = row_id
-                matches[str_obj]['icon'] = {'style': 'fas', 'icon': '\uf02b', 'color': '#28a745', 'radius': 5}
-                matches[str_obj]['link'] = ''
-            elif ob_type == 'tracker':  # TODO put me in object class
-                matches[str_obj]['subtype'] = 'tracker'
-                matches[str_obj]['id'] = row_id
-                matches[str_obj]['icon'] = {'style': 'fas', 'icon': '\uf05b', 'color': '#ffc107', 'radius': 5}
-                matches[str_obj]['link'] = ''
-            elif ob_type == 'retro_hunt':  # TODO put me in object class
-                matches[str_obj]['subtype'] = 'retro_hunt'
-                matches[str_obj]['id'] = row_id
-                matches[str_obj]['icon'] = {'style': 'fas', 'icon': '\uf05b', 'color': '#008107', 'radius': 5}
-                matches[str_obj]['link'] = ''
-            else:
-                row_id = row_id.split(':', 1)
-                if len(row_id) == 2:
-                    subtype = row_id[0]
-                    obj_id = row_id[1]
+        for raw in raw_objs:
+            str_obj, str_match = raw
+
+            if str_obj not in matches:
+                matches[str_obj] = {}
+                ob_type, row_id = str_obj.split(':', 1)
+                if ob_type == 'tag':  # TODO put me in object class
+                    matches[str_obj]['subtype'] = 'tag'
+                    matches[str_obj]['id'] = row_id
+                    matches[str_obj]['icon'] = {'style': 'fas', 'icon': '\uf02b', 'color': '#28a745', 'radius': 5}
+                    matches[str_obj]['link'] = ''
+                elif ob_type == 'tracker':  # TODO put me in object class
+                    matches[str_obj]['subtype'] = 'tracker'
+                    matches[str_obj]['id'] = row_id
+                    matches[str_obj]['icon'] = {'style': 'fas', 'icon': '\uf05b', 'color': '#ffc107', 'radius': 5}
+                    matches[str_obj]['link'] = ''
+                elif ob_type == 'retro_hunt':  # TODO put me in object class
+                    matches[str_obj]['subtype'] = 'retro_hunt'
+                    matches[str_obj]['id'] = row_id
+                    matches[str_obj]['icon'] = {'style': 'fas', 'icon': '\uf05b', 'color': '#008107', 'radius': 5}
+                    matches[str_obj]['link'] = ''
                 else:
-                    subtype = ''
-                    obj_id = row_id[0]
-                matches[str_obj]['subtype'] = subtype
-                matches[str_obj]['id'] = obj_id
-                matches[str_obj]['icon'] = ail_objects.get_object_svg(ob_type, subtype, obj_id)
-                matches[str_obj]['link'] = ail_objects.get_object_link(ob_type, subtype, obj_id)
+                    row_id = row_id.split(':', 1)
+                    if len(row_id) == 2:
+                        subtype = row_id[0]
+                        obj_id = row_id[1]
+                    else:
+                        subtype = ''
+                        obj_id = row_id[0]
+                    matches[str_obj]['subtype'] = subtype
+                    matches[str_obj]['id'] = obj_id
+                    matches[str_obj]['icon'] = ail_objects.get_object_svg(ob_type, subtype, obj_id)
+                    matches[str_obj]['link'] = ail_objects.get_object_link(ob_type, subtype, obj_id)
 
-            matches[str_obj]['matches'] = []
+                matches[str_obj]['matches'] = []
 
-        match = [start, end, value]
-        matches[str_obj]['matches'].append(match)
+            match = [start, end, str_match]
+            matches[str_obj]['matches'].append(match)
     return matches
 
 
 # if __name__ == '__main__':
 #     t0 = time.time()
-#     obj_id = 'crawled/2022/09/15/circl.lu179c7903-5b21-452e-9f25-4b61d9934e2b'
-#     obj_id = 'crawled/2022/09/15/circl.lu1e4f9721-06dc-404f-aabf-3c3bd0b533bd'
-#     obj_id = 'submitted/2022/09/13/submitted_ba3ee771-c91c-4f50-9d6a-8558cdac7aeb.gz'
-#     # obj_id = 'tests/2021/01/01/credit_cards.gz'
-#     # obj_id = 'crawled/2020/07/20/circl.luc9301321-f1b1-4d91-9082-5eb452b946c5'
-#     obj_id = 'submitted/2019/09/22/97172282-e4c2-4a1e-b82c-c4fb9490a56e.gz'
-#     obj_id = 'submitted/2019/09/20/4fb7f02d-1241-4ef4-b17e-80ae76038835.gz'
 #     obj_id = 'crawled/2023/02/21/circl.lu1c300acb-0cbe-480f-917e-9afe3ec958e8'
-#
 #     extract(obj_id)
 #
 #     # get_obj_correl('cve', obj_id, content)

bin/lib/objects/Chats.py

@@ -18,13 +18,10 @@ from lib.ConfigLoader import ConfigLoader
 from lib.objects.abstract_chat_object import AbstractChatObject, AbstractChatObjects
 
 
-from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id
-from lib.data_retention_engine import update_obj_date
-from lib.objects import ail_objects
+from lib.objects.abstract_subtype_object import get_all_id
+# from lib.data_retention_engine import update_obj_date
 from lib.timeline_engine import Timeline
 
-from lib.correlations_engine import get_correlation_by_correl_type
-
 config_loader = ConfigLoader()
 baseurl = config_loader.get_config_str("Notifications", "ail_domain")
 r_object = config_loader.get_db_conn("Kvrocks_Objects")
@@ -56,6 +53,13 @@ class Chat(AbstractChatObject):
             url = f'{baseurl}/correlation/show?type={self.type}&subtype={self.subtype}&id={self.id}'
         return url
 
+    def get_origin_link(self):
+        if self.subtype == '00098785-7e70-5d12-a120-c5cdc1252b2b':
+            username = self.get_username()
+            if username:
+                username = username.split(':', 2)[2]
+                return f'https://t.me/{username}'
+
     def get_svg_icon(self):  # TODO
         # if self.subtype == 'telegram':
         #     style = 'fab'
@@ -75,6 +79,7 @@ class Chat(AbstractChatObject):
         meta['name'] = self.get_name()
         meta['tags'] = self.get_tags(r_list=True)
         if 'icon' in options:
+            meta['svg_icon'] = self.get_svg_icon()
             meta['icon'] = self.get_icon()
             meta['img'] = meta['icon']
         if 'info' in options:
@@ -99,6 +104,8 @@ class Chat(AbstractChatObject):
             meta['threads'] = self.get_threads()
         if 'tags_safe' in options:
             meta['tags_safe'] = self.is_tags_safe(meta['tags'])
+        if 'origin_link' in options:
+            meta['origin_link'] = self.get_origin_link()
         return meta
 
     def get_misp_object(self):

bin/lib/objects/CryptoCurrencies.py

@@ -60,7 +60,7 @@ class CryptoCurrency(AbstractSubtypeObject):
         pass
 
     def is_valid_address(self):
-        if self.type == 'bitcoin' or self.type == 'dash' or self.type == 'litecoin':
+        if self.subtype == 'bitcoin' or self.subtype == 'dash' or self.subtype == 'litecoin' or self.subtype == 'tron':
             return check_base58_address(self.id)
         else:
             return True
@@ -80,6 +80,8 @@ class CryptoCurrency(AbstractSubtypeObject):
             return 'ZEC'
         elif self.subtype == 'dash':
             return 'DASH'
+        elif self.subtype == 'tron':
+            return 'TRX'
         return None
 
     def get_link(self, flask_context=False):
@@ -140,7 +142,7 @@ class CryptoCurrency(AbstractSubtypeObject):
 
 def get_all_subtypes():
     # return ail_core.get_object_all_subtypes(self.type)
-    return ['bitcoin', 'bitcoin-cash', 'dash', 'ethereum', 'litecoin', 'monero', 'zcash']
+    return ['bitcoin', 'bitcoin-cash', 'dash', 'ethereum', 'litecoin', 'monero', 'tron', 'zcash']
 
 
 # def build_crypto_regex(subtype, search_id):
@@ -172,6 +174,8 @@ def get_subtype_by_symbol(symbol):
         return 'zcash'
     elif symbol == 'DASH':
         return 'dash'
+    elif symbol == 'TRX':
+        return 'tron'
     return None
 
 
@@ -189,10 +193,6 @@ def get_all_cryptocurrencies_by_subtype(subtype):
 def sanitize_cryptocurrency_name_to_search(name_to_search, subtype): # TODO FILTER NAME + Key + mail
     if subtype == '':
         pass
-    elif subtype == 'name':
-        pass
-    elif subtype == 'mail':
-        pass
     return name_to_search
 
 def search_cryptocurrency_by_name(name_to_search, subtype, r_pos=False):

bin/lib/objects/Domains.py

@@ -209,7 +209,7 @@ class Domain(AbstractObject):
     def get_screenshot(self):
         last_item = self.get_last_item_root()
         if last_item:
-            screenshot = self._get_external_correlation('item', '', last_item, 'screenshot').get('screenshot')
+            screenshot = self.get_obj_correlations('item', '', last_item, ['screenshot']).get('screenshot')
             if screenshot:
                 return screenshot.pop()[1:]
 
@@ -392,7 +392,7 @@ class Domain(AbstractObject):
                     print(har)
                     _write_in_zip_buffer(zf, os.path.join(hars_dir, har), f'{basename}.json.gz')
                 # Screenshot
-                screenshot = self._get_external_correlation('item', '', item_id, 'screenshot')
+                screenshot = self.get_obj_correlations('item', '', item_id, ['screenshot'])
                 if screenshot and screenshot['screenshot']:
                     screenshot = screenshot['screenshot'].pop()[1:]
                     screenshot = os.path.join(screenshot[0:2], screenshot[2:4], screenshot[4:6], screenshot[6:8],

bin/lib/objects/Images.py

@@ -2,6 +2,7 @@
 # -*-coding:UTF-8 -*
 
 import base64
+import magic
 import os
 import sys
 
@@ -50,7 +51,7 @@ class Image(AbstractDaterangeObject):
         if flask_context:
             url = url_for('correlation.show_correlation', type=self.type, id=self.id)
         else:
-            url = f'{baseurl}/correlation/show?type={self.type}&id={self.id}'
+            url = f'/correlation/show?type={self.type}&id={self.id}'
         return url
 
     def get_svg_icon(self):
@@ -64,6 +65,14 @@ class Image(AbstractDaterangeObject):
         filename = os.path.join(IMAGE_FOLDER, self.get_rel_path())
         return os.path.realpath(filename)
 
+    def is_gif(self, filepath=None):
+        if not filepath:
+            filepath = self.get_filepath()
+        mime = magic.from_file(filepath, mime=True)
+        if mime == 'image/gif':
+            return True
+        return False
+
     def get_file_content(self):
         filepath = self.get_filepath()
         with open(filepath, 'rb') as f:
@@ -109,6 +118,20 @@ class Image(AbstractDaterangeObject):
 def get_screenshot_dir():
     return IMAGE_FOLDER
 
+def get_all_images():
+    images = []
+    for root, dirs, files in os.walk(get_screenshot_dir()):
+        for file in files:
+            path = f'{root}{file}'
+            image_id = path.replace(IMAGE_FOLDER, '').replace('/', '')
+            images.append(image_id)
+    return images
+
+
+def get_all_images_objects(filters={}):
+    for image_id in get_all_images():
+        yield Image(image_id)
+
 
 def create(content, size_limit=5000000, b64=False, force=False):
     size = (len(content)*3) / 4
@@ -134,5 +157,6 @@ class Images(AbstractDaterangeObjects):
 
 
 # if __name__ == '__main__':
+#     print(json.dumps(get_all_images()))
 #     name_to_search = '29ba'
 #     print(search_screenshots_by_name(name_to_search))

bin/lib/objects/Messages.py

@@ -71,6 +71,10 @@ class Message(AbstractObject):
     def get_basename(self):
         return os.path.basename(self.id)
 
+    def get_chat_instance(self):
+        c_id = self.id.split('/')
+        return c_id[0]
+
     def get_content(self, r_type='str'): # TODO ADD cache # TODO Compress content ???????
         """
         Returns content
@@ -136,12 +140,15 @@ class Message(AbstractObject):
     # TODO get channel ID
     # TODO get thread  ID
 
+    def _get_image_ocr(self, obj_id):
+        return bool(self.get_correlation('ocr').get('ocr'))
+
     def get_images(self):
         images = []
         for child in self.get_childrens():
             obj_type, _, obj_id = child.split(':', 2)
             if obj_type == 'image':
-                images.append(obj_id)
+                images.append({'id': obj_id, 'ocr': self._get_image_ocr(obj_id)})
         return images
 
     def get_user_account(self, meta=False):
@@ -202,12 +209,6 @@ class Message(AbstractObject):
         else:
             return None
 
-    def _set_translation(self, translation):
-        """
-        Set translated content
-        """
-        return self._set_field('translated', translation)  # translation by hash ??? -> avoid translating multiple time
-
     # def get_ail_2_ail_payload(self):
     #     payload = {'raw': self.get_gzip_content(b64=True)}
     #     return payload
@@ -259,7 +260,7 @@ class Message(AbstractObject):
         else:
             timestamp = float(timestamp)
         timestamp = datetime.utcfromtimestamp(float(timestamp))
-        meta['date'] = timestamp.strftime('%Y/%m/%d')
+        meta['date'] = timestamp.strftime('%Y-%m-%d')
         meta['hour'] = timestamp.strftime('%H:%M:%S')
         meta['full_date'] = timestamp.isoformat(' ')
         if 'last_full_date' in options:
@@ -319,7 +320,6 @@ class Message(AbstractObject):
     #         content = self.get_content()
     #     translated = argostranslate.translate.translate(content, 'ru', 'en')
     #     # Save translation
-    #     self._set_translation(translated)
     #     return translated
 
     ## Language ##
@@ -343,7 +343,6 @@ class Message(AbstractObject):
         if not language and content:
             language = self.detect_language()
         if translation and content:
-            self._set_translation(translation)
             self.set_translation(language, translation)
         for tag in tags:
             self.add_tag(tag)

bin/lib/objects/Ocrs.py

@@ -0,0 +1,336 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import os
+import sys
+
+from datetime import datetime
+from io import BytesIO
+from PIL import Image
+from PIL import ImageDraw
+
+from pymisp import MISPObject
+
+sys.path.append(os.environ['AIL_BIN'])
+##################################
+# Import Project packages
+##################################
+from lib.objects.abstract_daterange_object import AbstractDaterangeObject, AbstractDaterangeObjects
+from lib.ConfigLoader import ConfigLoader
+from packages import Date
+# from lib import Language
+# from lib.data_retention_engine import update_obj_date, get_obj_date_first
+
+from flask import url_for
+
+config_loader = ConfigLoader()
+r_cache = config_loader.get_redis_conn("Redis_Cache")
+r_object = config_loader.get_db_conn("Kvrocks_Objects")
+baseurl = config_loader.get_config_str("Notifications", "ail_domain")
+IMAGE_FOLDER = config_loader.get_files_directory('images')
+config_loader = None
+
+# SET x1,y1:x2,y2:x3,y3:x4,y4:extracted_text
+
+class Ocr(AbstractDaterangeObject):
+    """
+    AIL Message Object. (strings)
+    """
+
+    def __init__(self, id):
+        super(Ocr, self).__init__('ocr', id)
+
+    def exists(self):
+        return r_object.exists(f'ocr:{self.id}')
+
+    def get_content(self, r_type='str'):
+        """
+        Returns content
+        """
+        global_id = self.get_global_id()
+        content = r_cache.get(f'content:{global_id}')
+        if not content:
+            dict_content = {}
+            for extracted in r_object.smembers(f'ocr:{self.id}'):
+                extracted = extracted.split(':', 4)
+                x, y = extracted[0].split(',', 1)
+                # get text line, y +- 20
+                rounded_y = round(int(y) / 20) * 20
+                if rounded_y not in dict_content:
+                    dict_content[rounded_y] = []
+                dict_content[rounded_y].append((int(x), int(y), extracted[-1]))
+
+            content = ''
+            new_line = True
+            l_key = sorted(dict_content.keys())
+            for key in l_key:
+                dict_content[key] = sorted(dict_content[key], key=lambda c: c[0])
+                for text in dict_content[key]:
+                    if new_line:
+                        content = f'{content}{text[2]}'
+                        new_line = False
+                    else:
+                        content = f'{content}      {text[2]}'
+                content = f'{content}\n'
+                new_line = True
+
+            # Set Cache
+            if content:
+                global_id = self.get_global_id()
+                r_cache.set(f'content:{global_id}', content)
+                r_cache.expire(f'content:{global_id}', 300)
+
+        if r_type == 'str':
+            return content
+        elif r_type == 'bytes':
+            if content:
+                return content.encode()
+
+    def get_date(self): # TODO
+        return Date.get_today_date_str()
+
+    def get_source(self): # TODO
+        """
+        Returns source/feeder name
+        """
+        return 'ocr'
+        # l_source = self.id.split('/')[:-2]
+        # return os.path.join(*l_source)
+
+    def get_basename(self):  # TODO
+        return 'ocr'
+
+    def get_language(self):
+        languages = self.get_languages()
+        if languages:
+            return languages.pop()
+        else:
+            return None
+
+    def get_link(self, flask_context=False):
+        if flask_context:
+            url = url_for('correlation.show_correlation', type=self.type, id=self.id)
+        else:
+            url = f'{baseurl}/correlation/show?type={self.type}&id={self.id}'
+        return url
+
+    def get_svg_icon(self):
+        return {'style': 'fas', 'icon': '\uf065', 'color': 'yellow', 'radius': 5}
+
+    def get_image_path(self):
+        rel_path = os.path.join(self.id[0:2], self.id[2:4], self.id[4:6], self.id[6:8], self.id[8:10], self.id[10:12], self.id[12:])
+        filename = os.path.join(IMAGE_FOLDER, rel_path)
+        return os.path.realpath(filename)
+
+    def get_misp_object(self):  # TODO
+        obj = MISPObject('instant-message', standalone=True)
+        obj_date = self.get_date()
+        if obj_date:
+            obj.first_seen = obj_date
+        else:
+            self.logger.warning(
+                f'Export error, None seen {self.type}:{self.subtype}:{self.id}, first={obj_date}')
+
+        # obj_attrs = [obj.add_attribute('first-seen', value=obj_date),
+        #              obj.add_attribute('raw-data', value=self.id, data=self.get_raw_content()),
+        #              obj.add_attribute('sensor', value=get_ail_uuid())]
+        obj_attrs = []
+        for obj_attr in obj_attrs:
+            for tag in self.get_tags():
+                obj_attr.add_tag(tag)
+        return obj
+
+    # options: set of optional meta fields
+    def get_meta(self, options=None, translation_target=''):
+        """
+        :type options: set
+        """
+        if options is None:
+            options = set()
+        meta = self._get_meta(options=options)
+        meta['tags'] = self.get_tags()
+        meta['content'] = self.get_content()
+
+        # optional meta fields
+        if 'investigations' in options:
+            meta['investigations'] = self.get_investigations()
+        if 'link' in options:
+            meta['link'] = self.get_link(flask_context=True)
+        if 'icon' in options:
+            meta['svg_icon'] = self.get_svg_icon()
+        if 'img' in options:
+            meta['img'] = self.draw_bounding_boxs()
+        if 'map' in options:
+            meta['map'] = self.get_img_map_coords()
+        if 'language' in options:
+            meta['language'] = self.get_language()
+        if 'translation' in options and translation_target:
+            if meta.get('language'):
+                source = meta['language']
+            else:
+                source = None
+            meta['translation'] = self.translate(content=meta.get('content'), source=source, target=translation_target)
+            if 'language' in options:
+                meta['language'] = self.get_language()
+        return meta
+
+    def get_objs_container(self):
+        objs_containers = set()
+        # chat
+        objs_containers.add(self.get_first_correlation('chat'))
+        subchannel = self.get_first_correlation('chat-subchannel')
+        if subchannel:
+            objs_containers.add(subchannel)
+        thread = self.get_first_correlation('chat-thread')
+        if thread:
+            objs_containers.add(thread)
+        return objs_containers
+
+    def create_coord_str(self, bbox):
+        c1, c2, c3, c4 = bbox
+        x1, y1 = c1
+        x2, y2 = c2
+        x3, y3 = c3
+        x4, y4 = c4
+        return f'{int(x1)},{int(y1)}:{int(x2)},{int(y2)}:{int(x3)},{int(y3)}:{int(x4)},{int(y4)}'
+
+    def _unpack_coord(self, coord):
+        return coord.split(',', 1)
+
+    def get_coords(self):
+        coords = []
+        for extracted in r_object.smembers(f'ocr:{self.id}'):
+            coord = []
+            bbox = extracted.split(':', 4)[:-1]
+            for c in bbox:
+                x, y = self._unpack_coord(c)
+                coord.append((int(x), int(y)))
+            coords.append(coord)
+        return coords
+
+    def get_img_map_coords(self):
+        coords = []
+        for extracted in r_object.smembers(f'ocr:{self.id}'):
+            extract = extracted.split(':', 4)
+            x1, y1 = self._unpack_coord(extract[0])
+            x2, y2 = self._unpack_coord(extract[1])
+            x3, y3 = self._unpack_coord(extract[2])
+            x4, y4 = self._unpack_coord(extract[3])
+            coords.append((f'{x1},{y1},{x2},{y2},{x3},{y3},{x4},{y4}', extract[4]))
+        return coords
+
+    def edit_text(self, coordinates, text, new_text, new_coordinates=None):
+        pass
+
+    def add_text(self, coordinates, text):
+        val = f'{coordinates}:{text}'
+        return r_object.sadd(f'ocr:{self.id}', val)
+
+    def remove_text(self, val):
+        return r_object.srem(f'ocr:{self.id}', val)
+
+    def update_correlation(self, date=None):
+        if date:
+            self.add(date, None)
+        image_correl = self.get_obj_correlations('image', '', self.id)
+        for obj_type in image_correl:
+            if obj_type != 'ocr':
+                for obj_raw in image_correl[obj_type]:
+                    obj_subtype, obj_id = obj_raw.split(':', 1)
+                    self.add_correlation(obj_type, obj_subtype, obj_id)
+
+    def create(self, extracted_texts, tags=[]):
+        # r_object.sadd(f'{self.type}:all', self.id)
+        created = False
+        for extracted in extracted_texts:
+            bbox, text = extracted
+            if len(text) > 1:
+                str_coords = self.create_coord_str(bbox)
+                self.add_text(str_coords, text)
+                created = True
+
+        if created:
+            # Correlations
+            self._copy_from('image', self.id)
+            self.update_correlation()
+            self.add_correlation('image', '', self.id)
+
+            for tag in tags:
+                self.add_tag(tag)
+            return self.id
+
+    # # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\
+    def delete(self):
+        r_object.delete(f'ocr:{self.id}')
+
+    def draw_bounding_boxs(self):
+        img = Image.open(self.get_image_path()).convert("RGBA")
+        draw = ImageDraw.Draw(img)
+        for bbox in self.get_coords():
+            c1, c2, c3, c4 = bbox
+            draw.line((tuple(c1), tuple(c2)), fill="yellow", width=2)
+            draw.line((tuple(c2), tuple(c3)), fill="yellow", width=2)
+            draw.line((tuple(c3), tuple(c4)), fill="yellow", width=2)
+            draw.line((tuple(c4), tuple(c1)), fill="yellow", width=2)
+        # img.show()
+        buff = BytesIO()
+        img.save(buff, "PNG")
+        return buff.getvalue()
+
+
+def create(obj_id, detections, tags=[]):
+    obj = Ocr(obj_id)
+    if not obj.exists():
+        obj_id = obj.create(detections, tags=tags)
+        if obj_id:
+            return obj
+
+# TODO preload languages
+def extract_text(image_path, languages, threshold=0.2):
+    import easyocr
+    reader = easyocr.Reader(languages, verbose=False)
+    texts = reader.readtext(image_path)
+    # print(texts)
+    extracted = []
+    for bbox, text, score in texts:
+        if score > threshold:
+            extracted.append((bbox, text))
+    return extracted
+
+
+def get_ocr_languages():
+    return {'af', 'ar', 'as', 'az', 'be', 'bg', 'bh', 'bs', 'cs', 'cy', 'da', 'de', 'en', 'es', 'et', 'fa', 'fr', 'ga', 'hi', 'hr', 'hu', 'id', 'is', 'it', 'ja', 'kn', 'ko', 'ku', 'la', 'lt', 'lv', 'mi', 'mn', 'mr', 'ms', 'mt', 'ne', 'nl', 'no', 'oc', 'pi', 'pl', 'pt', 'ro', 'ru', 'sk', 'sl', 'sq', 'sr', 'sv', 'sw', 'ta', 'te', 'th', 'tl', 'tr', 'ug', 'uk', 'ur', 'uz', 'vi', 'zh'}
+
+
+def sanityze_ocr_languages(languages, ocr_languages=None):
+    langs = set()
+    if not ocr_languages:
+        ocr_languages = get_ocr_languages()
+    for lang in languages:
+        if lang in ocr_languages:
+            if lang == 'zh':
+                langs.add('ch_sim')
+            elif lang == 'sr':
+                langs.add('rs_latin')
+            else:
+                langs.add(lang)
+    return langs
+
+class Ocrs(AbstractDaterangeObjects):
+    """
+        OCR Objects
+    """
+    def __init__(self):
+        super().__init__('ocr', Ocr)
+
+    def sanitize_id_to_search(self, name_to_search):
+        return name_to_search  # TODO
+
+
+#### API ####
+def api_get_ocr(obj_id, translation_target=None):
+    ocr = Ocr(obj_id)
+    if not ocr.exists():
+        return {"status": "error", "reason": "Unknown ocr"}, 404
+    meta = ocr.get_meta({'content', 'icon', 'img', 'language', 'link', 'map', 'translation'}, translation_target=translation_target)
+    return meta, 200

bin/lib/objects/UsersAccount.py

@@ -150,13 +150,14 @@ class UserAccount(AbstractSubtypeObject):
             if meta['username']:
                 _, username_account_subtype, username_account_id = meta['username'].split(':', 3)
                 if 'username_meta' in options:
-                    meta['username'] = Usernames.Username(username_account_id, username_account_subtype).get_meta()
+                    meta['username'] = Usernames.Username(username_account_id, username_account_subtype).get_meta(options={'icon'})
                 else:
                     meta['username'] = {'type': 'username', 'subtype': username_account_subtype, 'id': username_account_id}
         if 'usernames' in options:
             meta['usernames'] = self.get_usernames()
         if 'icon' in options:
             meta['icon'] = self.get_icon()
+            meta['svg_icon'] = meta['icon']
         if 'info' in options:
             meta['info'] = self.get_info()
             if 'translation' in options and translation_target:

bin/lib/objects/abstract_chat_object.py

@@ -307,6 +307,9 @@ class AbstractChatObject(AbstractSubtypeObject, ABC):
     def get_nb_participants(self):
         return self.get_nb_correlation('user-account')
 
+    def get_user_messages(self, user_id):
+        return self.get_correlation_iter('user-account', self.subtype, user_id, 'message')
+
 # TODO move me to abstract subtype
 class AbstractChatObjects(ABC):
     def __init__(self, type):

bin/lib/objects/abstract_daterange_object.py

@@ -71,7 +71,7 @@ class AbstractDaterangeObject(AbstractObject, ABC):
         else:
             return last_seen
 
-    def get_nb_seen(self): # TODO REPLACE ME -> correlation image
+    def get_nb_seen(self): # TODO REPLACE ME -> correlation image chats
         return self.get_nb_correlation('item') + self.get_nb_correlation('message')
 
     def get_nb_seen_by_date(self, date):
@@ -127,6 +127,20 @@ class AbstractDaterangeObject(AbstractObject, ABC):
     def _add_create(self):
         r_object.sadd(f'{self.type}:all', self.id)
 
+    def _copy_from(self, obj_type, obj_id):
+        first_seen = r_object.hget(f'meta:{obj_type}:{obj_id}', 'first_seen')
+        last_seen = r_object.hget(f'meta:{obj_type}:{obj_id}', 'last_seen')
+        if first_seen and last_seen:
+            for date in Date.get_daterange(first_seen, last_seen):
+                nb = r_object.zscore(f'{obj_type}:date:{date}', self.id)
+                if nb:
+                    r_object.zincrby(f'{self.type}:date:{date}', nb, self.id)
+            update_obj_date(first_seen, self.type)
+            update_obj_date(last_seen, self.type)
+            self._add_create()
+            self.set_first_seen(first_seen)
+            self.set_last_seen(last_seen)
+
     def _add(self, date, obj): # TODO OBJ=None
         if not self.exists():
             self._add_create()

bin/lib/objects/abstract_object.py

@@ -25,7 +25,7 @@ from lib import Duplicate
 from lib.correlations_engine import get_nb_correlations, get_correlations, add_obj_correlation, delete_obj_correlation, delete_obj_correlations, exists_obj_correlation, is_obj_correlated, get_nb_correlation_by_correl_type, get_obj_inter_correlation
 from lib.Investigations import is_object_investigated, get_obj_investigations, delete_obj_investigations
 from lib.relationships_engine import get_obj_nb_relationships, add_obj_relationship
-from lib.Language import get_obj_languages, add_obj_language, remove_obj_language, detect_obj_language, get_obj_language_stats, get_obj_translation, set_obj_translation, delete_obj_translation
+from lib.Language import get_obj_languages, add_obj_language, remove_obj_language, detect_obj_language, get_obj_language_stats, get_obj_translation, set_obj_translation, delete_obj_translation, get_obj_main_language
 from lib.Tracker import is_obj_tracked, get_obj_trackers, delete_obj_trackers
 
 logging.config.dictConfig(ail_logger.get_config(name='ail'))
@@ -225,11 +225,11 @@ class AbstractObject(ABC):
 
     ## Correlation ##
 
-    def _get_external_correlation(self, req_type, req_subtype, req_id, obj_type):
+    def get_obj_correlations(self, obj_type, obj_subtype, obj_id, filter_types=[]):
         """
         Get object correlation
         """
-        return get_correlations(req_type, req_subtype, req_id, filter_types=[obj_type])
+        return get_correlations(obj_type, obj_subtype, obj_id, filter_types=filter_types)
 
     def get_correlation(self, obj_type):
         """
@@ -237,6 +237,11 @@ class AbstractObject(ABC):
         """
         return get_correlations(self.type, self.subtype, self.id, filter_types=[obj_type])
 
+    def get_first_correlation(self, obj_type):
+        correlation = self.get_correlation(obj_type)
+        if correlation.get(obj_type):
+            return f'{obj_type}:{correlation[obj_type].pop()}'
+
     def get_correlations(self, filter_types=[], unpack=False):
         """
         Get object correlations
@@ -330,6 +335,9 @@ class AbstractObject(ABC):
     def get_obj_language_stats(self):
         return get_obj_language_stats(self.type, self.get_subtype(r_str=True), self.id)
 
+    def get_main_language(self):
+        return get_obj_main_language(self.type, self.get_subtype(r_str=True), self.id)
+
     def get_translation(self, language, field=''):
         return get_obj_translation(self.get_global_id(), language, field=field, objs_containers=self.get_objs_container())
 

bin/lib/objects/ail_objects.py

@@ -9,13 +9,12 @@ sys.path.append(os.environ['AIL_BIN'])
 ##################################
 from lib.exceptions import AILObjectUnknown
 
-
-
 from lib.ConfigLoader import ConfigLoader
 from lib.ail_core import get_all_objects, get_object_all_subtypes, get_objects_with_subtypes, get_default_correlation_objects
 from lib import correlations_engine
 from lib import relationships_engine
 from lib import btc_ail
+from lib import Language
 from lib import Tag
 
 from lib import chats_viewer
@@ -35,10 +34,11 @@ from lib.objects import HHHashs
 from lib.objects.Items import Item, get_all_items_objects, get_nb_items_objects
 from lib.objects import Images
 from lib.objects import Messages
+from lib.objects import Ocrs
 from lib.objects import Pgps
 from lib.objects.Screenshots import Screenshot
 from lib.objects import Titles
-from lib.objects.UsersAccount import UserAccount
+from lib.objects import UsersAccount
 from lib.objects import Usernames
 
 config_loader = ConfigLoader()
@@ -93,6 +93,8 @@ def get_object(obj_type, subtype, obj_id):
             return Images.Image(obj_id)
         elif obj_type == 'message':
             return Messages.Message(obj_id)
+        elif obj_type == 'ocr':
+            return Ocrs.Ocr(obj_id)
         elif obj_type == 'screenshot':
             return Screenshot(obj_id)
         elif obj_type == 'title':
@@ -112,7 +114,7 @@ def get_object(obj_type, subtype, obj_id):
         elif obj_type == 'pgp':
             return Pgps.Pgp(obj_id, subtype)
         elif obj_type == 'user-account':
-            return UserAccount(obj_id, subtype)
+            return UsersAccount.UserAccount(obj_id, subtype)
         elif obj_type == 'username':
             return Usernames.Username(obj_id, subtype)
         else:
@@ -254,7 +256,7 @@ def get_objects_meta(objs, options=set(), flask_context=False):
 
 def get_object_card_meta(obj_type, subtype, id, related_btc=False):
     obj = get_object(obj_type, subtype, id)
-    meta = obj.get_meta(options={'chat', 'chats', 'created_at', 'icon', 'info', 'nb_messages', 'nb_participants', 'threads', 'username'})
+    meta = obj.get_meta(options={'chat', 'chats', 'created_at', 'icon', 'info', 'map', 'nb_messages', 'nb_participants', 'threads', 'username'})
     # meta['icon'] = obj.get_svg_icon()
     meta['svg_icon'] = obj.get_svg_icon()
     if subtype or obj_type == 'cookie-name' or obj_type == 'cve' or obj_type == 'etag' or obj_type == 'title' or obj_type == 'favicon' or obj_type == 'hhhash':
@@ -274,6 +276,34 @@ def get_object_card_meta(obj_type, subtype, id, related_btc=False):
     meta["add_tags_modal"] = Tag.get_modal_add_tags(obj.id, obj.get_type(), obj.get_subtype(r_str=True))
     return meta
 
+#### OBJ LANGUAGES ####
+
+def api_detect_language(obj_type, subtype, obj_id):
+    obj = get_object(obj_type, subtype, obj_id)
+    if not obj.exists():
+        return {"status": "error", "reason": "Unknown obj"}, 404
+    lang = obj.detect_language()
+    return {"language": lang}, 200
+
+def api_manually_translate(obj_type, subtype, obj_id, source, translation_target, translation):
+    obj = get_object(obj_type, subtype, obj_id)
+    if not obj.exists():
+        return {"status": "error", "reason": "Unknown obj"}, 404
+    if translation:
+        if len(translation) > 200000: # TODO REVIEW LIMIT
+            return {"status": "error", "reason": "Max Size reached"}, 400
+    all_languages = Language.get_translation_languages()
+    if source not in all_languages:
+        return {"status": "error", "reason": "Unknown source Language"}, 400
+    obj_language = obj.get_language()
+    if obj_language != source:
+        obj.edit_language(obj_language, source)
+    if translation:
+        if translation_target not in all_languages:
+            return {"status": "error", "reason": "Unknown target Language"}, 400
+        obj.set_translation(translation_target, translation)
+    # TODO SANITYZE translation
+    return None, 200
 
 #### OBJ FILTERS ####
 
@@ -295,6 +325,8 @@ def is_filtered(obj, filters):
 def obj_iterator(obj_type, filters):
     if obj_type == 'decoded':
         return get_all_decodeds_objects(filters=filters)
+    elif obj_type == 'image':
+        return Images.get_all_images_objects(filters=filters)
     elif obj_type == 'item':
         return get_all_items_objects(filters=filters)
     elif obj_type == 'pgp':

bin/modules/Categ.py

@@ -89,7 +89,7 @@ class Categ(AbstractModule):
         # Search for pattern categories in obj content
         for categ, pattern in self.categ_words:
 
-            if obj.type == 'message':
+            if obj.type == 'message' or obj.type == 'ocr':
                 self.add_message_to_queue(message='0', queue=categ)
             else:
 

bin/modules/CreditCards.py

@@ -58,9 +58,9 @@ class CreditCards(AbstractModule):
         if lib_refine.is_luhn_valid(clean_card):
             return clean_card
 
-    def extract(self, obj_id, content, tag):
+    def extract(self, obj, content, tag):
         extracted = []
-        cards = self.regex_finditer(self.regex, obj_id, content)
+        cards = self.regex_finditer(self.regex, obj.get_global_id(), content)
         for card in cards:
             start, end, value = card
             if self.get_valid_card(value):

bin/modules/Cryptocurrencies.py

@@ -92,7 +92,13 @@ CURRENCIES = {
         'regex': r'\b(?<![+/=])X[A-Za-z0-9]{33}(?![+/=])\b',
         'max_execution_time': default_max_execution_time,
         'tag': 'infoleak:automatic-detection="dash-address"',
-    }
+    },
+    'tron': {
+            'name': 'tron',  # e.g. TYdds9VLDjUshf9tbsXSfGUZNzJSbbBeat
+            'regex': r'\b(?<![+/=])T[0-9a-zA-Z]{33}(?![+/=])\b',
+            'max_execution_time': default_max_execution_time,
+            'tag': 'infoleak:automatic-detection="tron-address"',
+        },
 }
 ##################################
 ##################################

bin/modules/Global.py

@@ -128,11 +128,11 @@ class Global(AbstractModule):
 
             else:
                 self.logger.info(f"Empty Item: {message} not processed")
-        elif self.obj.type == 'message':
+        elif self.obj.type == 'message' or self.obj.type == 'ocr':
             # TODO send to specific object queue => image, ...
             self.add_message_to_queue(obj=self.obj, queue='Item')
         elif self.obj.type == 'image':
-            self.add_message_to_queue(obj=self.obj, queue='Image')
+            self.add_message_to_queue(obj=self.obj, queue='Image', message=message)
         else:
             self.logger.critical(f"Empty obj: {self.obj} {message} not processed")
 

bin/modules/Iban.py

@@ -62,9 +62,9 @@ class Iban(AbstractModule):
             return True
         return False
 
-    def extract(self, obj_id, content, tag):
+    def extract(self, obj, content, tag):
         extracted = []
-        ibans = self.regex_finditer(self.iban_regex, obj_id, content)
+        ibans = self.regex_finditer(self.iban_regex, obj.get_global_id(), content)
         for iban in ibans:
             start, end, value = iban
             value = ''.join(e for e in value if e.isalnum())

bin/modules/MISP_Thehive_Auto_Push.py

@@ -57,7 +57,7 @@ class MISP_Thehive_Auto_Push(AbstractModule):
                     Tag.set_auto_push_status('misp', 'ConnectionError')
                 else:
                     Tag.set_auto_push_status('misp', '')
-                    self.logger.info('MISP Pushed:', tag, '->', item_id)
+                    self.logger.info(f'MISP Pushed: {tag} -> {item_id}')
 
         if 'thehive' in self.tags:
             if tag in self.tags['thehive']:
@@ -68,7 +68,7 @@ class MISP_Thehive_Auto_Push(AbstractModule):
                     Tag.set_auto_push_status('thehive', 'Request Entity Too Large')
                 else:
                     Tag.set_auto_push_status('thehive', '')
-                    self.logger.info('thehive Pushed:', tag, '->', item_id)
+                    self.logger.info(f'thehive Pushed: {tag} -> {item_id}')
 
 
 if __name__ == "__main__":

bin/modules/Mail.py

@@ -26,7 +26,6 @@ sys.path.append(os.environ['AIL_BIN'])
 # Import Project packages        #
 ##################################
 from modules.abstract_module import AbstractModule
-from lib.objects.Items import Item
 from lib.ConfigLoader import ConfigLoader
 # from lib import Statistics
 
@@ -118,10 +117,10 @@ class Mail(AbstractModule):
                     print(e)
         return valid_mxdomain
 
-    def extract(self, obj_id, content, tag):
+    def extract(self, obj, content, tag):
         extracted = []
         mxdomains = {}
-        mails = self.regex_finditer(self.email_regex, obj_id, content)
+        mails = self.regex_finditer(self.email_regex, obj.get_global_id(), content)
         for mail in mails:
             start, end, value = mail
             mxdomain = value.rsplit('@', 1)[1].lower()

bin/modules/Mixer.py

@@ -218,7 +218,7 @@ class Mixer(AbstractModule):
             if self.obj.type == 'item':
                 self.add_message_to_queue(obj=self.obj, message=gzip64encoded)
             else:
-                self.add_message_to_queue(obj=self.obj)
+                self.add_message_to_queue(obj=self.obj, message=gzip64encoded)
 
 
 if __name__ == "__main__":

bin/modules/OcrExtractor.py

@@ -0,0 +1,132 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+"""
+The OcrExtractor Module
+======================
+
+"""
+
+##################################
+# Import External packages
+##################################
+import cv2
+import os
+import sys
+
+sys.path.append(os.environ['AIL_BIN'])
+##################################
+# Import Project packages
+##################################
+from modules.abstract_module import AbstractModule
+from lib.ConfigLoader import ConfigLoader
+from lib import chats_viewer
+from lib.objects import Messages
+from lib.objects import Ocrs
+
+
+# Default to eng
+def get_model_languages(obj, add_en=True):
+    if add_en:
+        model_languages = {'en'}
+    else:
+        model_languages = set()
+
+    ob = obj.get_first_correlation('message')
+    if ob:
+        message = Messages.Message(ob.split(':', 2)[-1])
+        lang = message.get_language()
+        if lang:
+            model_languages.add(lang)
+            return model_languages
+
+    ob = obj.get_first_correlation('chat-subchannel')
+    if ob:
+        ob = chats_viewer.get_obj_chat_from_global_id(ob)
+        lang = ob.get_main_language()
+        if lang:
+            model_languages.add(lang)
+            return model_languages
+
+    ob = obj.get_first_correlation('chat')
+    if ob:
+        ob = chats_viewer.get_obj_chat_from_global_id(ob)
+        lang = ob.get_main_language()
+        if lang:
+            model_languages.add(lang)
+            return model_languages
+
+    return model_languages
+
+    #  TODO thread
+
+
+class OcrExtractor(AbstractModule):
+    """
+    OcrExtractor for AIL framework
+    """
+
+    def __init__(self):
+        super(OcrExtractor, self).__init__()
+
+        # Waiting time in seconds between to message processed
+        self.pending_seconds = 1
+
+        config_loader = ConfigLoader()
+        self.r_cache = config_loader.get_redis_conn("Redis_Cache")
+
+        self.ocr_languages = Ocrs.get_ocr_languages()
+
+        # Send module state to logs
+        self.logger.info(f'Module {self.module_name} initialized')
+
+    def is_cached(self):
+        return self.r_cache.exists(f'ocr:no:{self.obj.id}')
+
+    def add_to_cache(self):
+        self.r_cache.setex(f'ocr:no:{self.obj.id}', 86400, 0)
+
+    def compute(self, message):
+        image = self.get_obj()
+        date = message
+
+        ocr = Ocrs.Ocr(image.id)
+        if self.is_cached():
+            return None
+
+        if self.obj.is_gif():
+            self.logger.warning(f'Ignoring GIF: {self.obj.id}')
+            return None
+
+        if not ocr.exists():
+            path = image.get_filepath()
+            languages = get_model_languages(image)
+            languages = Ocrs.sanityze_ocr_languages(languages, ocr_languages=self.ocr_languages)
+            print(image.id, languages)
+            try:
+                texts = Ocrs.extract_text(path, languages)
+            except (OSError, ValueError, cv2.error) as e:
+                self.logger.warning(e)
+                self.obj.add_tag('infoleak:confirmed="false-positive"')
+                texts = None
+            if texts:
+                print('create')
+                ocr = Ocrs.create(image.id, texts)
+                if ocr:
+                    self.add_message_to_queue(ocr)
+                else:
+                    print('no text')
+                    self.add_to_cache()
+            # Save in cache
+            else:
+                print('no text detected')
+                self.add_to_cache()
+        else:
+            # print(image.id)
+            # print('update correlation', date)
+            ocr.update_correlation(date=date)
+
+
+if __name__ == '__main__':
+
+    module = OcrExtractor()
+    module.run()

bin/modules/Onion.py

@@ -55,9 +55,9 @@ class Onion(AbstractModule):
         # TEMP var: SAVE I2P Domain (future I2P crawler)
         # self.save_i2p = config_loader.get_config_boolean("Onion", "save_i2p")
 
-    def extract(self, obj_id, content, tag):
+    def extract(self, obj, content, tag):
         extracted = []
-        onions = self.regex_finditer(self.onion_regex, obj_id, content)
+        onions = self.regex_finditer(self.onion_regex, obj.get_global_id(), content)
         for onion in onions:
             start, end, value = onion
             url_unpack = crawlers.unpack_url(value)

bin/modules/Phone.py

@@ -41,9 +41,9 @@ class Phone(AbstractModule):
         # Waiting time in seconds between to message processed
         self.pending_seconds = 1
 
-    def extract(self, obj_id, content, tag):
+    def extract(self, obj, content, tag):
         extracted = []
-        phones = self.regex_phone_iter('ZZ', obj_id, content)
+        phones = self.regex_phone_iter('ZZ', obj.get_global_id(), content)
         for phone in phones:
             extracted.append([phone[0], phone[1], phone[2], f'tag:{tag}'])
         return extracted

bin/packages/Date.py

@@ -1,6 +1,7 @@
 #!/usr/bin/python3
 
 import datetime
+import time
 from calendar import monthrange
 
 from dateutil.rrule import rrule, MONTHLY
@@ -91,6 +92,10 @@ def get_current_week_day():
     start = dt - datetime.timedelta(days=dt.weekday())
     return start.strftime("%Y%m%d")
 
+def get_current_utc_full_time():
+    timestamp = datetime.datetime.fromtimestamp(time.time())
+    return timestamp.strftime('%Y-%m-%d %H:%M:%S')
+
 def get_month_dates(date=None):
     if date:
         date = convert_date_str_to_datetime(date)
@@ -258,3 +263,9 @@ def sanitise_daterange(date_from, date_to, separator='', date_type='str'):
         date_from = date_to
         date_to = res
     return date_from, date_to
+
+def get_previous_month_date():
+    now = datetime.date.today()
+    first = now.replace(day=1)
+    last_month = first - datetime.timedelta(days=1)
+    return last_month.strftime("%Y%m%d")

configs/modules.cfg

@@ -162,6 +162,9 @@ publish = Tags
 subscribe = Image
 publish = Tags
 
+[OcrExtractor]
+subscribe = Image
+publish = Item
 
 ######## CORE ########
 

installing_deps.sh

@@ -72,6 +72,7 @@ popd
 # pgpdump
 test ! -d pgpdump && git clone https://github.com/kazu-yamamoto/pgpdump.git
 pushd pgpdump/
+autoreconf -fiW all
 ./configure
 make
 sudo make install

requirements.txt

@@ -82,6 +82,9 @@ bcrypt>3.1.6
 # Ail typo squatting
 ail_typo_squatting
 
+# OCR
+easyocr
+
 # Tests
 nose2>=0.12.0
 coverage>=5.5

tools/reprocess_objects.py

@@ -30,15 +30,21 @@ from lib.objects import ail_objects
 # from modules.Telegram import Telegram
 
 from modules.Languages import Languages
+from modules.OcrExtractor import OcrExtractor
 
 MODULES = {
-    'Languages': Languages
+    'Languages': Languages,
+    'OcrExtractor': OcrExtractor
+
 }
 
 def reprocess_message_objects(object_type, module_name=None):
     if module_name:
         module = MODULES[module_name]()
         for obj in ail_objects.obj_iterator(object_type, filters={}):
+            if not obj.exists():
+                print(f'ERROR: object does not exist, {obj.id}')
+                continue
             module.obj = obj
             module.compute(None)
     else:
@@ -62,7 +68,7 @@ if __name__ == "__main__":
     obj_type = args.type
     if not is_object_type(obj_type):
         raise Exception(f'Invalid Object Type: {obj_type}')
-    if obj_type not in ['item', 'message']:  # TODO image
+    if obj_type not in ['image', 'item', 'message']:
         raise Exception(f'Currently not supported Object Type: {obj_type}')
 
     modulename = args.module

update/v5.5/Update.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import os
+import sys
+
+sys.path.append(os.environ['AIL_HOME'])
+##################################
+# Import Project packages
+##################################
+from update.bin.ail_updater import AIL_Updater
+from lib import ail_updates
+from lib import chats_viewer
+
+class Updater(AIL_Updater):
+    """default Updater."""
+
+    def __init__(self, version):
+        super(Updater, self).__init__(version)
+
+
+if __name__ == '__main__':
+    chats_viewer.fix_correlations_subchannel_message()
+    updater = Updater('v5.5')
+    updater.run_update()
+

update/v5.5/Update.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
+
+export PATH=$AIL_HOME:$PATH
+export PATH=$AIL_REDIS:$PATH
+export PATH=$AIL_BIN:$PATH
+export PATH=$AIL_FLASK:$PATH
+
+GREEN="\\033[1;32m"
+DEFAULT="\\033[0;39m"
+
+echo -e $GREEN"Shutting down AIL ..."$DEFAULT
+bash ${AIL_BIN}/LAUNCH.sh -ks
+wait
+
+# SUBMODULES #
+git submodule update
+
+echo ""
+echo -e $GREEN"Updating python packages ..."$DEFAULT
+echo ""
+pip install -U easyocr
+
+
+bash ${AIL_BIN}/LAUNCH.sh -lrv
+bash ${AIL_BIN}/LAUNCH.sh -lkv
+
+echo ""
+echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT
+echo ""
+python ${AIL_HOME}/update/v5.5/Update.py
+wait
+echo ""
+echo ""
+
+exit 0

var/www/Flask_server.py

@@ -35,6 +35,7 @@ import Flask_config
 from blueprints.root import root
 from blueprints.crawler_splash import crawler_splash
 from blueprints.correlation import correlation
+from blueprints.languages_ui import languages_ui
 from blueprints.tags_ui import tags_ui
 from blueprints.import_export import import_export
 from blueprints.investigations_b import investigations_b
@@ -52,6 +53,7 @@ from blueprints.objects_etag import objects_etag
 from blueprints.objects_hhhash import objects_hhhash
 from blueprints.chats_explorer import chats_explorer
 from blueprints.objects_image import objects_image
+from blueprints.objects_ocr import objects_ocr
 from blueprints.objects_favicon import objects_favicon
 from blueprints.api_rest import api_rest
 
@@ -97,6 +99,7 @@ app.config['MAX_CONTENT_LENGTH'] = 900 * 1024 * 1024
 app.register_blueprint(root, url_prefix=baseUrl)
 app.register_blueprint(crawler_splash, url_prefix=baseUrl)
 app.register_blueprint(correlation, url_prefix=baseUrl)
+app.register_blueprint(languages_ui, url_prefix=baseUrl)
 app.register_blueprint(tags_ui, url_prefix=baseUrl)
 app.register_blueprint(import_export, url_prefix=baseUrl)
 app.register_blueprint(investigations_b, url_prefix=baseUrl)
@@ -114,6 +117,7 @@ app.register_blueprint(objects_etag, url_prefix=baseUrl)
 app.register_blueprint(objects_hhhash, url_prefix=baseUrl)
 app.register_blueprint(chats_explorer, url_prefix=baseUrl)
 app.register_blueprint(objects_image, url_prefix=baseUrl)
+app.register_blueprint(objects_ocr, url_prefix=baseUrl)
 app.register_blueprint(objects_favicon, url_prefix=baseUrl)
 app.register_blueprint(api_rest, url_prefix=baseUrl)
 

var/www/blueprints/chats_explorer.py

@@ -23,6 +23,7 @@ from lib import ail_core
 from lib import chats_viewer
 from lib import Language
 from lib import Tag
+from lib import module_extractor
 
 # ============ BLUEPRINT ============
 chats_explorer = Blueprint('chats_explorer', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/chats_explorer'))
@@ -235,6 +236,10 @@ def objects_message():
     else:
         message = message[0]
         languages = Language.get_translation_languages()
+        extracted = module_extractor.extract('message', '', message['id'], content=message['content'])
+        extracted_matches = module_extractor.get_extracted_by_match(extracted)
+        message['extracted'] = extracted
+        message['extracted_matches'] = extracted_matches
         return render_template('ChatMessage.html', meta=message, bootstrap_label=bootstrap_label,
                                translation_languages=languages, translation_target=target,
                                modal_add_tags=Tag.get_modal_add_tags(message['id'], object_type='message'))
@@ -291,3 +296,38 @@ def objects_user_account():
         return render_template('user_account.html', meta=user_account, bootstrap_label=bootstrap_label,
                                ail_tags=Tag.get_modal_add_tags(user_account['id'], user_account['type'], user_account['subtype']),
                                translation_languages=languages, translation_target=target)
+
+@chats_explorer.route("/objects/user-account/chat", methods=['GET'])
+@login_required
+@login_read_only
+def objects_user_account_chat():
+    instance_uuid = request.args.get('subtype')
+    user_id = request.args.get('id')
+    chat_id = request.args.get('chat_id')
+    target = request.args.get('target')
+    if target == "Don't Translate":
+        target = None
+    meta = chats_viewer.api_get_user_account_chat_messages(user_id, instance_uuid, chat_id, translation_target=target)
+    if meta[1] != 200:
+        return create_json_response(meta[0], meta[1])
+    else:
+        meta = meta[0]
+        languages = Language.get_translation_languages()
+        return render_template('chats_explorer/user_chat_messages.html', meta=meta, bootstrap_label=bootstrap_label,
+                               ail_tags=Tag.get_modal_add_tags(meta['user-account']['id'], meta['user-account']['type'], meta['user-account']['subtype']),
+                               translation_languages=languages, translation_target=target)
+
+@chats_explorer.route("objects/user-account/messages/stats/week/all", methods=['GET'])
+@login_required
+@login_read_only
+def user_account_messages_stats_week_all():
+    instance_uuid = request.args.get('subtype')
+    user_id = request.args.get('id')
+    week = chats_viewer.api_get_user_account_nb_all_week_messages(user_id, instance_uuid)
+    if week[1] != 200:
+        return create_json_response(week[0], week[1])
+    else:
+        return jsonify(week[0])
+
+
+

var/www/blueprints/correlation.py

@@ -87,61 +87,10 @@ def show_correlation():
 
         ## get all selected correlations
         filter_types = []
-        correl_option = request.form.get('CookieNameCheck')
-        if correl_option:
-            filter_types.append('cookie-name')
-        correl_option = request.form.get('EtagCheck')
-        if correl_option:
-            filter_types.append('etag')
-        correl_option = request.form.get('FaviconCheck')
-        if correl_option:
-            filter_types.append('favicon')
-        correl_option = request.form.get('CveCheck')
-        if correl_option:
-            filter_types.append('cve')
-        correl_option = request.form.get('CryptocurrencyCheck')
-        if correl_option:
-            filter_types.append('cryptocurrency')
-        correl_option = request.form.get('HHHashCheck')
-        if correl_option:
-            filter_types.append('hhhash')
-        correl_option = request.form.get('PgpCheck')
-        if correl_option:
-            filter_types.append('pgp')
-        correl_option = request.form.get('UsernameCheck')
-        if correl_option:
-            filter_types.append('username')
-        correl_option = request.form.get('DecodedCheck')
-        if correl_option:
-            filter_types.append('decoded')
-        correl_option = request.form.get('ScreenshotCheck')
-        if correl_option:
-            filter_types.append('screenshot')
-        # correlation_objects
-        correl_option = request.form.get('DomainCheck')
-        if correl_option:
-            filter_types.append('domain')
-        correl_option = request.form.get('ItemCheck')
-        if correl_option:
-            filter_types.append('item')
-        correl_option = request.form.get('chatCheck')
-        if correl_option:
-            filter_types.append('chat')
-        correl_option = request.form.get('subchannelCheck')
-        if correl_option:
-            filter_types.append('chat-subchannel')
-        correl_option = request.form.get('threadCheck')
-        if correl_option:
-            filter_types.append('chat-thread')
-        correl_option = request.form.get('messageCheck')
-        if correl_option:
-            filter_types.append('message')
-        correl_option = request.form.get('imageCheck')
-        if correl_option:
-            filter_types.append('image')
-        correl_option = request.form.get('user_accountCheck')
-        if correl_option:
-            filter_types.append('user-account')
+        for ob_type in ail_objects.get_all_objects():
+            correl_option = request.form.get(f'{ob_type}_Check')
+            if correl_option:
+                filter_types.append(ob_type)
 
         # list as params
         filter_types = ",".join(filter_types)
@@ -359,6 +308,10 @@ def show_relationship():
                 dict_object["metadata"]['type_id'] = subtype
             else:
                 dict_object["subtype"] = ''
-                dict_object["metadata_card"] = ail_objects.get_object_card_meta(obj_type, subtype, obj_id)
+            dict_object["metadata_card"] = ail_objects.get_object_card_meta(obj_type, subtype, obj_id)
+            dict_object["metadata_card"]['tags_safe'] = True
             return render_template("show_relationship.html", dict_object=dict_object, bootstrap_label=bootstrap_label,
-                                       tags_selector_data=Tag.get_tags_selector_data())
+                                   tags_selector_data=Tag.get_tags_selector_data(),
+                                   meta=dict_object["metadata_card"],
+                                   ail_tags=dict_object["metadata_card"]["add_tags_modal"])
+

var/www/blueprints/crawler_splash.py

@@ -316,6 +316,17 @@ def crawlers_last_domains_month_json():
     stats = crawlers.get_crawlers_stats_by_month(domain_type)
     return jsonify(stats)
 
+@crawler_splash.route('/crawlers/last/domains/month/previous/json')
+@login_required
+@login_read_only
+def crawlers_last_domains_previous_month_json():
+    domain_type = request.args.get('type')
+    if domain_type not in crawlers.get_crawler_all_types():
+        return jsonify({'error': 'Invalid domain type'}), 400
+    date = Date.get_previous_month_date()
+    stats = crawlers.get_crawlers_stats_by_month(domain_type, date=date)
+    return jsonify(stats)
+
 @crawler_splash.route('/crawlers/last/domains/status/month/json')
 @login_required
 @login_read_only

var/www/blueprints/hunters.py

@@ -24,6 +24,7 @@ sys.path.append(os.environ['AIL_BIN'])
 ##################################
 from lib import ail_core
 from lib.objects import ail_objects
+from lib import chats_viewer
 from lib import item_basic
 from lib import Tracker
 from lib import Tag
@@ -372,6 +373,78 @@ def get_json_tracker_graph():
         res = Tracker.get_trackers_graph_by_day([tracker_uuid])
     return jsonify(res)
 
+@hunters.route('/tracker/object/add', methods=['GET'])
+@login_required
+@login_admin
+def tracker_object_add():
+    user_id = current_user.get_id()
+    tracker_uuid = request.args.get('uuid')
+    object_global_id = request.args.get('gid')
+    if object_global_id.startswith('messages::'):
+        obj = ail_objects.get_obj_from_global_id(object_global_id)
+        date = obj.get_date()
+    else:
+        date = request.args.get('date')  # TODO check daterange
+    res = Tracker.api_tracker_add_object({'uuid': tracker_uuid, 'gid': object_global_id, 'date': date}, user_id)
+    if res[1] != 200:
+        return create_json_response(res[0], res[1])
+    else:
+        if request.referrer:
+            return redirect(request.referrer)
+        else:
+            return redirect(url_for('hunters.show_tracker', uuid=tracker_uuid))
+
+@hunters.route('/tracker/object/remove', methods=['GET'])
+@login_required
+@login_analyst
+def tracker_object_remove():
+    user_id = current_user.get_id()
+    tracker_uuid = request.args.get('uuid')
+    object_global_id = request.args.get('gid')
+    res = Tracker.api_tracker_remove_object({'uuid': tracker_uuid, 'gid': object_global_id}, user_id)
+    if res[1] != 200:
+        return create_json_response(res[0], res[1])
+    else:
+        if request.referrer:
+            return redirect(request.referrer)
+        else:
+            return redirect(url_for('hunters.show_tracker', uuid=tracker_uuid))
+
+
+@hunters.route('/tracker/objects', methods=['GET'])
+@login_required
+@login_admin
+def tracker_objects():
+    user_id = current_user.get_id()
+    tracker_uuid = request.args.get('uuid', None)
+    res = Tracker.api_is_allowed_to_edit_tracker(tracker_uuid, user_id)
+    if res[1] != 200:  # invalid access
+        return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
+
+    tracker = Tracker.Tracker(tracker_uuid)
+    meta = tracker.get_meta(options={'description', 'sparkline', 'tags', 'nb_objs'})
+    if meta['type'] == 'yara':
+        yara_rule_content = Tracker.get_yara_rule_content(meta['tracked'])
+    else:
+        yara_rule_content = None
+
+    chats, messages = chats_viewer.get_message_report(tracker.get_objs())
+
+    meta['date'] = Date.get_current_utc_full_time()
+
+    return render_template("messages_report.html", meta=meta, yara_rule_content=yara_rule_content,
+                           chats=chats, messages=messages, bootstrap_label=bootstrap_label)
+
+    # TODO
+
+    # Manual - Title
+    #        - Summary
+
+    # Messages table
+
+    # Timeline messages by chats - line
+    # pie charts NB messages all chats
+    # Barchart NB messages by days
 
 ####################
 #    RETRO HUNT    #

var/www/blueprints/languages_ui.py

@@ -0,0 +1,83 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+"""
+    Blueprint Flask: crawler splash endpoints: dashboard, onion crawler ...
+"""
+
+import os
+import sys
+import json
+
+from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, abort
+from flask_login import login_required, current_user
+
+# Import Role_Manager
+from Role_Manager import login_admin, login_analyst, login_read_only
+
+sys.path.append(os.environ['AIL_BIN'])
+##################################
+# Import Project packages
+##################################
+from lib import ail_core
+from lib import Language
+from lib import Tag
+from lib.objects import ail_objects
+
+# ============ BLUEPRINT ============
+languages_ui = Blueprint('languages_ui', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/chats_explorer'))
+
+# ============ VARIABLES ============
+# bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info']
+
+def create_json_response(data, status_code):
+    return Response(json.dumps(data, indent=2, sort_keys=True), mimetype='application/json'), status_code
+
+# ============ FUNCTIONS ============
+
+# ============= ROUTES ==============
+@languages_ui.route("/languages/object/translate", methods=['POST'])
+@login_required
+@login_read_only
+def translate_object():
+    obj_type = request.form.get('type')
+    subtype = request.form.get('subtype')
+    obj_id = request.form.get('id')
+    source = request.form.get('language_target')
+    target = request.form.get('target')
+    translation = request.form.get('translation')
+    if target == "Don't Translate":
+        target = None
+
+    resp = ail_objects.api_manually_translate(obj_type, subtype, obj_id, source, target, translation)
+    if resp[1] != 200:
+        return create_json_response(resp[0], resp[1])
+    else:
+        if request.referrer:
+            return redirect(request.referrer)
+        else:
+            if obj_type == 'ocr':
+                return redirect(url_for('objects_ocr.object_ocr', id=obj_id, target=target))  # TODO change to support all objects
+
+@languages_ui.route("/languages/object/detect/language", methods=['GET'])
+@login_required
+@login_read_only
+def detect_object_language():
+    obj_type = request.args.get('type')
+    subtype = request.args.get('subtype')
+    obj_id = request.args.get('id')
+    target = request.args.get('target')
+
+    resp = ail_objects.api_detect_language(obj_type, subtype, obj_id)
+    if resp[1] != 200:
+        return create_json_response(resp[0], resp[1])
+    else:
+        if request.referrer:
+            return redirect(request.referrer)
+        else:
+            if obj_type == 'ocr':
+                return redirect(url_for('objects_ocr.object_ocr', id=obj_id, target=target))  # TODO change to support all objects
+
+
+
+

var/www/blueprints/objects_item.py

@@ -85,7 +85,7 @@ def showItem():  # # TODO: support post
     else:
         meta['investigations'] = []
 
-    extracted = module_extractor.extract(item.id, content=meta['content'])
+    extracted = module_extractor.extract('item', '', item.id, content=meta['content'])
     extracted_matches = module_extractor.get_extracted_by_match(extracted)
 
     return render_template("show_item.html", bootstrap_label=bootstrap_label,

var/www/blueprints/objects_ocr.py

@@ -0,0 +1,117 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+'''
+    Blueprint Flask: crawler splash endpoints: dashboard, onion crawler ...
+'''
+
+import json
+import os
+import sys
+
+from io import BytesIO
+
+from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, abort, send_file, send_from_directory
+from flask_login import login_required, current_user
+
+# Import Role_Manager
+from Role_Manager import login_admin, login_analyst, login_read_only, no_cache
+
+sys.path.append(os.environ['AIL_BIN'])
+##################################
+# Import Project packages
+##################################
+from lib import Language
+from lib import Tag
+from lib.objects import Ocrs
+
+from packages import Date
+
+# ============ BLUEPRINT ============
+objects_ocr = Blueprint('objects_ocr', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/objects/ocr'))
+
+# ============ VARIABLES ============
+bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info']
+
+def create_json_response(data, status_code):
+    return Response(json.dumps(data, indent=2, sort_keys=True), mimetype='application/json'), status_code
+
+# ============ FUNCTIONS ============
+@objects_ocr.route('/ocr/<path:filename>')
+@login_required
+@login_read_only
+@no_cache
+def ocr_image(filename):
+    if not filename:
+        abort(404)
+    if not 64 <= len(filename) <= 70:
+        abort(404)
+    filename = filename.replace('/', '')
+    ocr = Ocrs.Ocr(filename)
+    return send_file(BytesIO(ocr.draw_bounding_boxs()), mimetype='image/png')
+
+
+@objects_ocr.route("/objects/ocrs", methods=['GET'])
+@login_required
+@login_read_only
+def objects_ocrs():
+    date_from = request.args.get('date_from')
+    date_to = request.args.get('date_to')
+    show_objects = request.args.get('show_objects')
+    date = Date.sanitise_date_range(date_from, date_to)
+    date_from = date['date_from']
+    date_to = date['date_to']
+
+    if show_objects:
+        dict_objects = Ocrs.Ocrs().api_get_meta_by_daterange(date_from, date_to)
+    else:
+        dict_objects = {}
+
+    return render_template("OcrDaterange.html", date_from=date_from, date_to=date_to,
+                           dict_objects=dict_objects, show_objects=show_objects)
+
+
+@objects_ocr.route("/objects/ocrs/post", methods=['POST'])
+@login_required
+@login_read_only
+def objects_ocrs_post():
+    date_from = request.form.get('date_from')
+    date_to = request.form.get('date_to')
+    show_objects = request.form.get('show_objects')
+    return redirect(url_for('objects_ocr.objects_ocrs', date_from=date_from, date_to=date_to, show_objects=show_objects))
+
+
+@objects_ocr.route("/objects/ocrs/range/json", methods=['GET'])
+@login_required
+@login_read_only
+def objects_ocrs_range_json():
+    date_from = request.args.get('date_from')
+    date_to = request.args.get('date_to')
+    date = Date.sanitise_date_range(date_from, date_to)
+    date_from = date['date_from']
+    date_to = date['date_to']
+    return jsonify(Ocrs.Ocrs().api_get_chart_nb_by_daterange(date_from, date_to))
+
+
+@objects_ocr.route("/objects/ocr", methods=['GET'])
+@login_required
+@login_read_only
+def object_ocr():
+    obj_id = request.args.get('id')
+    target = request.args.get('target')
+    if target == "Don't Translate":
+        target = None
+    meta = Ocrs.api_get_ocr(obj_id, target)
+    if meta[1] != 200:
+        return create_json_response(meta[0], meta[1])
+    else:
+        meta = meta[0]
+        languages = Language.get_translation_languages()
+        return render_template("ShowOcr.html", meta=meta,
+                               bootstrap_label=bootstrap_label,
+                               ail_tags=Tag.get_modal_add_tags(meta['id'], meta['type'], meta['subtype']),
+                               translation_languages=languages, translation_target=target)
+
+
+# ============= ROUTES ==============
+

var/www/blueprints/tags_ui.py

@@ -293,6 +293,24 @@ def tags_search_messages():
     dict_tagged['date'] = Date.sanitise_date_range('', '', separator='-')
     return render_template("tags/search_obj_by_tags.html", bootstrap_label=bootstrap_label, dict_tagged=dict_tagged)
 
+@tags_ui.route('/tag/search/image')
+@login_required
+@login_read_only
+def tags_search_images():
+    object_type = 'image'
+    dict_tagged = {"object_type": object_type, "object_name": object_type.title() + "s"}
+    dict_tagged['date'] = Date.sanitise_date_range('', '', separator='-')
+    return render_template("tags/search_obj_by_tags.html", bootstrap_label=bootstrap_label, dict_tagged=dict_tagged)
+
+@tags_ui.route('/tag/search/ocr')
+@login_required
+@login_read_only
+def tags_search_ocrs():
+    object_type = 'ocr'
+    dict_tagged = {"object_type": object_type, "object_name": object_type.title() + "s"}
+    dict_tagged['date'] = Date.sanitise_date_range('', '', separator='-')
+    return render_template("tags/search_obj_by_tags.html", bootstrap_label=bootstrap_label, dict_tagged=dict_tagged)
+
 @tags_ui.route('/tag/search/domain')
 @login_required
 @login_read_only

var/www/templates/chats_explorer/ChatMessage.html

@@ -14,8 +14,8 @@
 
   <!-- JS -->
 	<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
-	<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
-	<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
+    <script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
+    <script src="{{ url_for('static', filename='js/bootstrap4.min.js') }}"></script>
 	<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
 	<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
     <script src="{{ url_for('static', filename='js/tags.js') }}"></script>
@@ -134,6 +134,65 @@
                 </div>
 
 
+                {% if meta['extracted_matches'] %}
+                    <div id="accordion_extracted" class="mb-3 mx-3">
+                        <div class="card">
+                            <div class="card-header py-1" id="heading_extracted">
+                                <div class="row">
+                                    <div class="col-11">
+                                        <div class="mt-2">
+                                            <img id="misp-logo" src="{{ url_for('static', filename='image/ail-icon.png')}}" height="32"> Extracted&nbsp;&nbsp;
+                                            <div class="badge badge-warning">{{meta['extracted_matches']|length}}</div>
+                                        </div>
+                                    </div>
+                                    <div class="col-1">
+                                        <button class="btn btn-link btn-lg py-2 float-right rotate down" data-toggle="collapse" data-target="#collapse_extracted" aria-expanded="true" aria-controls="collapseDecoded">
+                                            <i class="fas fa-chevron-circle-down"></i>
+                                        </button>
+                                    </div>
+                                </div>
+                            </div>
+
+                            <div id="collapse_extracted" class="collapse" aria-labelledby="heading_extracted" data-parent="#accordion_extracted">
+                                <div class="card-body">
+                                    <table id="table_extracted" class="table table-striped">
+                                        <thead class="thead-dark">
+                                        <tr>
+                                            <th>Type</th>
+                                            <th>ID</th>
+                                            <th>Extracted</th>
+                                        </tr>
+                                        </thead>
+                                        <tbody>
+                                        {% for match in meta['extracted_matches'] %}
+                                            <tr>
+                                                <td>
+                                                    <svg height="26" width="26">
+                                                        <g class="nodes">
+                                                            <circle cx="13" cy="13" r="13" fill="{{ meta['extracted_matches'][match]['icon']['color'] }}"></circle>
+                                                            <text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ meta['extracted_matches'][match]['icon']['style'] }}" font-size="16px">{{ meta['extracted_matches'][match]['icon']['icon'] }}</text>
+                                                        </g>
+                                                    </svg>
+                                                    {{ meta['extracted_matches'][match]['subtype'] }}
+                                                </td>
+                                                <td>{{ meta['extracted_matches'][match]['id'] }}</td>
+                                                <td>
+                                                    {% for row in meta['extracted_matches'][match]['matches'] %}
+                                                        <a href="#{{ row[0] }}:{{row[1] }}">{{ row[2] }}</a><br>
+                                                    {% endfor %}
+                                                </td>
+                                            </tr>
+                                        {% endfor %}
+                                        </tbody>
+                                    </table>
+                                </div>
+                            </div>
+
+                        </div>
+                    </div>
+                {% endif %}
+
+
                 {% include 'objects/image/block_blur_img_slider.html' %}
                 {% with translate_url=url_for('chats_explorer.objects_message', id=meta['id']), obj_id=meta['id'] %}
                     {% include 'chats_explorer/block_translation.html' %}
@@ -166,6 +225,9 @@
     $(document).ready(function(){
         $("#page-Decoded").addClass("active");
         $("#nav_chat").addClass("active");
+        $('[data-toggle="popover"]').popover({
+            boundary:'window',
+        });
 
   });
 

var/www/templates/chats_explorer/basic_card_chat.html

@@ -13,48 +13,88 @@
                     <text x="15" y="15" text-anchor="middle" dominant-baseline="central" class="{{ meta["svg_icon"]["style"] }}" font-size="16px">{{ meta["svg_icon"]["icon"] }}</text>
                 </g>
             </svg>
-            {% if meta['username'] %}{{ meta["username"]["id"] }} {% else %} {{ meta['name'] }}{% endif %} :
+            {% if meta['name'] %}{{ meta['name'] }}{% endif %}{% if not report_mode %} :  <small><a href="{{ url_for('chats_explorer.chats_explorer_chat') }}?subtype={{ meta['subtype'] }}&id={{ meta['id'] }}">{{ meta['id'] }}</a></small>{% endif %}
         </h4>
     </div>
     <div class="card-body py-0">
-        <span class="">
-            {% if meta["tags_safe"] %}
-                {% if meta['icon'] %}
-                    <span><img src="{{ url_for('objects_image.image', filename=meta['icon'])}}" class="my-1" alt="{{ meta['id'] }}" width="200" height="200"></span>
-                {% endif %}
-            {% else %}
-                <span class="my-2 fa-stack fa-8x">
-                    <i class="fas fa-stack-1x fa-image"></i>
-                    <i class="fas fa-stack-2x fa-ban" style="color:Red"></i>
-                </span>
-            {% endif %}
-        </span>
 
-        <span>
-            <span class="badge badge-dark">
-                <span class="badge badge-info" style="font-size: 0.8rem;">
-                    <i class="fas fa-hourglass-start"></i>
-                </span>
-                {{meta["first_seen"]}}
-                <span class="badge badge-light mx-1" style="font-size: 1rem;">
-                    <i class="far fa-calendar-alt"></i>
-                </span>
-                {{meta["last_seen"]}}
-                <span class="badge badge-secondary" style="font-size: 0.8rem;">
-                    <i class="fas fa-hourglass-end"></i>
-                </span>
-            </span>
-            <span class="badge badge-dark">
-                <span class="badge badge-info" style="font-size: 0.8rem;">
-                        <i class="far fa-comments"></i>
+        <div class="d-flex align-items-center">
+            <div>
+                {% if meta["tags_safe"] %}
+                    {% if meta['icon'] %}
+                        <span><img src="{{ url_for('objects_image.image', filename=meta['icon'])}}" class="my-1" alt="{{ meta['id'] }}" width="200" height="200"></span>
+                    {% endif %}
+                {% else %}
+                    <span class="my-2 fa-stack fa-8x">
+                        <i class="fas fa-stack-1x fa-image"></i>
+                        <i class="fas fa-stack-2x fa-ban" style="color:Red"></i>
                     </span>
-                {{meta["nb_subchannels"]}}&nbsp;&nbsp;
-                <span class="badge badge-info" style="font-size: 0.8rem;">
-                    <i class="fas fa-user-circle"></i>
-                </span>
-                {{meta["nb_participants"]}}
-            </span>
-        </span>
+                {% endif %}
+            </div>
+
+
+            <div>
+                {% if meta['username'] %}
+                    <div class="mx-2">
+                        <svg height="30" width="30">
+                            <g class="nodes">
+                                <circle cx="15" cy="15" r="15" fill="{{ meta["username"]["icon"]["color"] }}"></circle>
+                                <text x="15" y="15" text-anchor="middle" dominant-baseline="central" class="{{ meta["username"]["icon"]["style"] }}" font-size="16px">{{ meta["username"]["icon"]["icon"] }}</text>
+                            </g>
+                        </svg>
+                        {{ meta['username']['id'] }}
+                    </div>
+                {% endif %}
+
+
+                <div class="badge badge-dark mx-2 my-1">
+                    <span class="badge badge-info" style="font-size: 0.8rem;">
+                        <i class="fas fa-hourglass-start"></i>
+                    </span>
+                    {{meta["first_seen"][0:4]}}-{{meta["first_seen"][4:6]}}-{{meta["first_seen"][6:8]}}
+                    <span class="badge badge-light mx-1" style="font-size: 1rem;">
+                        <i class="far fa-calendar-alt"></i>
+                    </span>
+                    {{meta["last_seen"][0:4]}}-{{meta["last_seen"][4:6]}}-{{meta["last_seen"][6:8]}}
+                    <span class="badge badge-secondary" style="font-size: 0.8rem;">
+                        <i class="fas fa-hourglass-end"></i>
+                    </span>
+                </div>
+{#                <div class="mx-2">#}
+{#                    <span class="badge badge-dark">#}
+{#                        <span class="badge badge-info" style="font-size: 0.8rem;">#}
+{#                            <i class="fas fa-calendar-plus"></i>#}
+{#                        </span>#}
+{#                        {{meta["created_at"]}}#}
+{#                    </span>#}
+{#                </div>#}
+                <div class="mx-2">
+                    <span class="badge badge-dark">
+                        <span class="badge badge-info" style="font-size: 0.8rem;">
+                            <i class="far fa-comments"></i> Subchannels
+                        </span>
+                        {{meta["nb_subchannels"]}}&nbsp;&nbsp;
+                        <span class="badge badge-info" style="font-size: 0.8rem;">
+                            <i class="fas fa-user-circle"></i> Participants
+                        </span>
+                        {{meta["nb_participants"]}}
+                    </span>
+                    {% if "nb_messages" in meta %}
+                        <span class="badge badge-dark">
+                            <span class="badge badge-info" style="font-size: 0.8rem;">
+                                <i class="fas fa-user-circle"></i>
+                                <i class="far fa-comment-dots"></i>
+                            </span>
+                            <a class="badge-primary px-1 py-0" href="{{ url_for('chats_explorer.objects_user_account_chat') }}?subtype={{ meta['subtype'] }}&id={{ main_obj_id }}&chat_id={{ meta['id'] }}">{{ meta["nb_messages"] }}&nbsp;&nbsp;</a>
+                        </span>
+                    {% endif %}
+                </div>
+            </div>
+        </div>
+
+        <div>
+            <pre class="my-0" style="white-space: pre-wrap;">{{ meta['info'] }}</pre>
+        </div>
 
         <div class="">
             {% for tag in meta['tags'] %}
@@ -62,6 +102,54 @@
             {%  endfor %}
         </div>
 
+        {% if meta['subchannels'] %}
+            <table id="tablesubchannels" class="table">
+                <thead class="bg-dark text-white">
+                <tr>
+                    <th>Name</th>
+                    <th>ID</th>
+                    <th>Created at</th>
+                    <th>First Seen</th>
+                    <th>Last Seen</th>
+                    <th>
+                        <i class="fas fa-user-circle"></i>
+                        <i class="fas fa-comment-dots"></i>
+                    </th>
+                </tr>
+                </thead>
+                <tbody>
+                {% for meta_s in meta["subchannels"] %}
+                    <tr>
+                        <td>
+                            <b>{{ meta_s['name'] }}</b>
+                            {% if meta_s['translation_name'] %}
+                                <div class="text-secondary">{{ meta_s['translation_name'] }}</div>
+                            {% endif %}
+                        </td>
+                        <td><a href="{{ url_for('chats_explorer.objects_subchannel_messages') }}?subtype={{ meta_s['subtype'] }}&id={{ meta_s['id'] }}">{{ meta_s['id'] }}</a></td>
+                        <td>{{ meta_s['created_at'] }}</td>
+                        <td>
+                            {% if meta_s['first_seen'] %}
+                                {{ meta_s['first_seen'][0:4] }}-{{ meta_s['first_seen'][4:6] }}-{{ meta_s['first_seen'][6:8] }}
+                            {% endif %}
+                        </td>
+                        <td>
+                            {% if meta_s['last_seen'] %}
+                                {{ meta_s['last_seen'][0:4] }}-{{ meta_s['last_seen'][4:6] }}-{{ meta_s['last_seen'][6:8] }}
+                            {% endif %}
+                        </td>
+                        <td>
+                            {{ meta_s['nb_messages'] }}
+                        </td>
+                    </tr>
+                {% endfor %}
+                </tbody>
+            </table>
+
+        {% endif %}
+
 	</div>
-    {% include 'objects/block_object_footer_small.html' %}
+    {% if not report_mode %}
+        {% include 'objects/block_object_footer_small.html' %}
+    {% endif %}
 </div>

var/www/templates/chats_explorer/block_message.html

@@ -22,15 +22,16 @@
     }
 </style>
 
-
-
 <div class="chat-message-left pb-1">
     <div>
         <a href="{{ url_for('chats_explorer.objects_user_account')}}?subtype={{ message['user-account']['subtype'] }}&id={{ message['user-account']['id'] }}">
             <img src="{% if message['user-account']['icon'] %}{{ url_for('objects_image.image', filename=message['user-account']['icon'])}}{% else %}{{ url_for('static', filename='image/ail-icon.png') }}{% endif %}"
-                 class="rounded-circle mr-1" alt="{{ message['user-account']['id'] }}" width="40" height="40">
+                 class="rounded-circle mr-1" alt="{{ message['user-account']['id'] }}" width="60" height="60">
         </a>
-        <div class="text-muted small text-nowrap mt-2">{{ message['hour'] }}</div>
+        <div class="text-center">
+            <div class="text-muted small text-nowrap">{{ message['date'] }}</div>
+            <div class="text-muted small text-nowrap" style="font-size: 90%">{{ message['hour'] }}</div>
+        </div>
     </div>
     <div class="flex-shrink-1 bg-light rounded py-2 px-3 ml-4 pb-4" style="overflow-x: auto">
         <div class="font-weight-bold mb-1">
@@ -66,7 +67,12 @@
         {% endif %}
         {% if message['images'] %}
             {% for message_image in message['images'] %}
-                <img class="object_image mb-1" src="{{ url_for('objects_image.image', filename=message_image)}}">
+                <img class="object_image mb-1" src="{{ url_for('objects_image.image', filename=message_image['id'])}}">
+                {% if message_image['ocr'] %}
+                    <span>
+                        <a class="btn btn-info" target="_blank" href="{{ url_for('objects_ocr.object_ocr', id=message_image['id'])}}"><i class="fas fa-expand"></i> OCR</a>
+                    </span>
+                {% endif %}
             {%  endfor %}
         {% endif %}
         {% if message['files-names'] %}
@@ -76,10 +82,14 @@
                 </div>
             {%  endfor %}
         {% endif %}
-        <pre class="my-0">{{ message['content'] }}</pre>
+        {% if not message['extracted'] %}
+            <pre class="my-0" style="white-space: pre-wrap;">{{ message['content'] }}</pre>
+        {% else %}
+            <pre class="my-0" style="white-space: pre-wrap;">{{ message['content'][:message['extracted'][0][0]] }}{% for row in message['extracted'] %}<span class="hg-text" data-toggle="popover" data-trigger="hover" data-html="true" title="Extracted:" data-content="<ul class=&quot;list-group&quot;>{% for r in row[3] %}<li class=&quot;list-group-item&quot;><div><svg height=&quot;26&quot; width=&quot;26&quot;><g class=&quot;nodes&quot;><circle cx=&quot;13&quot; cy=&quot;13&quot; r=&quot;13&quot; fill=&quot;{{ message['extracted_matches'][r[0]]['icon']['color'] }}&quot;></circle><text x=&quot;13&quot; y=&quot;13&quot; text-anchor=&quot;middle&quot; dominant-baseline=&quot;central&quot; class=&quot;{{ message['extracted_matches'][r[0]]['icon']['style'] }}&quot; font-size=&quot;16px&quot;>{{ message['extracted_matches'][r[0]]['icon']['icon'] }}</text></g></svg> {{ message['extracted_matches'][r[0]]['subtype'] }}</div>{{ message['extracted_matches'][r[0]]['id'] }} <div><b>{{ r[1] }}</b></div></li>{% endfor %}</ul>" id="{{ row[0] }}:{{ row[1] }}">{{ message['content'][row[0]:row[1]] }}</span>{% if loop.index + 1 > message['extracted']|length %}{{ message['content'][message['extracted'][-1][1]:] }}{% else %}{{ message['content'][row[1]:message['extracted'][loop.index][0]] }}{% endif %}{% endfor %}</pre>
+        {% endif %}
         {% if message['translation'] %}
             <hr class="m-1">
-            <pre class="my-0 text-secondary">{{ message['translation'] }}</pre>
+            <pre class="my-0 text-secondary" style="white-space: pre-wrap;">{{ message['translation'] }}</pre>
 
         {% endif %}
         {% for reaction in message['reactions'] %}

var/www/templates/chats_explorer/user_account.html

@@ -38,44 +38,19 @@
                     {% include 'chats_explorer/block_translation.html' %}
                 {% endwith %}
 
+                {% if meta['chats'] %}
+                    <h4 class="mx-5 mt-2 text-secondary">User All Messages:</h4>
+                    <div id="heatmapweekhourall"></div>
 
-{#                {% if meta['subchannels'] %}#}
-{#                    <h4>Sub-Channels:</h4>#}
-{#                    <table id="tablesubchannels" class="table">#}
-{#                        <thead class="bg-dark text-white">#}
-{#                        <tr>#}
-{#                            <th></th>#}
-{#                            <th></th>#}
-{#                            <th></th>#}
-{#                            <th></th>#}
-{#                        </tr>#}
-{#                        </thead>#}
-{#                        <tbody style="font-size: 15px;">#}
-{#                        {% for meta in meta["subchannels"] %}#}
-{#                            <tr>#}
-{#                                <td>#}
-{#                                   <img src="{{ url_for('static', filename='image/ail-icon.png') }}" class="rounded-circle mr-1" alt="{{ meta['id'] }}" width="40" height="40">#}
-{#                                </td>#}
-{#                                <td><b>{{ meta['name'] }}</b></td>#}
-{#                                <td><a href="{{ url_for('metas_explorer.objects_subchannel_messages') }}?uuid={{ meta['subtype'] }}&id={{ meta['id'] }}">{{ meta['id'] }}</a></td>#}
-{#                                <td>{{ meta['created_at'] }}</td>#}
-{#                                <td>#}
-{#                                    {% if meta['first_seen'] %}#}
-{#                                        {{ meta['first_seen'][0:4] }}-{{ meta['first_seen'][4:6] }}-{{ meta['first_seen'][6:8] }}#}
-{#                                    {% endif %}#}
-{#                                </td>#}
-{#                                <td>#}
-{#                                    {% if meta['last_seen'] %}#}
-{#                                        {{ meta['last_seen'][0:4] }}-{{ meta['last_seen'][4:6] }}-{{ meta['last_seen'][6:8] }}#}
-{#                                    {% endif %}#}
-{#                                </td>#}
-{#                                <td>{{ meta['nb_messages'] }}</td>#}
-{#                            </tr>#}
-{#                        {% endfor %}#}
-{#                        </tbody>#}
-{#                    </table>#}
-{##}
-{#                {% endif %}#}
+                    <h4>User Chats:</h4>
+                    {% for meta_chats in meta['chats'] %}
+                        <div class="my-2">
+                            {% with meta=meta_chats,main_obj_id=meta['id']  %}
+                                {% include 'chats_explorer/basic_card_chat.html' %}
+                            {% endwith %}
+                        </div>
+                    {% endfor %}
+                {% endif %}
 
 
             </div>
@@ -97,6 +72,11 @@
 {#        {% endif %}#}
   });
 
+d3.json("{{ url_for('chats_explorer.user_account_messages_stats_week_all') }}?subtype={{ meta['subtype'] }}&id={{ meta['id'] }}")
+    .then(function(data) {
+        create_heatmap_week_hour('#heatmapweekhourall', data);
+    })
+
 function toggle_sidebar(){
 	if($('#nav_menu').is(':visible')){
 		$('#nav_menu').hide();

var/www/templates/chats_explorer/user_chat_messages.html

@@ -0,0 +1,133 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+  <title>User Account - AIL</title>
+	<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
+
+  <!-- Core CSS -->
+	<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
+
+  <!-- JS -->
+	<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
+    <script src="{{ url_for('static', filename='js/d3.min.js')}}"></script>
+    <script src="{{ url_for('static', filename='js/d3/heatmap_week_hour.js')}}"></script>
+
+
+</head>
+<body>
+
+	{% include 'nav_bar.html' %}
+
+	<div class="container-fluid">
+		<div class="row">
+
+			{% include 'sidebars/sidebar_objects.html' %}
+
+			<div class="col-12 col-lg-10" id="core_content">
+
+                <h3>User:</h3>
+                {% with meta=meta['user-account'] %}
+                    {% include 'chats_explorer/card_user_account.html' %}
+                {% endwith %}
+
+                <h3>Chat:</h3>
+                {% with meta=meta['chat']  %}
+                    {% include 'chats_explorer/basic_card_chat.html' %}
+                {% endwith %}
+
+                <div class="mt-2">
+                {% with translate_url=url_for('chats_explorer.objects_user_account', subtype=meta['subtype']), obj_id=meta['id'] %}
+                    {% include 'chats_explorer/block_translation.html' %}
+                {% endwith %}
+                {% include 'objects/image/block_blur_img_slider.html' %}
+                </div>
+
+
+                <div class="position-relative">
+                    <div class="chat-messages p-2">
+
+                        {% for date in meta['messages'] %}
+
+                            <div class="divider d-flex align-items-center mb-4">
+                                <p class="text-center h2 mx-3 mb-0" style="color: #a2aab7;">
+                                    <span class="badge badge-secondary mb-2" id="date_section_{{ date }}">{{ date }}</span>
+                                </p>
+                            </div>
+
+                            {% for mess in meta['messages'][date] %}
+
+                                {% with message=mess %}
+                                    {% include 'chats_explorer/block_message.html' %}
+                                {% endwith %}
+
+                            {% endfor %}
+                            <br>
+                        {% endfor %}
+
+                    </div>
+                </div>
+
+
+
+{#                {% if meta['chats'] %}#}
+{#                    <h4 class="mx-5 mt-2 text-secondary">User All Messages:</h4>#}
+{#                    <div id="heatmapweekhourall"></div>#}
+{##}
+{#                    <h4>User Chats:</h4>#}
+{#                    {% for meta_chats in meta['chats'] %}#}
+{#                        <div class="my-2">#}
+{#                            {% with meta=meta_chats  %}#}
+{#                                {% include 'chats_explorer/basic_card_chat.html' %}#}
+{#                            {% endwith %}#}
+{#                        </div>#}
+{#                    {% endfor %}#}
+{#                {% endif %}#}
+
+
+            </div>
+
+        </div>
+    </div>
+
+<script>
+    $(document).ready(function(){
+        $("#page-Decoded").addClass("active");
+        $("#nav_chat").addClass("active");
+
+  });
+
+{#d3.json("{{ url_for('chats_explorer.user_account_messages_stats_week_all') }}?subtype={{ meta['subtype'] }}&id={{ meta['id'] }}")#}
+{#    .then(function(data) {#}
+{#        create_heatmap_week_hour('#heatmapweekhourall', data);#}
+{#    })#}
+
+function toggle_sidebar(){
+	if($('#nav_menu').is(':visible')){
+		$('#nav_menu').hide();
+		$('#side_menu').removeClass('border-right')
+		$('#side_menu').removeClass('col-lg-2')
+		$('#core_content').removeClass('col-lg-10')
+	}else{
+		$('#nav_menu').show();
+		$('#side_menu').addClass('border-right')
+		$('#side_menu').addClass('col-lg-2')
+		$('#core_content').addClass('col-lg-10')
+	}
+}
+
+</script>
+
+
+
+
+
+</body>
+
+</html>

var/www/templates/correlation/show_correlation.html

@@ -130,6 +130,8 @@
 						{% include 'correlation/metadata_card_hhhash.html' %}
                     {% elif dict_object["object_type"] == "image" %}
 						{% include 'chats_explorer/card_image.html' %}
+                    {% elif dict_object["object_type"] == "ocr" %}
+						{% include 'objects/ocr/card_ocr.html' %}
 					{% elif dict_object["object_type"] == "item" %}
 						{% include 'correlation/metadata_card_item.html' %}
                     {% elif dict_object["object_type"] == "favicon" %}
@@ -240,84 +242,92 @@
 
 
                                             <div class="form-check">
-                                                <input class="form-check-input" type="checkbox" value="True" id="CookieNameCheck" name="CookieNameCheck" {%if "cookie-name" in dict_object["filter"]%}checked{%endif%}>
-                                                <label class="form-check-label" for="CookieNameCheck">Cookie Name</label>
+                                                <input class="form-check-input" type="checkbox" value="True" id="cookie-name_Check" name="cookie-name_Check" {%if "cookie-name" in dict_object["filter"]%}checked{%endif%}>
+                                                <label class="form-check-label" for="cookie-name_Check">Cookie Name</label>
                                             </div>
                                             <div class="form-check">
-                                                <input class="form-check-input" type="checkbox" value="True" id="CveCheck" name="CveCheck" {%if "cve" in dict_object["filter"]%}checked{%endif%}>
-                                                <label class="form-check-label" for="CveCheck">Cve</label>
+                                                <input class="form-check-input" type="checkbox" value="True" id="cve_Check" name="cve_Check" {%if "cve" in dict_object["filter"]%}checked{%endif%}>
+                                                <label class="form-check-label" for="cve_Check">Cve</label>
                                             </div>
                                             <div class="form-check">
-                                                <input class="form-check-input" type="checkbox" value="True" id="CryptocurrencyCheck" name="CryptocurrencyCheck" {%if "cryptocurrency" in dict_object["filter"]%}checked{%endif%}>
-                                                <label class="form-check-label" for="CryptocurrencyCheck">Cryptocurrency</label>
+                                                <input class="form-check-input" type="checkbox" value="True" id="cryptocurrency_Check" name="cryptocurrency_Check" {%if "cryptocurrency" in dict_object["filter"]%}checked{%endif%}>
+                                                <label class="form-check-label" for="cryptocurrency_Check">Cryptocurrency</label>
                                             </div>
                                             <div class="form-check">
-                                                <input class="form-check-input" type="checkbox" value="True" id="DecodedCheck" name="DecodedCheck" {%if "decoded" in dict_object["filter"]%}checked{%endif%}>
-                                                <label class="form-check-label" for="DecodedCheck">Decoded</label>
+                                                <input class="form-check-input" type="checkbox" value="True" id="decoded_Check" name="decoded_Check" {%if "decoded" in dict_object["filter"]%}checked{%endif%}>
+                                                <label class="form-check-label" for="decoded_Check">Decoded</label>
                                             </div>
                                             <div class="form-check">
-                                                <input class="form-check-input" type="checkbox" value="True" id="EtagCheck" name="EtagCheck" {%if "etag" in dict_object["filter"]%}checked{%endif%}>
-                                                <label class="form-check-label" for="EtagCheck">Etag</label>
+                                                <input class="form-check-input" type="checkbox" value="True" id="etag_Check" name="EtagCheck" {%if "etag" in dict_object["filter"]%}checked{%endif%}>
+                                                <label class="form-check-label" for="etag_Check">Etag</label>
                                             </div>
                                             <div class="form-check">
-                                                <input class="form-check-input" type="checkbox" value="True" id="FaviconCheck" name="FaviconCheck" {%if "favicon" in dict_object["filter"]%}checked{%endif%}>
-                                                <label class="form-check-label" for="FaviconCheck">Favicon</label>
+                                                <input class="form-check-input" type="checkbox" value="True" id="favicon_Check" name="favicon_Check" {%if "favicon" in dict_object["filter"]%}checked{%endif%}>
+                                                <label class="form-check-label" for="favicon_Check">Favicon</label>
                                             </div>
                                             <div class="form-check">
-                                                <input class="form-check-input" type="checkbox" value="True" id="HHHashCheck" name="HHHashCheck" {%if "hhhash" in dict_object["filter"]%}checked{%endif%}>
-                                                <label class="form-check-label" for="HHHashCheck">HHHash</label>
+                                                <input class="form-check-input" type="checkbox" value="True" id="hhhash_Check" name="hhhash_Check" {%if "hhhash" in dict_object["filter"]%}checked{%endif%}>
+                                                <label class="form-check-label" for="hhhash_Check">HHHash</label>
                                             </div>
                                             <div class="form-check">
-                                                <input class="form-check-input" type="checkbox" value="True" id="ScreenshotCheck" name="ScreenshotCheck" {%if "screenshot" in dict_object["filter"]%}checked{%endif%}>
-                                                <label class="form-check-label" for="ScreenshotCheck">Screenshot</label>
+                                                <input class="form-check-input" type="checkbox" value="True" id="screenshot_Check" name="screenshot_Check" {%if "screenshot" in dict_object["filter"]%}checked{%endif%}>
+                                                <label class="form-check-label" for="screenshot_Check">Screenshot</label>
                                             </div>
                                             <div class="form-check">
-                                                <input class="form-check-input" type="checkbox" value="True" id="TitleCheck" name="TitleCheck" {%if "title" in dict_object["filter"]%}checked{%endif%}>
-                                                <label class="form-check-label" for="TitleCheck">Title</label>
+                                                <input class="form-check-input" type="checkbox" value="True" id="title_Check" name="title_Check" {%if "title" in dict_object["filter"]%}checked{%endif%}>
+                                                <label class="form-check-label" for="title_Check">Title</label>
                                             </div>
                                             <div class="form-check">
-                                                <input class="form-check-input" type="checkbox" value="True" id="PgpCheck" name="PgpCheck" {%if "pgp" in dict_object["filter"]%}checked{%endif%}>
-                                                <label class="form-check-label" for="PgpCheck">PGP</label>
+                                                <input class="form-check-input" type="checkbox" value="True" id="pgp_Check" name="pgp_Check" {%if "pgp" in dict_object["filter"]%}checked{%endif%}>
+                                                <label class="form-check-label" for="pgp_Check">PGP</label>
                                             </div>
                                             <div class="form-check">
-                                                <input class="form-check-input" type="checkbox" value="True" id="DomainCheck" name="DomainCheck" {%if "domain" in dict_object["filter"]%}checked{%endif%}>
-                                                <label class="form-check-label" for="DomainCheck">Domain</label>
+                                                <input class="form-check-input" type="checkbox" value="True" id="domain_Check" name="domain_Check" {%if "domain" in dict_object["filter"]%}checked{%endif%}>
+                                                <label class="form-check-label" for="domain_Check">Domain</label>
                                             </div>
                                             <div class="form-check">
-                                                <input class="form-check-input" type="checkbox" value="True" id="ItemCheck" name="ItemCheck" {%if "item" in dict_object["filter"]%}checked{%endif%}>
-                                                <label class="form-check-label" for="ItemCheck">Item</label>
+                                                <input class="form-check-input" type="checkbox" value="True" id="item_Check" name="item_Check" {%if "item" in dict_object["filter"]%}checked{%endif%}>
+                                                <label class="form-check-label" for="item_Check">Item</label>
                                             </div>
 
                                             <hr>
                                             <div class="form-check">
-                                                <input class="form-check-input" type="checkbox" value="True" id="chatCheck" name="chatCheck" {%if "chat" in dict_object["filter"]%}checked{%endif%}>
-                                                <label class="form-check-label" for="chatCheck">Chat</label>
+                                                <input class="form-check-input" type="checkbox" value="True" id="chat_Check" name="chat_Check" {%if "chat" in dict_object["filter"]%}checked{%endif%}>
+                                                <label class="form-check-label" for="chat_Check">Chat</label>
                                             </div>
                                             <div class="form-check">
-                                                <input class="form-check-input" type="checkbox" value="True" id="subchannelCheck" name="subchannelCheck" {%if "chat-subchannel" in dict_object["filter"]%}checked{%endif%}>
-                                                <label class="form-check-label" for="subchannelCheck">Chat-Subchannel</label>
+                                                <input class="form-check-input" type="checkbox" value="True" id="chat-subchannel_Check" name="chat-subchannel_Check" {%if "chat-subchannel" in dict_object["filter"]%}checked{%endif%}>
+                                                <label class="form-check-label" for="chat-subchannel_Check">Chat-Subchannel</label>
                                             </div>
                                             <div class="form-check">
-                                                <input class="form-check-input" type="checkbox" value="True" id="threadCheck" name="threadCheck" {%if "chat-thread" in dict_object["filter"]%}checked{%endif%}>
-                                                <label class="form-check-label" for="threadCheck">Chat-Thread</label>
+                                                <input class="form-check-input" type="checkbox" value="True" id="chat-thread_Check" name="chat-thread_Check" {%if "chat-thread" in dict_object["filter"]%}checked{%endif%}>
+                                                <label class="form-check-label" for="chat-thread_Check">Chat-Thread</label>
                                             </div>
                                             <div class="form-check">
-                                                <input class="form-check-input" type="checkbox" value="True" id="messageCheck" name="messageCheck" {%if "message" in dict_object["filter"]%}checked{%endif%}>
-                                                <label class="form-check-label" for="messageCheck">Message</label>
+                                                <input class="form-check-input" type="checkbox" value="True" id="message_Check" name="message_Check" {%if "message" in dict_object["filter"]%}checked{%endif%}>
+                                                <label class="form-check-label" for="message_Check">Message</label>
                                             </div>
                                             <div class="form-check">
-                                                <input class="form-check-input" type="checkbox" value="True" id="imageCheck" name="imageCheck" {%if "image" in dict_object["filter"]%}checked{%endif%}>
-                                                <label class="form-check-label" for="imageCheck">Image</label>
+                                                <input class="form-check-input" type="checkbox" value="True" id="file-name_Check" name="file-name_Check" {%if "file-name" in dict_object["filter"]%}checked{%endif%}>
+                                                <label class="form-check-label" for="file-name_Check">File Name</label>
+                                            </div>
+                                            <div class="form-check">
+                                                <input class="form-check-input" type="checkbox" value="True" id="image_Check" name="image_Check" {%if "image" in dict_object["filter"]%}checked{%endif%}>
+                                                <label class="form-check-label" for="image_Check">Image</label>
+                                            </div>
+                                            <div class="form-check">
+                                                <input class="form-check-input" type="checkbox" value="True" id="ocr_Check" name="ocr_Check" {%if "ocr" in dict_object["filter"]%}checked{%endif%}>
+                                                <label class="form-check-label" for="ocr_Check">OCR</label>
                                             </div>
 
                                             <hr>
                                             <div class="form-check">
-                                                <input class="form-check-input" type="checkbox" value="True" id="user_accountCheck" name="user_accountCheck" {%if "user-account" in dict_object["filter"]%}checked{%endif%}>
-                                                <label class="form-check-label" for="user_accountCheck">User-Account</label>
+                                                <input class="form-check-input" type="checkbox" value="True" id="user-account_Check" name="user-account_Check" {%if "user-account" in dict_object["filter"]%}checked{%endif%}>
+                                                <label class="form-check-label" for="user-account_Check">User-Account</label>
                                             </div>
                                             <div class="form-check">
-                                                <input class="form-check-input" type="checkbox" value="True" id="UsernameCheck" name="UsernameCheck" {%if "username" in dict_object["filter"]%}checked{%endif%}>
-                                                <label class="form-check-label" for="UsernameCheck">Username</label>
+                                                <input class="form-check-input" type="checkbox" value="True" id="username_Check" name="username_Check" {%if "username" in dict_object["filter"]%}checked{%endif%}>
+                                                <label class="form-check-label" for="username_Check">Username</label>
                                             </div>
 
                                     </li>
@@ -723,9 +733,14 @@ if (d.popover) {
 								desc = desc + "fa-times-circle\"></i>DOWN"
 							}
 							desc = desc + "</div></dd>"
-						} else if (key!="tags" && key!="id" && key!="img" && key!="icon" && key!="link" && key!="type") {
+						} else if (key!=="tags" && key!=="id" && key!=="img" && key!=="svg_icon" && key!=="icon" && key!=="link" && key!=="type") {
                             if (data[key]) {
-							    desc = desc + "<dt class=\"col-sm-3 px-0\">" + sanitize_text(key) + "</dt><dd class=\"col-sm-9 px-0\">" + sanitize_text(data[key]) + "</dd>"
+                                if ((key==="first_seen" || key==="last_seen") && data[key].length===8) {
+                                    let date = sanitize_text(data[key])
+                                    desc = desc + "<dt class=\"col-sm-3 px-0\">" + sanitize_text(key) + "</dt><dd class=\"col-sm-9 px-0\">" + date.slice(0, 4) + "-" + date.slice(4, 6) + "-" + date.slice(6, 8) + "</dd>"
+                                } else {
+							        desc = desc + "<dt class=\"col-sm-3 px-0\">" + sanitize_text(key) + "</dt><dd class=\"col-sm-9 px-0\">" + sanitize_text(data[key]) + "</dd>"
+                                }
                             }
 						}
 					});

var/www/templates/crawler/crawler_splash/last_crawled.html

@@ -95,6 +95,10 @@
                 <div id="barchart_type_month"></div>
                 <div class="text-center" id="pie_chart_month"></div>
 
+                <h3>Previous Month Stats:</h3>
+                <div id="barchart_type_month"></div>
+                <div class="text-center" id="barchart_type_previous_month"></div>
+
 			</div>
 
 		</div>
@@ -156,6 +160,13 @@ $(document).ready(function(){
         }
     );
 
+    $.getJSON("{{ url_for('crawler_splash.crawlers_last_domains_previous_month_json') }}?type={{type}}",
+        function (data) {
+                let div_width = $("#barchart_type_previous_month").width();
+                barchart_stack("barchart_type_previous_month", data, {"width": div_width});
+        }
+    );
+
 });
 
 function toggle_sidebar(){

var/www/templates/domains/card_img_domain.html

@@ -55,7 +55,7 @@
                     <span class="badge badge-light mx-1" style="font-size: 1rem;">
                         <i class="far fa-calendar-alt"></i>
                     </span>
-                    {{dict_domain["last_seen"]}}
+                    {{dict_domain["last_seen"]}}{{dict_domain["last_check"]}}
                     <span class="badge badge-secondary" style="font-size: 0.8rem;">
                         <i class="fas fa-hourglass-end"></i>
                     </span>

var/www/templates/hunter/messages_report.html

@@ -0,0 +1,274 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+  <title>Report - AIL</title>
+	<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
+
+  <!-- Core CSS -->
+	<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
+
+  <!-- JS -->
+	<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
+    <script src="{{ url_for('static', filename='js/d3.min.js')}}"></script>
+    <script src="{{ url_for('static', filename='js/d3/heatmap_week_hour.js')}}"></script>
+    <script src="{{ url_for('static', filename='js/d3/sparklines.js') }}"></script>
+
+    <style>
+        .chat-message-left,
+        .chat-message-right {
+            display: flex;
+            flex-shrink: 0;
+        }
+        .chat-message-right {
+            flex-direction: row-reverse;
+            margin-left: auto
+        }
+        .divider:after,
+        .divider:before {
+            content: "";
+            flex: 1;
+            height: 2px;
+            background: #eee;
+        }
+        .object_image {
+            max-width: 50%;
+        }
+	</style>
+
+</head>
+<body>
+    {% include 'nav_bar.html' %}
+
+    <div class="container-fluid">
+        <div class="row">
+
+            <div class="col-12">
+
+            <h1>Tracker Report:</h1>
+
+            <div class="row">
+                <div class="col-8">
+
+                    <div class="card my-2">
+                        <div class="card-header bg-dark text-white">
+                            <span class="badge badge-light lex-row-reverse float-right">
+                                <span id="sparkline"></span>
+                            </span>
+                            <h4 class="card-title">
+                                {% if meta['description'] %}
+                                    {{ meta['description'] }}
+                                {% endif %}
+                            </h4>
+                        </div>
+                        <div class="card-body bg-light pt-2">
+                            <table class="table table-borderless">
+                                <tbody>
+                                <tr>
+                                    <td class="text-right"><b>Type</b></td>
+                                    <td>
+                                        {% if meta['type'] == 'word' %}
+                                            <i class="fas fa-font"></i>&nbsp;
+                                        {% elif meta['type'] == 'set' %}
+                                            <i class="fas fa-layer-group"></i>&nbsp;
+                                        {% elif meta['type'] == 'regex' %}
+                                            <i class="fas fa-compass"></i>&nbsp;
+                                        {% elif meta['type'] == 'typosquatting' %}
+                                            <i class="fas fa-clone"></i>&nbsp;
+                                        {% elif meta['type'] == 'yara' %}
+                                            <span class="bg-danger text-white font-weight-bold" style="font-size: 120%">&nbsp;{ </span>&nbsp;
+                                        {% endif %}
+                                        {{ meta['type'] }}
+                                    </td>
+                                </tr>
+                                <tr>
+                                    <td class="text-right"><b>Generation Date</b></td>
+                                    <td>
+                                        <b>{{meta['date']}}</b>
+                                    </td>
+                                </tr>
+                                <tr>
+                                    <td class="text-right"><b>First Seen &nbsp;<i class="fas fa-hourglass-start"></i></b></td>
+                                    <td>
+                                        {% if meta['first_seen'] %}
+                                            {{ meta['first_seen'][0:4] }} - {{ meta['first_seen'][4:6] }} - {{ meta['first_seen'][6:8] }}
+                                        {% endif %}
+                                    </td>
+                                </tr>
+                                <tr>
+                                    <td class="text-right"><b>Last Seen &nbsp;<i class="fas fa-hourglass-end"></i></b></td>
+                                    <td>
+                                        {% if meta['last_seen'] %}
+                                            {{ meta['last_seen'][0:4] }} - {{ meta['last_seen'][4:6] }} - {{ meta['last_seen'][6:8] }}
+                                        {% endif %}
+                                    </td>
+                                </tr>
+                                <tr>
+                                    <td class="text-right"><b>Tags</b></td>
+                                    <td>
+                                        {%for tag in meta['tags']%}
+                                            <span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
+                                        {%endfor%}
+                                    </td>
+                                </tr>
+
+
+
+                                <tr>
+                                    <td class="text-right"><b>Objects Match</b></td>
+                                    <td>
+                                        {%for obj_type in meta['nb_objs']%}
+                                            <h4><span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">
+                                                {{ obj_type }}
+                                                <span class="badge badge-light">{{ meta['nb_objs'][obj_type] }}</span>
+                                            </span></h4>
+                                        {%endfor%}
+                                    </td>
+                                </tr>
+                                </tbody>
+                                {% if meta['type'] != 'yara' %}
+                                    <tr>
+                                        <td class="text-right"><b>Tracker</b></td>
+                                        <td>
+                                            {{ meta['tracked'] }}
+                                        </td>
+                                    </tr>
+                                {% endif %}
+                            </table>
+
+                            {% if yara_rule_content %}
+                                <h5 class="mb-0">Yara Rule:</h5>
+                                <p class="my-0">
+                                    <pre class="border bg-white" style="white-space: pre-wrap;">{{ yara_rule_content }}</pre>
+                                </p>
+                            {% endif %}
+
+                        </div>
+                    </div>
+
+                </div>
+                <div class="col-4">
+                    <div>
+                        <img src="{{ url_for('static', filename='image/ail-project.png') }}" width="200">
+                    </div>
+                    <div>
+                        <img src="https://circl.lu/assets/images/circl-logo.png" width="200">
+                    </div>
+                </div>
+            </div>
+
+            <h3>Messages:</h3>
+            {% for message in messages %}
+                <div class="d-flex justify-content-between py-2 px-3 border-top" style="background-color: rgba(0,0,0,.03)">
+                    <div>
+                        <svg height="30" width="30">
+                            <g class="nodes">
+                                <circle cx="15" cy="15" r="15" fill="{{ chats[message['chat']]["svg_icon"]["color"] }}"></circle>
+                                <text x="15" y="15" text-anchor="middle" dominant-baseline="central" class="{{ chats[message['chat']]["svg_icon"]["style"] }}" font-size="16px">{{ chats[message['chat']]["svg_icon"]["icon"] }}</text>
+                            </g>
+                        </svg>
+                        {% if chats[message['chat']]['name'] %}{{ chats[message['chat']]['name'] }}{% endif %}
+                    </div>
+                    <div>
+                        {% if chats[message['chat']]['origin_link'] %}
+                            <span class="flex-row-reverse">{{ chats[message['chat']]['origin_link'] }}</span>
+                        {% endif %}
+                    </div>
+                </div>
+
+                {% with message=message,show_full_message=True %}
+                    {% include 'chats_explorer/block_message.html' %}
+                {% endwith %}
+
+            {% endfor %}
+
+            <h3 class="mt-4">Chats Metadata:</h3>
+            {% for chat in chats %}
+                <div class="my-2">
+                    {% with meta=chats[chat],report_mode=True %}
+                        {% include 'chats_explorer/basic_card_chat.html' %}
+                    {% endwith %}
+                </div>
+            {% endfor %}
+
+            </div>
+        </div>
+    </div>
+
+{#                <h5 class="mx-5 mt-2 text-secondary">All Messages:</h5>#}
+{#                <div id="heatmapweekhourall"></div>#}
+
+{#                {% with translate_url=url_for('chats_explorer.chats_explorer_chat', subtype=chat['subtype']), obj_id=chat['id'], pagination=chat['pagination'] %}#}
+{#                    {% include 'chats_explorer/block_translation.html' %}#}
+{#                {% endwith %}#}
+
+{#                {% if chat['messages'] %}#}
+{##}
+{#                    <div class="position-relative">#}
+{#                        <div class="chat-messages p-2">#}
+{##}
+{#                            {% for date in chat['messages'] %}#}
+{##}
+{#                                <div class="divider d-flex align-items-center mb-4">#}
+{#                                    <p class="text-center h2 mx-3 mb-0" style="color: #a2aab7;">#}
+{#                                        <span class="badge badge-secondary mb-2" id="date_section_{{ date }}">{{ date }}</span>#}
+{#                                    </p>#}
+{#                                </div>#}
+{##}
+{#                                {% for mess in chat['messages'][date] %}#}
+{##}
+{#                                    {% with message=mess %}#}
+{#                                        {% include 'chats_explorer/block_message.html' %}#}
+{#                                    {% endwith %}#}
+{##}
+{#                                {% endfor %}#}
+{#                                <br>#}
+{#                            {% endfor %}#}
+{##}
+{#                        </div>#}
+{#                    </div>#}
+{##}
+{#                {% endif %}#}
+
+
+
+<script>
+    $(document).ready(function(){
+        $("#page-Decoded").addClass("active");
+        $("#nav_chat").addClass("active");
+
+        // unblur images
+        let images = document.getElementsByClassName('object_image');
+        for(i = 0; i < images.length; i++) {
+            images[i].style.filter = "blur(0px)";
+        }
+        sparkline("sparkline", {{ meta['sparkline'] }}, {});
+  });
+
+</script>
+
+
+{#<script>#}
+{#    d3.json("{{ url_for('chats_explorer.chats_explorer_messages_stats_week_all') }}?type=chat&subtype={{ chat['subtype'] }}&id={{ chat['id'] }}")#}
+{#        .then(function(data) {#}
+{#            create_heatmap_week_hour('#heatmapweekhourall', data);#}
+{#        })#}
+{##}
+{#{% if not chat['subchannels'] %}#}
+{#d3.json("{{ url_for('chats_explorer.chats_explorer_messages_stats_week') }}?type=chat&subtype={{ chat['subtype'] }}&id={{ chat['id'] }}")#}
+{#.then(function(data) {#}
+{#    create_heatmap_week_hour('#heatmapweekhour', data);#}
+{#})#}
+{#{% endif %}#}
+{#</script>#}
+
+
+</body>
+
+</html>

var/www/templates/hunter/tracker_add.html

@@ -140,6 +140,10 @@
                                                         <input class="custom-control-input" type="checkbox" name="message_obj" id="message_obj" checked="">
                                                         <label class="custom-control-label" for="message_obj"><i class="fas fa-comment-dots"></i>&nbsp;Message <i class="fas fa-info-circle text-info" data-toggle="tooltip" data-placement="right" title="Messages from Chats"></i></label>
                                                     </div>
+                                                    <div class="custom-control custom-switch mt-1">
+                                                        <input class="custom-control-input" type="checkbox" name="ocr_obj" id="ocr_obj" checked="">
+                                                        <label class="custom-control-label" for="ocr_obj"><i class="fas fa-comment-dots"></i>&nbsp;OCR <i class="fas fa-expand text-info" data-toggle="tooltip" data-placement="right" title="Text extracted from Images"></i></label>
+                                                    </div>
 
 {#                                                    <div class="custom-control custom-switch mt-1">#}
 {#                                                        <input class="custom-control-input" type="checkbox" name="level" id="screenshot_obj" checked="">#}

var/www/templates/hunter/tracker_show.html

@@ -44,6 +44,8 @@
 
 {% include 'nav_bar.html' %}
 
+{% include 'modals/tracker_remove_object.html' %}
+
 <div class="container-fluid">
     <div class="row">
         {% include 'hunter/menu_sidebar.html' %}
@@ -295,6 +297,11 @@
 									{% endif %}
 								</td>
 								<td class="text-right">
+                                    <button class="btn btn-outline-danger py-0 px-1"
+                                            data-toggle="modal" data-target="#modal_tracker_remove_object"
+                                            data-tracker_uuid="{{ meta['uuid'] }}" data-obj_gid="{{ object['type'] }}:{{ object['subtype'] }}:{{ object['id'] }}">
+                                            <i class="fas fa-trash-alt"></i></button>
+                                    </button>
 {#									<a href="{{ url_for('investigations_b.unregister_investigation') }}?uuid={{ meta['uuid']}}&type={{ object['type'] }}&subtype={{ object['subtype']}}&id={{ object['id']}}">#}
 {#										<button type="button" class="btn btn-danger"><i class="fas fa-trash-alt"></i></button>#}
 {#									</a>#}

var/www/templates/modals/tracker_remove_object.html

@@ -0,0 +1,39 @@
+<div id="modal_tracker_remove_object" class="modal fade" role="dialog">
+  <div class="modal-dialog modal-lg">
+
+    <div id="modal_tracker_remove_object_content" class="modal-content">
+      <div class="modal-header" style="border-bottom: 4px solid #cccccc; background-color: #cccccc; color: #ffffff;">
+        <h4>Remove Object From Tracker</h4>
+        <button type="button" class="close" data-dismiss="modal" aria-label="Close">
+          <span aria-hidden="true">&times;</span>
+        </button>
+      </div>
+
+      <div class="modal-body text-center">
+        <h2><span class="badge-warning" id="modal_tracker_remove_object_gid">Warning</span></h2>
+      </div>
+
+      <div class="modal-footer">
+
+        <button type="button" class="btn btn-default" data-dismiss="modal" >Close</button>
+
+        <a href="#" class="btn btn-danger" id="modal_tracker_remove_object_delete">
+          <i class="fas fa-trash-alt"></i> Remove
+        </a>
+
+      </div>
+    </div>
+  </div>
+</div>
+
+<script>
+// tagid + objtype + objid
+$('#modal_tracker_remove_object').on('show.bs.modal', function (event) {
+    var button = $(event.relatedTarget);
+    let obj_gid = button.data('obj_gid')
+    let tracker_uuid = button.data('tracker_uuid')
+    let modal = $(this)
+    modal.find('#modal_tracker_remove_object_gid').text(obj_gid)
+    modal.find('#modal_tracker_remove_object_delete').prop("href", "{{ url_for('hunters.tracker_object_remove') }}?uuid="+ tracker_uuid +"&gid="+ obj_gid);
+  })
+</script>

var/www/templates/objects/item/show_item.html

@@ -547,7 +547,7 @@
             {% if not extracted %}
                 <p class="my-0"> <pre class="border">{{ meta['content'] }}</pre></p>
             {% else %}
-                <p class="my-0"> <pre class="border">{{ meta['content'][:extracted[0][0]] }}{% for row in extracted %}<span class="hg-text" data-toggle="popover" data-trigger="hover" data-html="true" title="<svg height=&quot;26&quot; width=&quot;26&quot;><g class=&quot;nodes&quot;><circle cx=&quot;13&quot; cy=&quot;13&quot; r=&quot;13&quot; fill=&quot;{{ extracted_matches[row[3]]['icon']['color'] }}&quot;></circle><text x=&quot;13&quot; y=&quot;13&quot; text-anchor=&quot;middle&quot; dominant-baseline=&quot;central&quot; class=&quot;graph_node_icon {{ extracted_matches[row[3]]['icon']['style'] }}&quot; font-size=&quot;16px&quot;>{{ extracted_matches[row[3]]['icon']['icon'] }}</text></g></svg> {{ extracted_matches[row[3]]['subtype'] }}" data-content="{{ extracted_matches[row[3]]['id'] }}" id="{{ row[0] }}:{{ row[1] }}">{{ meta['content'][row[0]:row[1]] }}</span>{% if loop.index + 1 > extracted|length %}{{ meta['content'][extracted[-1][1]:] }}{% else %}{{ meta['content'][row[1]:extracted[loop.index][0]] }}{% endif %}{% endfor %}</pre></p>
+                <p class="my-0"> <pre class="border">{{ meta['content'][:extracted[0][0]] }}{% for row in extracted %}<span class="hg-text" data-toggle="popover" data-trigger="hover" data-html="true" title="Extracted:" data-content="<ul class=&quot;list-group&quot;>{% for r in row[3] %}<li class=&quot;list-group-item&quot;><div><svg height=&quot;26&quot; width=&quot;26&quot;><g class=&quot;nodes&quot;><circle cx=&quot;13&quot; cy=&quot;13&quot; r=&quot;13&quot; fill=&quot;{{ extracted_matches[r[0]]['icon']['color'] }}&quot;></circle><text x=&quot;13&quot; y=&quot;13&quot; text-anchor=&quot;middle&quot; dominant-baseline=&quot;central&quot; class=&quot;{{ extracted_matches[r[0]]['icon']['style'] }}&quot; font-size=&quot;16px&quot;>{{ extracted_matches[r[0]]['icon']['icon'] }}</text></g></svg> {{ extracted_matches[r[0]]['subtype'] }}</div>{{ extracted_matches[r[0]]['id'] }} <div><b>{{ r[1] }}</b></div></li>{% endfor %}</ul>" id="{{ row[0] }}:{{ row[1] }}">{{ meta['content'][row[0]:row[1]] }}</span>{% if loop.index + 1 > extracted|length %}{{ meta['content'][extracted[-1][1]:] }}{% else %}{{ meta['content'][row[1]:extracted[loop.index][0]] }}{% endif %}{% endfor %}</pre></p>
             {% endif %}
         </div>
         <div class="tab-pane fade" id="pills-html2text" role="tabpanel" aria-labelledby="pills-html2text-tab">
@@ -576,7 +576,10 @@
         $(".rotate").click(function(){
             $(this).toggleClass("down");
         })
-      });
+        $('[data-toggle="popover"]').popover({
+            boundary:'window',
+        })
+    });
 
       $('#pills-html2text-tab').on('shown.bs.tab', function (e) {
         if ($('#html2text-container').is(':empty')){
@@ -639,9 +642,6 @@
   }
 
   blocks.addEventListener('change', pixelate, false);
-  $(function () {
-    $('[data-toggle="popover"]').popover()
-  })
   </script>
 {% endif %}
 

var/www/templates/objects/ocr/OcrDaterange.html

@@ -0,0 +1,602 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+  <title>Ocrs - AIL</title>
+	<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
+
+  <!-- Core CSS -->
+	<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet">
+
+  <!-- JS -->
+	<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
+	<script src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>
+	<script src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
+    <script src="{{ url_for('static', filename='js/d3/sparklines.js')}}"></script>
+
+	<style>
+		.input-group .form-control {
+			position: unset;
+		}
+		.line {
+  		fill: none;
+  		stroke: #000;
+  		stroke-width: 2.0px;
+		}
+		.bar {
+      fill: steelblue;
+    }
+    .bar:hover{
+      fill: brown;
+			cursor: pointer;
+    }
+		.bar_stack:hover{
+			cursor: pointer;
+    }
+		.pie_path:hover{
+			cursor: pointer;
+		}
+		.svgText {
+  		pointer-events: none;
+		}
+		div.tooltip {
+			position: absolute;
+			text-align: center;
+			padding: 2px;
+			font: 12px sans-serif;
+			background: #ebf4fb;
+			border: 2px solid #b7ddf2;
+			border-radius: 8px;
+			pointer-events: none;
+			color: #000000;
+		}
+	</style>
+</head>
+<body>
+
+	{% include 'nav_bar.html' %}
+
+	<div class="container-fluid">
+		<div class="row">
+
+			{% include 'sidebars/sidebar_objects.html' %}
+
+			<div class="col-12 col-lg-10" id="core_content">
+
+				<div class="row">
+					<div class="col-xl-10">
+						<div class="mt-1" id="barchart_type"></div>
+
+{#                        {% include 'image/block_images_search.html' %}#}
+
+                    </div>
+
+
+					<div class="col-xl-2">
+
+						<div class="card mb-3 mt-2" style="background-color:#d9edf7;">
+						  <div class="card-body text-center py-2">
+						    <h6 class="card-title" style="color:#286090;">Select a date range :</h6>
+								<form action="{{ url_for('objects_ocr.objects_ocrs_post') }}" id="hash_selector_form" method='post'>
+									<div class="input-group" id="date-range-from">
+										<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
+										<input class="form-control" id="date-range-from-input" placeholder="yyyy-mm-dd" value="{{ date_from }}" name="date_from" autocomplete="off">
+									</div>
+									<div class="input-group" id="date-range-to">
+										<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
+										<input class="form-control" id="date-range-to-input" placeholder="yyyy-mm-dd" value="{{ date_to }}" name="date_to" autocomplete="off">
+									</div>
+									<div class="form-check my-1">
+										<input class="form-check-input" type="checkbox" id="checkbox-input-show" name="show_objects" value="True" {% if show_objects %}checked{% endif %}>
+										<label class="form-check-label" for="checkbox-input-show">
+											<span style="color:#286090; font-size: 14px;">
+												Show Ocrs <i class="fas fa-key"></i>
+											</span>
+										</label>
+									</div>
+									<button class="btn btn-primary" style="text-align:center;">
+										<i class="fas fa-copy"></i> Search
+									</button>
+								</form>
+						  </div>
+						</div>
+
+						<div id="pie_chart_encoded">
+						</div>
+						<div id="pie_chart_top5_types">
+						</div>
+					</div>
+				</div>
+
+				{% if dict_objects %}
+                    {% if date_from|string == date_to|string %}
+                        <h3> {{ date_from }} Ocrs Name: </h3>
+                    {% else %}
+                        <h3> {{ date_from }} to {{ date_to }} Ocrs Name: </h3>
+                    {% endif %}
+                    <table id="tableb64" class="table table-striped table-bordered">
+                        <thead class="bg-dark text-white">
+                        <tr>
+                            <th></th>
+                            <th>First Seen</th>
+                            <th>Last Seen</th>
+                            <th>Total</th>
+                            <th>Last days</th>
+                        </tr>
+                        </thead>
+						<tbody style="font-size: 15px;">
+                        {% for obj_id in dict_objects %}
+                            <tr>
+                                <td><a target="_blank" href="{{ url_for('correlation.show_correlation') }}?type=ocr&id={{ obj_id }}">{{ dict_objects[obj_id]['id'] }}</a></td>
+                                <td>{{ dict_objects[obj_id]['first_seen'] }}</td>
+                                <td>{{ dict_objects[obj_id]['last_seen'] }}</td>
+                                <td>{{ dict_objects[obj_id]['nb_seen'] }}</td>
+                                <td id="sparklines_{{ obj_id }}" style="text-align:center;"></td>
+                            </tr>
+                        {% endfor %}
+                        </tbody>
+                    </table>
+
+
+                {% else %}
+                    {% if show_objects %}
+                        {% if date_from|string == date_to|string %}
+							<h3> {{ date_from }}, No OCR</h3>
+                        {% else %}
+							<h3> {{ date_from }} to {{ date_to }}, No OCR</h3>
+                        {% endif %}
+                    {% endif %}
+                {% endif %}
+            </div>
+
+        </div>
+    </div>
+
+<script>
+    var chart = {};
+    $(document).ready(function(){
+        $("#page-Decoded").addClass("active");
+        $("#nav_ocr").addClass("active");
+
+        $('#date-range-from').dateRangePicker({
+            separator : ' to ',
+            getValue: function()
+            {
+                if ($('#date-range-from-input').val() && $('#date-range-to').val() )
+                    return $('#date-range-from-input').val() + ' to ' + $('#date-range-to').val();
+                else
+                    return '';
+            },
+            setValue: function(s,s1,s2)
+            {
+                $('#date-range-from-input').val(s1);
+                $('#date-range-to-input').val(s2);
+            },
+        });
+        $('#date-range-to').dateRangePicker({
+            separator : ' to ',
+            getValue: function()
+            {
+                if ($('#date-range-from-input').val() && $('#date-range-to').val() )
+                    return $('#date-range-from-input').val() + ' to ' + $('#date-range-to').val();
+                else
+                    return '';
+            },
+            setValue: function(s,s1,s2)
+            {
+                $('#date-range-from-input').val(s1);
+                $('#date-range-to-input').val(s2);
+            },
+        });
+
+        $('#date-range-from').data('dateRangePicker').setDateRange('{{date_from}}','{{date_to}}');
+        $('#date-range-to').data('dateRangePicker').setDateRange('{{date_from}}','{{date_to}}');
+
+        $('#tableb64').DataTable({
+            "aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
+            "iDisplayLength": 10,
+            "order": [[ 3, "desc" ]]
+        });
+
+
+        chart.stackBarChart = barchart_type_stack("{{ url_for('objects_ocr.objects_ocrs_range_json') }}?date_from={{date_from}}&date_to={{date_to}}", 'id');
+
+
+        chart.onResize();
+        $(window).on("resize", function() {
+            chart.onResize();
+        });
+  });
+
+function toggle_sidebar(){
+	if($('#nav_menu').is(':visible')){
+		$('#nav_menu').hide();
+		$('#side_menu').removeClass('border-right')
+		$('#side_menu').removeClass('col-lg-2')
+		$('#core_content').removeClass('col-lg-10')
+	}else{
+		$('#nav_menu').show();
+		$('#side_menu').addClass('border-right')
+		$('#side_menu').addClass('col-lg-2')
+		$('#core_content').addClass('col-lg-10')
+	}
+}
+</script>
+
+<script>
+	{% for obj_id in dict_objects %}
+    sparkline("sparklines_{{ obj_id }}", {{ dict_objects[obj_id]['sparkline'] }}, {});
+	{% endfor %}
+</script>
+
+<script>
+var margin = {top: 20, right: 100, bottom: 55, left: 45},
+				width  = 1000 - margin.left - margin.right,
+				height = 500  - margin.top  - margin.bottom;
+var x = d3.scaleBand().rangeRound([0, width]).padding(0.1);
+
+var y = d3.scaleLinear().rangeRound([height, 0]);
+
+var xAxis = d3.axisBottom(x);
+
+var yAxis = d3.axisLeft(y);
+
+var color = d3.scaleOrdinal(d3.schemeSet3);
+
+var svg = d3.select("#barchart_type").append("svg")
+				.attr("id", "thesvg")
+				.attr("viewBox", "0 0 1000 500")
+				.attr("width",  width  + margin.left + margin.right)
+				.attr("height", height + margin.top  + margin.bottom)
+			.append("g")
+				.attr("transform", "translate(" + margin.left + "," + margin.top + ")");
+
+function barchart_type_stack(url, id) {
+
+  d3.json(url)
+    .then(function(data){
+
+			var labelVar = 'date';  //A
+		  var varNames = d3.keys(data[0])
+		      .filter(function (key) { return key !== labelVar;}); //B
+
+		  data.forEach(function (d) { //D
+		    var y0 = 0;
+		    d.mapping = varNames.map(function (name) {
+		      return {
+		        name: name,
+		        label: d[labelVar],
+		        y0: y0,
+		        y1: y0 += +d[name]
+		      };
+		    });
+		    d.total = d.mapping[d.mapping.length - 1].y1;
+		  });
+
+		  x.domain(data.map(function (d) { return (d.date); })); //E
+		  y.domain([0, d3.max(data, function (d) { return d.total; })]);
+
+			svg.append("g")
+        .attr("class", "x axis")
+        .attr("transform", "translate(0," + height + ")")
+        .call(xAxis)
+				.selectAll("text")
+					.attr("class", "bar")
+					{% if  date_from|string == date_to|string and type is none %}
+						.on("click", function (d) { window.location.href = "{{ url_for('objects_ocr.objects_ocrs') }}?date_from={{date_from}}&date_to={{date_to}}&type_id="+d })
+						.attr("transform", "rotate(-18)" )
+					{% elif  date_from|string == date_to|string and type is not none %}
+						.on("click", function (d) { window.location.href = "{{ url_for('objects_ocr.objects_ocrs') }}?date_from="+d+'&date_to='+d })
+						.attr("transform", "rotate(-18)" )
+					{% else %}
+						.on("click", function (d) { window.location.href = "{{ url_for('objects_ocr.objects_ocrs') }}?date_from="+d+'&date_to='+d })
+						.attr("transform", "rotate(-40)" )
+					{% endif %}
+					.style("text-anchor", "end");
+
+    	svg.append("g")
+        .attr("class", "y axis")
+        .call(yAxis)
+      	.append("text")
+        	.attr("transform", "rotate(-90)")
+        	.attr("y", 6)
+        	.attr("dy", ".71em")
+        	.style("text-anchor", "end");
+
+			var selection = svg.selectAll(".series")
+		    .data(data)
+		    .enter().append("g")
+		      .attr("class", "series")
+		      .attr("transform", function (d) { return "translate(" + x((d.date)) + ",0)"; });
+
+			selection.selectAll("rect")
+	      .data(function (d) { return d.mapping; })
+	    .enter().append("rect")
+				.attr("class", "bar_stack")
+	      .attr("width", x.bandwidth())
+	      .attr("y", function (d) { return y(d.y1); })
+	      .attr("height", function (d) { return y(d.y0) - y(d.y1); })
+	      .style("fill", function (d) { return color(d.name); })
+	      .style("stroke", "grey")
+	      .on("mouseover", function (d) { showPopover.call(this, d); })
+				.on("mouseout", function (d) { removePopovers(); })
+				{% if  date_from|string == date_to|string and type is none %}
+					.on("click", function(d){ window.location.href = "{{ url_for('objects_ocr.objects_ocrs') }}" +'?date_from={{date_from}}&date_to={{date_to}}&type_id='+d.label+'&encoding='+d.name; });
+				{% elif  date_from|string == date_to|string and type is not none %}
+					.on("click", function(d){ window.location.href = "{{ url_for('objects_ocr.objects_ocrs') }}" +'?type_id={{type_id}}&date_from='+d.label+'&date_to='+d.label+'&encoding='+d.name; });
+				{% else %}
+					.on("click", function(d){ window.location.href = "{{ url_for('objects_ocr.objects_ocrs') }}" +'?type_id='+ d.name +'&date_from='+d.label+'&date_to='+d.label; });
+				{% endif %}
+
+				data.forEach(function(d) {
+					if(d.total !== 0){
+						svg.append("text")
+								.attr("class", "bar")
+								.attr("dy", "-.35em")
+								.attr('x', x(d.date) + x.bandwidth()/2)
+								.attr('y', y(d.total))
+								{% if  date_from|string == date_to|string and type is none %}
+									.on("click", function () {window.location.href = "{{ url_for('objects_ocr.objects_ocrs') }}"+'?date_from={{date_from}}&date_to={{date_to}}&type_id='+d.date })
+								{% elif  date_from|string == date_to|string and type is not none %}
+									.on("click", function () {window.location.href = "{{ url_for('objects_ocr.objects_ocrs') }}?type_id={{type_id}}&date_from="+d.date+'&date_to='+d.date })
+								{% else %}
+									.on("click", function () {window.location.href = "{{ url_for('objects_ocr.objects_ocrs') }}"+'?date_from='+d.date+'&date_to='+d.date })
+								{% endif %}
+								.style("text-anchor", "middle")
+								.text(d.total);
+					}
+				});
+
+			drawLegend(varNames);
+    });
+
+}
+
+function drawLegend (varNames) {
+    var legend = svg.selectAll(".legend")
+        .data(varNames.slice().reverse())
+      .enter().append("g")
+        .attr("class", "legend")
+        .attr("transform", function (d, i) { return "translate(0," + i * 20 + ")"; });
+
+    legend.append("rect")
+        .attr("x", 943)
+        .attr("width", 10)
+        .attr("height", 10)
+        .style("fill", color)
+        .style("stroke", "grey");
+
+    legend.append("text")
+				.attr("class", "svgText")
+        .attr("x", 941)
+        .attr("y", 6)
+        .attr("dy", ".35em")
+        .style("text-anchor", "end")
+        .text(function (d) { return d; });
+}
+
+function removePopovers () {
+    $('.popover').each(function() {
+      $(this).remove();
+    });
+  }
+
+function showPopover (d) {
+  $(this).popover({
+    title: "<b><span id='tooltip-id-name-bar'></span></b>",
+    placement: 'top',
+    container: 'body',
+    trigger: 'manual',
+    html : true,
+    content: function() {
+      return "<span id='tooltip-id-label'></span>" +
+            "<br/>num: <span id='tooltip-id-value-bar'></span>"; }
+  });
+  $(this).popover('show');
+  $("#tooltip-id-name-bar").text(d.name);
+  $("#tooltip-id-label").text(d.label);
+  $("#tooltip-id-value-bar").text(d3.format(",")(d.value ? d.value: d.y1 - d.y0));
+}
+
+chart.onResize = function () {
+    var aspect = 1000 / 500, chart = $("#thesvg");
+    var targetWidth = chart.parent().width();
+    chart.attr("width", targetWidth);
+    chart.attr("height", targetWidth / aspect);
+  }
+
+window.chart = chart;
+
+</script>
+
+<script>
+function draw_pie_chart(id, url_json, pie_on_click_url) {
+
+		var width_pie = 200;
+		var height_pie = 200;
+		var padding_pie = 10;
+		var opacity_pie = .8;
+
+		var radius_pie = Math.min(width_pie - padding_pie, height_pie - padding_pie) / 2;
+		//var color_pie = d3.scaleOrdinal(d3.schemeCategory10);
+		var color_pie = d3.scaleOrdinal(d3.schemeSet3);
+
+		var div_pie = d3.select("body").append("div")
+				.attr("class", "tooltip")
+				.style("opacity", 0);
+
+		var svg_pie = d3.select("#"+id)
+				.append('svg')
+				.attr("width", '100%')
+				.attr("height", '100%')
+				.attr('viewBox','0 0 '+Math.min(width_pie,height_pie) +' '+Math.min(width_pie,height_pie) )
+				.attr('preserveAspectRatio','xMinYMin')
+
+
+		var g_pie = svg_pie.append('g')
+				.attr('transform', 'translate(' + (width_pie/2) + ',' + (height_pie/2) + ')');
+
+		var arc_pie = d3.arc()
+				.innerRadius(0)
+				.outerRadius(radius_pie);
+
+		d3.json(url_json)
+			.then(function(data){
+
+			var pie_pie = d3.pie()
+					.value(function(d) { return d.value; })
+					.sort(null);
+
+			var path_pie = g_pie.selectAll('path')
+				  .data(pie_pie(data))
+				  .enter()
+				  .append("g")
+				  .append('path')
+				  .attr('d', arc_pie)
+				  .attr('fill', (d,i) => color_pie(i))
+					.attr('class', 'pie_path')
+					.on("mouseover", mouseovered_pie)
+					.on("mouseout", mouseouted_pie)
+					.on("click", function (d) {window.location.href = pie_on_click_url+d.data.name })
+				  .style('opacity', opacity_pie)
+				  .style('stroke', 'white');
+			});
+
+
+		function mouseovered_pie(d) {
+      //remove old content
+      $("#tooltip-id-name").remove();
+      $("#tooltip-id-value").remove();
+
+			// tooltip
+			var content;
+
+			content = "<b><span id='tooltip-id-name'></span></b><br/>"+
+								"<br/>"+
+								"<i>Decoded</i>: <span id='tooltip-id-value'></span><br/>"
+
+			div_pie.transition()
+				.duration(200)
+				.style("opacity", .9);
+			div_pie.html(content)
+				.style("left", (d3.event.pageX) + "px")
+				.style("top", (d3.event.pageY - 28) + "px");
+
+      $("#tooltip-id-name").text(d.data.name);
+      $("#tooltip-id-value").text(d.data.value);
+		}
+
+		function mouseouted_pie() {
+			div_pie.transition()
+				.duration(500)
+				.style("opacity", 0);
+		}
+}
+
+</script>
+
+
+
+<script>
+function barchart_type(url, id) {
+
+
+  var margin = {top: 20, right: 20, bottom: 70, left: 40};
+
+  var width = 960 - margin.left - margin.right;
+  var height = 500 - margin.top - margin.bottom;
+
+  var x = d3.scaleBand().rangeRound([0, width]).padding(0.1);
+  var y = d3.scaleLinear().rangeRound([height, 0]);
+
+  var xAxis = d3.axisBottom(x)
+    //.tickFormat(d3.time.format("%Y-%m"));
+
+  var yAxis = d3.axisLeft(y)
+    .ticks(10);
+
+/*var svg = d3.select(id).append("svg")
+      .attr("width", width + margin.left + margin.right)
+      .attr("height", height + margin.top + margin.bottom)
+			.attr("id", "thesvg")
+      .append("g")
+        .attr("transform",
+          "translate(" + margin.left + "," + margin.top + ")");*/
+
+
+  d3.json(url)
+    .then(function(data){
+
+        data.forEach(function(d) {
+          d.value = +d.value;
+        });
+
+				x.domain(data.map(function(d) { return d.date; }));
+        y.domain([0, d3.max(data, function(d) { return d.value; })]);
+
+        var label = svg.append("g")
+            .attr("class", "x axis")
+            .attr("transform", "translate(0," + height + ")")
+            .call(xAxis)
+          .selectAll("text")
+            .style("text-anchor", "end")
+            .attr("dx", "-.8em")
+            .attr("dy", "-.55em")
+						{% if  daily_type_chart %}
+						.attr("transform", "rotate(-20)" );
+						{% else %}
+						.attr("transform", "rotate(-70)" )
+						.attr("class", "bar")
+						{% endif %}
+
+        svg.append("g")
+            .attr("class", "y axis")
+            .call(yAxis)
+          .append("text")
+            .attr("transform", "rotate(-90)")
+            .attr("y", 6)
+            .attr("dy", ".71em")
+            .style("text-anchor", "end")
+            .text("Value ($)");
+
+        var bar = svg.selectAll("bar")
+            .data(data)
+          .enter().append("rect")
+            .attr("class", "bar")
+            //.style("fill", "steelblue")
+						.attr("x", function(d) { return x(d.date); })
+            .attr("width", x.bandwidth())
+            .attr("y", function(d) { return y(d.value); })
+            .attr("height", function(d) { return height - y(d.value); })
+
+
+        data.forEach(function(d) {
+          if(d.value != 0){
+            svg.append("text")
+                .attr("class", "bar")
+                .attr("dy", "-.35em")
+                //.text(function(d) { return d.value; });
+                .text(d.value)
+								.style("text-anchor", "middle")
+								.attr('x', x(d.date) + x.bandwidth()/2)
+                .attr('y', y(d.value));
+          }
+        });
+
+    });
+
+}
+</script>
+
+
+</body>
+
+</html>

var/www/templates/objects/ocr/ShowOcr.html

@@ -0,0 +1,133 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+  <title>OCR - AIL</title>
+	<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
+
+  <!-- Core CSS -->
+	<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
+    <link href="{{ url_for('static', filename='css/tags.css') }}" rel="stylesheet" type="text/css" />
+    <link href="{{ url_for('static', filename='css/ail-project.css') }}" rel="stylesheet">
+
+  <!-- JS -->
+	<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+    <script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
+    <script src="{{ url_for('static', filename='js/bootstrap4.min.js') }}"></script>
+	<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
+    <script src="{{ url_for('static', filename='js/tags.js') }}"></script>
+
+</head>
+<body>
+
+	{% include 'nav_bar.html' %}
+
+	<div class="container-fluid">
+		<div class="row">
+
+			{% include 'sidebars/sidebar_objects.html' %}
+
+			<div class="col-12 col-lg-10" id="core_content">
+
+                {% with meta=meta, is_correlation=False %}
+                    {% include 'objects/ocr/card_ocr.html' %}
+                {% endwith %}
+
+
+{#                {% if meta['extracted_matches'] %}#}
+{#                    <div id="accordion_extracted" class="mb-3 mx-3">#}
+{#                        <div class="card">#}
+{#                            <div class="card-header py-1" id="heading_extracted">#}
+{#                                <div class="row">#}
+{#                                    <div class="col-11">#}
+{#                                        <div class="mt-2">#}
+{#                                            <img id="misp-logo" src="{{ url_for('static', filename='image/ail-icon.png')}}" height="32"> Extracted&nbsp;&nbsp;#}
+{#                                            <div class="badge badge-warning">{{meta['extracted_matches']|length}}</div>#}
+{#                                        </div>#}
+{#                                    </div>#}
+{#                                    <div class="col-1">#}
+{#                                        <button class="btn btn-link btn-lg py-2 float-right rotate down" data-toggle="collapse" data-target="#collapse_extracted" aria-expanded="true" aria-controls="collapseDecoded">#}
+{#                                            <i class="fas fa-chevron-circle-down"></i>#}
+{#                                        </button>#}
+{#                                    </div>#}
+{#                                </div>#}
+{#                            </div>#}
+{##}
+{#                            <div id="collapse_extracted" class="collapse" aria-labelledby="heading_extracted" data-parent="#accordion_extracted">#}
+{#                                <div class="card-body">#}
+{#                                    <table id="table_extracted" class="table table-striped">#}
+{#                                        <thead class="thead-dark">#}
+{#                                        <tr>#}
+{#                                            <th>Type</th>#}
+{#                                            <th>ID</th>#}
+{#                                            <th>Extracted</th>#}
+{#                                        </tr>#}
+{#                                        </thead>#}
+{#                                        <tbody>#}
+{#                                        {% for match in meta['extracted_matches'] %}#}
+{#                                            <tr>#}
+{#                                                <td>#}
+{#                                                    <svg height="26" width="26">#}
+{#                                                        <g class="nodes">#}
+{#                                                            <circle cx="13" cy="13" r="13" fill="{{ meta['extracted_matches'][match]['icon']['color'] }}"></circle>#}
+{#                                                            <text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ meta['extracted_matches'][match]['icon']['style'] }}" font-size="16px">{{ meta['extracted_matches'][match]['icon']['icon'] }}</text>#}
+{#                                                        </g>#}
+{#                                                    </svg>#}
+{#                                                    {{ meta['extracted_matches'][match]['subtype'] }}#}
+{#                                                </td>#}
+{#                                                <td>{{ meta['extracted_matches'][match]['id'] }}</td>#}
+{#                                                <td>#}
+{#                                                    {% for row in meta['extracted_matches'][match]['matches'] %}#}
+{#                                                        <a href="#{{ row[0] }}:{{row[1] }}">{{ row[2] }}</a><br>#}
+{#                                                    {% endfor %}#}
+{#                                                </td>#}
+{#                                            </tr>#}
+{#                                        {% endfor %}#}
+{#                                        </tbody>#}
+{#                                    </table>#}
+{#                                </div>#}
+{#                            </div>#}
+{##}
+{#                        </div>#}
+{#                    </div>#}
+{#                {% endif %}#}
+
+
+                {% with translate_url=url_for('objects_ocr.object_ocr', id=meta['id']), obj_id=meta['id'] %}
+                    {% include 'chats_explorer/block_translation.html' %}
+                {% endwith %}
+                <div class="mb-4"></div>
+
+            </div>
+
+        </div>
+    </div>
+
+<script>
+    var chart = {};
+    $(document).ready(function(){
+        $("#page-Decoded").addClass("active");
+        $("#nav_chat").addClass("active");
+  });
+
+function toggle_sidebar(){
+	if($('#nav_menu').is(':visible')){
+		$('#nav_menu').hide();
+		$('#side_menu').removeClass('border-right')
+		$('#side_menu').removeClass('col-lg-2')
+		$('#core_content').removeClass('col-lg-10')
+	}else{
+		$('#nav_menu').show();
+		$('#side_menu').addClass('border-right')
+		$('#side_menu').addClass('col-lg-2')
+		$('#core_content').addClass('col-lg-10')
+	}
+}
+</script>
+
+</body>
+
+</html>

var/www/templates/objects/ocr/card_ocr.html

@@ -0,0 +1,165 @@
+<link href="{{ url_for('static', filename='css/tags.css') }}" rel="stylesheet" type="text/css" />
+<script src="{{ url_for('static', filename='js/tags.js') }}"></script>
+
+{% with modal_add_tags=ail_tags %}
+	{% include 'modals/add_tags.html' %}
+{% endwith %}
+
+{% include 'modals/edit_tag.html' %}
+
+<style>
+    .object_image {
+        filter: blur(5px);
+    }
+</style>
+
+<div class="card my-1">
+	<div class="card-header">
+		<h4 class="text-secondary">{{ meta["id"] }} :</h4>
+		<ul class="list-group mb-2">
+            <li class="list-group-item py-0">
+                <table class="table">
+                    <tbody style="font-size: 15px;">
+                    <tr>
+                        <td>
+                            <svg height="26" width="26">
+                                <g class="nodes">
+                                    <circle cx="13" cy="13" r="13" fill="orange"></circle>
+                                    <text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="{{ meta["svg_icon"]["style"] }}" font-size="16px">{{ meta["svg_icon"]["icon"] }}</text>
+                                </g>
+                            </svg>
+                            {{ meta['type'] }}
+                        </td>
+                    </tr>
+                    </tbody>
+                </table>
+            </li>
+			<li class="list-group-item py-0">
+				<div id="accordion_image" class="my-3">
+                    <div class="card">
+                        <div class="card-header py-1" id="headingImage">
+                            <button class="btn w-100 collapsed rotate" data-toggle="collapse" data-target="#collapseImage" aria-expanded="false" aria-controls="collapseImage">
+                                <span class="row text-left">
+                                    <div class="col-11">
+                                        <span class="mt-2">
+                                            <i class="far fa-image"></i> Show Image&nbsp;&nbsp;
+                                        </span>
+                                    </div>
+                                    <div class="col-1 text-primary">
+                                        <i class="fas fa-chevron-circle-down"></i>
+                                    </div>
+                                </span>
+                            </button>
+                        </div>
+                        <div id="collapseImage" class="collapse show" aria-labelledby="headingImage" data-parent="#accordion_image">
+                            <div class="card-body text-center">
+                                {% include 'objects/image/block_blur_img_slider.html' %}
+
+								<img class="object_image mb-1" usemap="#image-map" src="{{ url_for('objects_ocr.ocr_image', filename=meta['id'])}}">
+                                <map name="image-map">
+                                    {% for c in meta['map'] %}
+                                        <area shape="poly" coords="{{ c[0] }}" title="{{ c[1] }}">
+                                    {% endfor %}
+                                </map>
+                            </div>
+                        </div>
+                    </div>
+                </div>
+			</li>
+            <li class="list-group-item py-0">
+                <pre class="my-0" style="white-space: pre-wrap;">{{ meta['content'] }}</pre>
+                {% if meta['translation'] %}
+                    <hr class="m-1">
+                    <pre class="my-0 text-secondary" style="white-space: pre-wrap;">{{ meta['translation'] }}</pre>
+               {% endif %}
+
+
+
+                {% if not is_correlation %}
+                    <div class="my-1">
+
+                        {% set mess_id_escape= meta['id'] | replace("/", "_") %}
+                        <span class="btn btn-outline-dark p-0 px-1" type="button" data-toggle="collapse" data-target="#collapseTrans{{ mess_id_escape }}" aria-expanded="false" aria-controls="collapseTrans{{ mess_id_escape }}">
+                            <i class="fas fa-language"></i> {% if meta['language'] %}{{ meta['language'] }}{% endif %}
+                        </span>
+                        <div class="collapse" id="collapseTrans{{ mess_id_escape }}">
+                            <div class="card card-body">
+                                <form method="post" action="{{ url_for('languages_ui.translate_object') }}">
+                                    <input type="text" id="type" name="type" value="{{meta['type']}}" hidden>
+                                    <input type="text" id="id" name="id" value="{{meta['id']}}" hidden>
+                                    <span class="badge badge-primary">Source:</span>
+                                    <span class="">
+                                        <select id="language_target" name="language_target" class="form-select" aria-label="Message Language" onchange="$('#translation').val('');">
+                                                <option selected value="{{ meta['language'] }}">{{ meta['language'] }}</option>
+                                            {% for language in translation_languages %}
+                                                <option value="{{ language }}">{{ translation_languages[language] }}</option>
+                                            {% endfor %}
+                                        </select>
+                                    </span>
+                                    {% if translation_target %}
+                                        <input type="text" id="target" name="target" value="{{translation_target}}" hidden>
+                                        &nbsp;&nbsp;&nbsp;&nbsp;<span class="badge badge-primary">Target:</span><span>{{translation_target}}</span>
+                                        <textarea class="form-control" id="translation" name="translation">{{ meta['translation'] }}</textarea>
+                                        <button class="btn btn-dark" type="submit">
+                                            <i class="fas fa-pen-alt"> Update Language or Translation</i>
+                                        </button>
+                                    {% else %}
+                                        <button class="btn btn-dark" type="submit">
+                                            <i class="fas fa-pen-alt"> Update Language</i>
+                                        </button>
+                                    {% endif %}
+                                </form>
+                                <div>
+                                    <a class="btn btn-primary" href="{{ url_for('languages_ui.detect_object_language')}}?type={{ meta['type'] }}&id={{ meta['id'] }}">
+                                        <i class="fas fa-redo"></i> Detect Language
+                                    </a>
+                                </div>
+                            </div>
+                        </div>
+                    </div>
+                {% endif %}
+
+
+
+
+            </li>
+
+			<li class="list-group-item py-0">
+				<div class="my-2">
+					Tags:
+					{% for tag in meta['tags'] %}
+                        <button class="btn btn-{{ bootstrap_label[loop.index0 % 5] }}"
+                                data-toggle="modal" data-target="#edit_tags_modal"
+						        data-tagid="{{ tag }}" data-objtype="{{ meta['type'] }}" data-objsubtype="" data-objid="{{ meta["id"] }}">
+                                {{ tag }}
+						</button>
+					{%  endfor %}
+					<button type="button" class="btn btn-light" data-toggle="modal" data-target="#add_tags_modal">
+                        <i class="far fa-plus-square"></i>
+                    </button>
+				</div>
+			</li>
+		</ul>
+
+		{% with obj_type='ocr', obj_id=meta['id'], obj_subtype='' %}
+			{% include 'modals/investigations_register_obj.html' %}
+		{% endwith %}
+		<button type="button" class="btn btn-primary" data-toggle="modal" data-target="#investigations_register_obj_modal">
+			<i class="fas fa-microscope"></i> Investigations
+		</button>
+
+        <span class="mb-2 float-right">
+            {% if is_correlation %}
+                <a href="{{ url_for('objects_ocr.object_ocr')}}?subtype={{ meta['subtype'] }}&id={{ meta['id'] }}">
+                    <button class="btn btn-info"><i class="fas fa-expand"></i> Show Object</button>
+                </a>
+            {% else %}
+                <a href="{{ url_for('correlation.show_correlation')}}?type={{ meta['type'] }}&subtype={{ meta['subtype'] }}&id={{ meta['id'] }}">
+                    <button class="btn btn-info"><i class="far fa-eye"></i> Correlations &nbsp;
+                    </button>
+                </a>
+            {% endif %}
+        </span>
+
+	</div>
+</div>

var/www/templates/sidebars/sidebar_objects.html

@@ -82,6 +82,12 @@
                       <span>Image</span>
                   </a>
               </li>
+              <li class="nav-item">
+                  <a class="nav-link" href="{{url_for('objects_ocr.objects_ocrs')}}" id="nav_ocr">
+                      <i class="fas fa-expand"></i>
+                      <span>OCR</span>
+                  </a>
+              </li>
               <li class="nav-item">
                   <a class="nav-link" href="{{url_for('objects_title.objects_titles')}}" id="nav_title">
                       <i class="fas fa-heading"></i>

var/www/templates/tags/menu_sidebar.html

@@ -22,6 +22,18 @@
       	        Search Messages by Tags
       	      </a>
       	    </li>
+            <li class="nav-item">
+      	      <a class="nav-link" href="{{ url_for('tags_ui.tags_search_images') }}" id="nav_tags_search_image">
+      	        <i class="fas fa-image"></i>
+      	        Search Images by Tags
+      	      </a>
+      	    </li>
+            <li class="nav-item">
+      	      <a class="nav-link" href="{{ url_for('tags_ui.tags_search_ocrs') }}" id="nav_tags_search_message">
+      	        <i class="fas fa-expand"></i>
+      	        Search Ocrs by Tags
+      	      </a>
+      	    </li>
       	    <li class="nav-item">
       	      <a class="nav-link" href="{{ url_for('tags_ui.tags_search_domains') }}" id="nav_tags_search_domain">
       					<i class="fab fa-html5"></i>

var/www/templates/tags/search_obj_by_tags.html

@@ -125,7 +125,27 @@
                         </div>
                       </td>
                     </tr>
-                   {% endfor %}
+                  {% endfor %}
+                    {%elif dict_tagged["object_type"]=="screenshot"%}
+
+                {%elif dict_tagged["object_type"]=="image" or dict_tagged["object_type"]=="ocr"%}
+                    {% for dict_obj in dict_tagged["tagged_obj"] %}
+                        <tr>
+                            <td class="pb-0">
+                                <a target="_blank" href="{{ url_for('correlation.show_correlation') }}?type={{dict_tagged['object_type']}}&id={{dict_obj['id']}}" class="text-secondary">
+                                    <div style="line-height:0.9;">{{ dict_obj['id'] }}</div>
+                                </a>
+                                <div class="mb-2">
+                                    {% for tag in dict_obj['tags'] %}
+                                        <a href="{{ url_for('tags_ui.get_obj_by_tags') }}?object_type={{dict_tagged['object_type']}}&ltags={{ tag }}">
+                                            <span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
+                                        </a>
+                                    {%  endfor %}
+                                </div>
+                            </td>
+                        </tr>
+                    {% endfor %}
+
                 {%elif dict_tagged["object_type"]=="message"%}
                   {% for dict_obj in dict_tagged["tagged_obj"] %}
                     <tr>