chg: [telegram] migration

bin/DB_KVROCKS_MIGRATION.py

@@ -30,6 +30,7 @@ from packages import Date
 # # # # CONFIGS # # # #
 config_loader = ConfigLoader()
 r_kvrocks = config_loader.get_db_conn("Kvrocks_DB")
+r_obj = config_loader.get_db_conn("Kvrocks_Objects")
 
 r_serv_db = config_loader.get_redis_conn("ARDB_DB")
 r_serv_tracker = config_loader.get_redis_conn("ARDB_Tracker")
@@ -173,6 +174,9 @@ def user_migration():
         Users._delete_user_token(user_id)
         Users._set_user_token(user_id, token)
 
+    for invite_row in r_crawler.smembers('telegram:invite_code'):
+        r_obj.sadd('telegram:invite_code', invite_row)
+
 # # # # # # # # # # # # # # # #
 #       AIL 2 AIL
 def ail_2_ail_migration():

bin/lib/telegram.py

@@ -7,12 +7,14 @@ import sys
 
 from urllib.parse import urlparse
 
-sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
-import ConfigLoader
-import Username
+sys.path.append(os.environ['AIL_BIN'])
+##################################
+# Import Project packages
+##################################
+from lib.ConfigLoader import ConfigLoader
 
-config_loader = ConfigLoader.ConfigLoader()
-r_serv_crawler = config_loader.get_redis_conn("ARDB_Onion")
+config_loader = ConfigLoader()
+r_obj = config_loader.get_db_conn("Kvrocks_Objects")
 config_loader = None
 
 REGEX_USERNAME = re.compile(r'[0-9a-zA-z_]+')
@@ -24,7 +26,7 @@ def save_item_correlation(username, item_id, item_date):
     Username.save_item_correlation('telegram', username, item_id, item_date)
 
 def save_telegram_invite_hash(invite_hash, item_id):
-    r_serv_crawler.sadd('telegram:invite_code', '{};{}'.format(invite_hash, item_id))
+    r_obj.sadd('telegram:invite_code', f'{invite_hash};{item_id}')
 
 def get_data_from_telegram_url(base_url, url_path):
     dict_url = {}

bin/modules/Telegram.py

@@ -17,6 +17,7 @@ sys.path.append(os.environ['AIL_BIN'])
 ##################################
 from modules.abstract_module import AbstractModule
 from lib.objects.Items import Item
+from lib.objects.Usernames import Username
 from lib import regex_helper
 from lib import telegram
 
@@ -40,7 +41,6 @@ class Telegram(AbstractModule):
         self.redis_logger.info(f"Module {self.module_name} initialized")
 
     def compute(self, message, r_result=False):
-        # messsage = item_id
         item = Item(message)
         item_content = item.get_content()
         item_date = item.get_date()
@@ -54,24 +54,30 @@ class Telegram(AbstractModule):
             print(telegram_link_tuple[2:-2].split("', '", 1))
             base_url, url_path = telegram_link_tuple[2:-2].split("', '", 1)
             dict_url = telegram.get_data_from_telegram_url(base_url, url_path)
-            if dict_url.get('username'):
-                telegram.save_item_correlation(dict_url['username'], item.get_id(), item_date)
-                print(f'username: {dict_url["username"]}')
-            if dict_url.get('invite_hash'):
-                telegram.save_telegram_invite_hash(dict_url['invite_hash'], item.get_id())
-                print(f'invite code: {dict_url["invite_hash"]}')
+            user_id = dict_url.get('username')
+            if user_id:
+                username = Username(user_id, 'telegram')
+                username.add(item_date, item.id)
+                print(f'username: {user_id}')
+            invite_hash = dict_url.get('invite_hash')
+            if invite_hash:
+                telegram.save_telegram_invite_hash(invite_hash, item.id)
+                print(f'invite code: {invite_hash}')
                 invite_code_found = True
 
         # extract tg links
         tg_links = self.regex_findall(self.re_tg_link, item.get_id(), item_content)
         for tg_link in tg_links:
             dict_url = telegram.get_data_from_tg_url(tg_link)
-            if dict_url.get('username'):
-                telegram.save_item_correlation(dict_url['username'], item.get_id(), item_date)
-                print(f'username: {dict_url["username"]}')
-            if dict_url.get('invite_hash'):
-                telegram.save_telegram_invite_hash(dict_url['invite_hash'], item.get_id())
-                print(f'invite code: {dict_url["invite_hash"]}')
+            user_id = dict_url.get('username')
+            if user_id:
+                username = Username(user_id, 'telegram')
+                username.add(item_date, item.id)
+                print(f'username: {user_id}')
+            invite_hash = dict_url.get('invite_hash')
+            if invite_hash:
+                telegram.save_telegram_invite_hash(invite_hash, item.get_id())
+                print(f'invite code: {invite_hash}')
                 invite_code_found = True
             if dict_url.get('login_code'):
                 print(f'login code: {dict_url["login_code"]}')
@@ -79,7 +85,7 @@ class Telegram(AbstractModule):
         # CREATE TAG
         if invite_code_found:
             # tags
-            msg = f'infoleak:automatic-detection="telegram-invite-hash";{item.get_id()}'
+            msg = f'infoleak:automatic-detection="telegram-invite-hash";{item.id}'
             self.send_message_to_queue(msg, 'Tags')