fix: [Retro Hunt] fix item directory

bin/lib/Tracker.py

@@ -16,6 +16,7 @@ from flask import escape
 
 sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
 import Date
+import Tag
 
 sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
 import ConfigLoader
@@ -26,6 +27,8 @@ r_cache = config_loader.get_redis_conn("Redis_Cache")
 
 r_serv_db = config_loader.get_redis_conn("ARDB_DB")
 r_serv_tracker = config_loader.get_redis_conn("ARDB_Tracker")
+
+items_dir = config_loader.get_config_str("Directories", "pastes").replace('/', '')
 config_loader = None
 
 email_regex = r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}'
@@ -248,7 +251,6 @@ def update_tracker_daterange(tracker_uuid, date, op='add'):
     if op == 'del':
         pass
 
-
 def remove_tracked_item(item_id):
     item_date = item_basic.get_item_date(item_id)
     for tracker_uuid in get_item_all_trackers_uuid(item_id):
@@ -267,6 +269,11 @@ def is_obj_tracked(obj_type, subtype, id):
 def get_obj_all_trackers(obj_type, subtype, id):
     return r_serv_tracker.smembers(f'obj:trackers:{obj_type}:{obj_id}')
 
+# # TODO: ADD all Objects + Subtypes
+def delete_obj_trackers(obj_type, subtype, id):
+    if obj_type == 'item':
+        remove_tracked_item(id)
+
 def get_email_subject(tracker_uuid):
     tracker_description = get_tracker_description(tracker_uuid)
     if not tracker_description:
@@ -1039,7 +1046,7 @@ def get_retro_hunt_dir_day_to_analyze(task_uuid, date, filter_last=False, source
 
 # # TODO: move me
 def get_items_to_analyze(dir, last=None):
-    full_dir = os.path.join(os.environ['AIL_HOME'], 'PASTES', dir) # # TODO: # FIXME: use item config dir
+    full_dir = os.path.join(os.environ['AIL_HOME'], items_dir, dir)
     if os.path.isdir(full_dir):
         all_items = sorted([os.path.join(dir, f) for f in os.listdir(full_dir) if os.path.isfile(os.path.join(full_dir, f))])
         # remove processed items
@@ -1265,7 +1272,21 @@ def api_delete_retro_hunt_task(task_uuid):
     else:
         return (delete_retro_hunt_task(task_uuid), 200)
 
-# if __name__ == '__main__':
+#### DB FIX ####
+def get_trackers_custom_tags():
+    tags = set()
+    for tracker_uuid in get_all_tracker_uuid():
+        for tag in get_tracker_tags(tracker_uuid):
+            tags.add(tag)
+    for task_uuid in get_all_retro_hunt_tasks():
+        for tag in get_retro_hunt_task_tags(task_uuid):
+            tags.add(tag)
+    return tags
+
+#### -- ####
+
+if __name__ == '__main__':
+    print(get_trackers_custom_tags())
     # fix_all_tracker_uuid_list()
     # res = get_all_tracker_uuid()
     # print(len(res))

bin/trackers/Retro_Hunt.py

@@ -65,11 +65,9 @@ class Retro_Hunt(AbstractModule):
         # First launch
         # restart
         rule = Tracker.get_retro_hunt_task_rule(task_uuid, r_compile=True)
-        rule_str = Tracker.get_retro_hunt_task_rule(task_uuid)
-        self.redis_logger.warning(f'{self.module_name}, Retro Hunt rule {rule_str}')
 
         timeout = Tracker.get_retro_hunt_task_timeout(task_uuid)
-        self.redis_logger.warning(f'{self.module_name}, Retro Hunt rule {task_uuid} timeout {timeout}')
+        self.redis_logger.debug(f'{self.module_name}, Retro Hunt rule {task_uuid} timeout {timeout}')
         sources = Tracker.get_retro_hunt_task_sources(task_uuid, r_sort=True)
 
         self.date_from = Tracker.get_retro_hunt_task_date_from(task_uuid)
@@ -90,7 +88,7 @@ class Retro_Hunt(AbstractModule):
             # # TODO: Filter previous item
             for dir in dirs_date:
                 print(dir)
-                self.redis_logger.warning(f'{self.module_name}, Retro Hunt searching in directory {dir}')
+                self.redis_logger.debug(f'{self.module_name}, Retro Hunt searching in directory {dir}')
                 l_obj = Tracker.get_items_to_analyze(dir)
                 for id in l_obj:
                     #print(f'{dir} / {id}')
@@ -98,7 +96,7 @@ class Retro_Hunt(AbstractModule):
                     # save current item in cache
                     Tracker.set_cache_retro_hunt_task_id(task_uuid, id)
 
-                    self.redis_logger.warning(f'{self.module_name}, Retro Hunt rule {task_uuid}, searching item {id}')
+                    self.redis_logger.debug(f'{self.module_name}, Retro Hunt rule {task_uuid}, searching item {id}')
 
                     yara_match = rule.match(data=self.item.get_content(), callback=self.yara_rules_match, which_callbacks=yara.CALLBACK_MATCHES, timeout=timeout)
 
@@ -147,7 +145,7 @@ class Retro_Hunt(AbstractModule):
         #print(data)
         task_uuid = data['namespace']
 
-        self.redis_logger.warning(f'{self.module_name}, Retro hunt {task_uuid} match found:    {id}')
+        self.redis_logger.info(f'{self.module_name}, Retro hunt {task_uuid} match found:    {id}')
         print(f'Retro hunt {task_uuid} match found:    {id}')
 
         Tracker.save_retro_hunt_match(task_uuid, id)