fix: [queues] fix ended duplicate + sync queue

bin/core/Sync_module.py

@@ -33,8 +33,8 @@ class Sync_module(AbstractModule): # TODO KEEP A QUEUE ?????????????????????????
     Sync_module module for AIL framework
     """
 
-    def __init__(self):
-        super(Sync_module, self).__init__()
+    def __init__(self, queue=False): # FIXME MODIFY/ADD QUEUE
+        super(Sync_module, self).__init__(queue=queue)
 
         # Waiting time in seconds between to message processed
         self.pending_seconds = 10
@@ -102,13 +102,6 @@ class Sync_module(AbstractModule): # TODO KEEP A QUEUE ?????????????????????????
                     if isinstance(err, ModuleQueueError):
                         self.queue.error()
                         raise err
-                # remove from set_module
-                ## check if item process == completed
-
-                if self.obj:
-                    self.queue.end_message(self.obj.get_global_id(), self.sha256_mess)
-                    self.obj = None
-                    self.sha256_mess = None
 
             else:
                 self.computeNone()
@@ -119,5 +112,5 @@ class Sync_module(AbstractModule): # TODO KEEP A QUEUE ?????????????????????????
 
 if __name__ == '__main__':
 
-    module = Sync_module()
+    module = Sync_module(queue=False)  # FIXME MODIFY/ADD QUEUE
     module.run()

bin/lib/ail_queues.py

@@ -199,7 +199,7 @@ def is_processed_obj_moduled(obj_global_id):
     return r_obj_process.exists(f'obj:modules:{obj_global_id}')
 
 def is_processed_obj(obj_global_id):
-    return is_processed_obj_queued(obj_global_id) and is_processed_obj_moduled(obj_global_id)
+    return is_processed_obj_queued(obj_global_id) or is_processed_obj_moduled(obj_global_id)
 
 def get_processed_obj_modules(obj_global_id):
     return r_obj_process.zrange(f'obj:modules:{obj_global_id}', 0, -1)

configs/modules.cfg

@@ -24,7 +24,7 @@ publish = Importers,Tags
 
 [Global]
 subscribe = SaveObj
-publish = Item,Sync
+publish = Item
 
 [Duplicates]
 subscribe = Duplicate
@@ -108,11 +108,7 @@ publish = Tags
 
 [Tags]
 subscribe = Tags
-publish = Tag_feed,Sync
-
-# dirty fix
-[Sync_module]
-subscribe = Sync
+publish = Tag_feed
 
 [MISP_Thehive_Auto_Push]
 subscribe = Tag_feed
@@ -165,6 +161,9 @@ publish = Tags
 [Zerobins]
 subscribe = Url
 
+#[Sync_module]
+#publish = Sync
+
 # [My_Module_Name]
 # subscribe = Global # Queue name
 # publish = Tags # Queue name