fix: [ZMQ Feeder] performance: replace zmq recv NOBLOCK by Poller

bin/Global.py

@@ -95,7 +95,7 @@ if __name__ == '__main__':
                 #publisher.info(to_print)
                 time_1 = time.time()
                 processed_paste = 0
-            time.sleep(1)
+            time.sleep(0.5)
             continue
 
         # remove PASTES_FOLDER from item path (crawled item + submited)

bin/Helper.py

@@ -53,13 +53,14 @@ class PubSub(object): ## TODO: remove config, use ConfigLoader by default
             self.zmq_sub = True
             context = zmq.Context()
 
+            # Get all feeds
             self.subscribers = []
             addresses = self.config.get(conn_name, 'address')
             for address in addresses.split(','):
-                new_sub = context.socket(zmq.SUB)
+                subscriber = context.socket(zmq.SUB)
-                new_sub.connect(address)
+                subscriber.connect(address)
-                new_sub.setsockopt_string(zmq.SUBSCRIBE, channel)
+                subscriber.setsockopt_string(zmq.SUBSCRIBE, channel)
-                self.subscribers.append(new_sub)
+                self.subscribers.append(subscriber)
 
     def setup_publish(self, conn_name):
         if self.config.has_section(conn_name):
@@ -96,14 +97,18 @@ class PubSub(object): ## TODO: remove config, use ConfigLoader by default
                 if msg.get('data', None) is not None:
                     yield msg['data']
         elif self.zmq_sub:
+            # Initialize poll set
+            poller = zmq.Poller()
+            for subscriber in self.subscribers:
+                poller.register(subscriber, zmq.POLLIN)
+
             while True:
-                for sub in self.subscribers:
+                socks = dict(poller.poll())
-                    try:
+
-                        msg = sub.recv(zmq.NOBLOCK)
+                for subscriber in self.subscribers:
-                        yield msg.split(b" ", 1)[1]
+                    if subscriber in socks:
-                    except zmq.error.Again as e:
+                        message = subscriber.recv()
-                        time.sleep(0.2)
+                        yield message.split(b' ', 1)[1]
-                        pass
         else:
             raise Exception('No subscribe function defined')
 

bin/Mixer.py

@@ -186,7 +186,6 @@ if __name__ == '__main__':
                 print("Empty Paste: not processed")
                 publisher.debug("Empty Paste: {0} not processed".format(message))
         else:
-            print("Empty Queues: Waiting...")
 
             if int(time.time() - time_1) > refresh_time:
                 # update internal feeder