chg: make to_capture key a ranked set again

bin/async_capture.py

@@ -69,11 +69,11 @@ class AsyncCapture(AbstractManager):
             entries = self.lacus.get_capture(uuid, decode=True)
             if entries['status'] != CaptureStatusCore.DONE:
                 self.logger.warning(f'The capture {uuid} is reported as not done ({entries["status"]}) when it should.')
-                self.redis.srem('to_capture', uuid)
+                self.redis.zrem('to_capture', uuid)
                 return
         else:
             # Find a capture that is done
-            for uuid_b in self.redis.smembers('to_capture'):
+            for uuid_b in self.redis.zrevrangebyscore('to_capture', 'Inf', '-Inf'):
                 uuid = uuid_b.decode()
                 if not uuid:
                     return
@@ -158,7 +158,7 @@ class AsyncCapture(AbstractManager):
             lazy_cleanup.hset('lookup_dirs', uuid, str(dirpath))
             if queue and self.redis.zscore('queues', queue):
                 lazy_cleanup.zincrby('queues', -1, queue)
-            lazy_cleanup.srem('to_capture', uuid)
+            lazy_cleanup.zrem('to_capture', uuid)
             lazy_cleanup.srem('ongoing', uuid)
             lazy_cleanup.delete(uuid)
             # make sure to expire the key if nothing was processed for a while (= queues empty)

lookyloo/lookyloo.py

@@ -494,7 +494,7 @@ class Lookyloo():
             query['listing'] = 0
 
         p = self.redis.pipeline()
-        p.sadd('to_capture', perma_uuid)
+        p.zadd('to_capture', {perma_uuid: priority})
         if query:
             p.hset(perma_uuid, mapping=query)  # This will add the remaining entries that are lookyloo specific
         p.zincrby('queues', 1, f'{source}|{authenticated}|{user}')