fix: do not add already processed UUIDs in the queue

2022-09-27 22:00:47 +02:00 · 2022-09-27 22:00:47 +02:00 · 0074f33544
parent cdaa4a7563
commit 0074f33544
1 changed files with 2 additions and 1 deletions
--- a/lookyloo/lookyloo.py
+++ b/lookyloo/lookyloo.py
@ -483,7 +483,8 @@ class Lookyloo():
            priority=priority
        )

-        if self.redis.zscore('to_capture', perma_uuid) is None:
+        if (not self.redis.hexists('lookup_dirs', perma_uuid)  # already captured
+                and self.redis.zscore('to_capture', perma_uuid) is None):  # capture ongoing
            if priority < -10:
                # Someone is probably abusing the system with useless URLs, remove them from the index
                query['listing'] = 0