CIRCL
/
lookyloo
mirror of https://github.com/CIRCL/lookyloo
2
0
Fork 0
Code Issues Releases Wiki Activity

fix: potential race condition when checking if a capture is ongoing or not

pull/868/head
Raphaël Vinot 2024-01-25 14:21:57 +01:00
parent 96225314f5
commit fcfe9751f3
1 changed files with 10 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
bin/background_processing.py
View File

@ -77,27 +77,30 @@ class Processing(AbstractManager):
def _retry_failed_enqueue(self) -> None: def _retry_failed_enqueue(self) -> None:
'''If enqueuing failed, the settings are added, with a UUID in the 'to_capture key', and they have a UUID''' '''If enqueuing failed, the settings are added, with a UUID in the 'to_capture key', and they have a UUID'''
for uuid in self.lookyloo.redis.zrevrangebyscore('to_capture', 'Inf', '-Inf'): to_requeue: list[str] = []
try_reenqueue = False for uuid, _ in self.lookyloo.redis.zscan_iter('to_capture'):
if self.lookyloo.redis.hexists(uuid, 'not_queued'): if self.lookyloo.redis.hexists(uuid, 'not_queued'):
# The capture is marked as not queued # The capture is marked as not queued
try_reenqueue = True to_requeue.append(uuid)
elif self.lookyloo.lacus.get_capture_status(uuid) in [CaptureStatusPy.UNKNOWN, CaptureStatusCore.UNKNOWN]: elif self.lookyloo.lacus.get_capture_status(uuid) in [CaptureStatusPy.UNKNOWN, CaptureStatusCore.UNKNOWN]:
# The capture is unknown on lacus side. It might be a race condition. # The capture is unknown on lacus side. It might be a race condition.
# Let's retry a few times. # Let's retry a few times.
retry = 3 retry = 3
while retry > 0: while retry > 0:
time.sleep(1)
if self.lookyloo.lacus.get_capture_status(uuid) not in [CaptureStatusPy.UNKNOWN, CaptureStatusCore.UNKNOWN]: if self.lookyloo.lacus.get_capture_status(uuid) not in [CaptureStatusPy.UNKNOWN, CaptureStatusCore.UNKNOWN]:
# Was a race condition, the UUID is being processed by Lacus # Was a race condition, the UUID has been or is being processed by Lacus
self.logger.info(f'UUID {uuid} was only temporary unknown') self.logger.info(f'UUID {uuid} was only temporary unknown')
break break
retry -= 1 retry -= 1
time.sleep(3)
else: else:
# UUID is still unknown # UUID is still unknown
self.logger.info(f'UUID {uuid} is still unknown') self.logger.info(f'UUID {uuid} is still unknown')
try_reenqueue = True to_requeue.append(uuid)
if not try_reenqueue:
for uuid in to_requeue:
if not self.lookyloo.redis.zscore('to_capture', uuid):
# The capture has been captured in the meantime.
continue continue
self.logger.info(f'Found a non-queued capture ({uuid}), retrying now.') self.logger.info(f'Found a non-queued capture ({uuid}), retrying now.')
# This capture couldn't be queued and we created the uuid locally # This capture couldn't be queued and we created the uuid locally