diff --git a/bin/background_indexer.py b/bin/background_indexer.py index 48380256..821d0fd2 100755 --- a/bin/background_indexer.py +++ b/bin/background_indexer.py @@ -49,7 +49,7 @@ class BackgroundIndexer(AbstractManager): if is_locked(uuid_path.parent): # it is really locked - self.logger.info(f'{uuid_path.parent} is locked, pickle generated by another process.') + self.logger.debug(f'{uuid_path.parent} is locked, pickle generated by another process.') continue max_captures -= 1 @@ -83,8 +83,13 @@ class BackgroundIndexer(AbstractManager): return False def _check_indexes(self): - self.logger.info('Check indexes...') index_redis = self.lookyloo.indexing.redis + can_index = index_redis.set('ongoing_indexing', 1, ex=300, nx=True) + if not can_index: + # There is no reason to run this method in multiple scripts. + self.logger.info('Indexing already ongoing in another process.') + return + self.logger.info('Check indexes...') for cache in self.lookyloo.sorted_capture_cache(cached_captures_only=False): if self.lookyloo.is_public_instance and cache.no_index: # Capture unindexed @@ -118,6 +123,7 @@ class BackgroundIndexer(AbstractManager): self.lookyloo.indexing.index_http_headers_hashes_capture(ct) # NOTE: categories aren't taken in account here, should be fixed(?) # see indexing.index_categories_capture(capture_uuid, categories) + index_redis.delete('ongoing_indexing') self.logger.info('... done.') diff --git a/lookyloo/helpers.py b/lookyloo/helpers.py index c47a0d8f..6fd89644 100644 --- a/lookyloo/helpers.py +++ b/lookyloo/helpers.py @@ -269,7 +269,7 @@ def is_locked(locked_dir_path: Path, /) -> bool: return False # The lockfile is here for a good reason. - logger.info(f'Directory locked by {pid}.') + logger.debug(f'Directory locked by {pid}.') return True