chg: Improve logging, add lock on indexer.

bin/background_indexer.py

@@ -49,7 +49,7 @@ class BackgroundIndexer(AbstractManager):
 
             if is_locked(uuid_path.parent):
                 # it is really locked
-                self.logger.info(f'{uuid_path.parent} is locked, pickle generated by another process.')
+                self.logger.debug(f'{uuid_path.parent} is locked, pickle generated by another process.')
                 continue
 
             max_captures -= 1
@@ -83,8 +83,13 @@ class BackgroundIndexer(AbstractManager):
         return False
 
     def _check_indexes(self):
-        self.logger.info('Check indexes...')
         index_redis = self.lookyloo.indexing.redis
+        can_index = index_redis.set('ongoing_indexing', 1, ex=300, nx=True)
+        if not can_index:
+            # There is no reason to run this method in multiple scripts.
+            self.logger.info('Indexing already ongoing in another process.')
+            return
+        self.logger.info('Check indexes...')
         for cache in self.lookyloo.sorted_capture_cache(cached_captures_only=False):
             if self.lookyloo.is_public_instance and cache.no_index:
                 # Capture unindexed
@@ -118,6 +123,7 @@ class BackgroundIndexer(AbstractManager):
                 self.lookyloo.indexing.index_http_headers_hashes_capture(ct)
             # NOTE: categories aren't taken in account here, should be fixed(?)
             # see indexing.index_categories_capture(capture_uuid, categories)
+        index_redis.delete('ongoing_indexing')
         self.logger.info('... done.')
 
 

lookyloo/helpers.py

@@ -269,7 +269,7 @@ def is_locked(locked_dir_path: Path, /) -> bool:
         return False
 
     # The lockfile is here for a good reason.
-    logger.info(f'Directory locked by {pid}.')
+    logger.debug(f'Directory locked by {pid}.')
     return True