From fd9325bb0d1cc01a1123387c3c1f2f38c03228a5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= <raphael@vinot.info>
Date: Wed, 26 Jul 2023 12:37:12 +0200
Subject: [PATCH] chg: Improve logging, add lock on indexer.

---
 bin/background_indexer.py | 10 ++++++++--
 lookyloo/helpers.py       |  2 +-
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/bin/background_indexer.py b/bin/background_indexer.py
index 4838025..821d0fd 100755
--- a/bin/background_indexer.py
+++ b/bin/background_indexer.py
@@ -49,7 +49,7 @@ class BackgroundIndexer(AbstractManager):
 
             if is_locked(uuid_path.parent):
                 # it is really locked
-                self.logger.info(f'{uuid_path.parent} is locked, pickle generated by another process.')
+                self.logger.debug(f'{uuid_path.parent} is locked, pickle generated by another process.')
                 continue
 
             max_captures -= 1
@@ -83,8 +83,13 @@ class BackgroundIndexer(AbstractManager):
         return False
 
     def _check_indexes(self):
-        self.logger.info('Check indexes...')
         index_redis = self.lookyloo.indexing.redis
+        can_index = index_redis.set('ongoing_indexing', 1, ex=300, nx=True)
+        if not can_index:
+            # There is no reason to run this method in multiple scripts.
+            self.logger.info('Indexing already ongoing in another process.')
+            return
+        self.logger.info('Check indexes...')
         for cache in self.lookyloo.sorted_capture_cache(cached_captures_only=False):
             if self.lookyloo.is_public_instance and cache.no_index:
                 # Capture unindexed
@@ -118,6 +123,7 @@ class BackgroundIndexer(AbstractManager):
                 self.lookyloo.indexing.index_http_headers_hashes_capture(ct)
             # NOTE: categories aren't taken in account here, should be fixed(?)
             # see indexing.index_categories_capture(capture_uuid, categories)
+        index_redis.delete('ongoing_indexing')
         self.logger.info('... done.')
 
 
diff --git a/lookyloo/helpers.py b/lookyloo/helpers.py
index c47a0d8..6fd8964 100644
--- a/lookyloo/helpers.py
+++ b/lookyloo/helpers.py
@@ -269,7 +269,7 @@ def is_locked(locked_dir_path: Path, /) -> bool:
         return False
 
     # The lockfile is here for a good reason.
-    logger.info(f'Directory locked by {pid}.')
+    logger.debug(f'Directory locked by {pid}.')
     return True