mirror of https://github.com/CIRCL/lookyloo
chg: Improve logging, add lock on indexer.
parent
f60457a484
commit
fd9325bb0d
|
@ -49,7 +49,7 @@ class BackgroundIndexer(AbstractManager):
|
||||||
|
|
||||||
if is_locked(uuid_path.parent):
|
if is_locked(uuid_path.parent):
|
||||||
# it is really locked
|
# it is really locked
|
||||||
self.logger.info(f'{uuid_path.parent} is locked, pickle generated by another process.')
|
self.logger.debug(f'{uuid_path.parent} is locked, pickle generated by another process.')
|
||||||
continue
|
continue
|
||||||
|
|
||||||
max_captures -= 1
|
max_captures -= 1
|
||||||
|
@ -83,8 +83,13 @@ class BackgroundIndexer(AbstractManager):
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def _check_indexes(self):
|
def _check_indexes(self):
|
||||||
self.logger.info('Check indexes...')
|
|
||||||
index_redis = self.lookyloo.indexing.redis
|
index_redis = self.lookyloo.indexing.redis
|
||||||
|
can_index = index_redis.set('ongoing_indexing', 1, ex=300, nx=True)
|
||||||
|
if not can_index:
|
||||||
|
# There is no reason to run this method in multiple scripts.
|
||||||
|
self.logger.info('Indexing already ongoing in another process.')
|
||||||
|
return
|
||||||
|
self.logger.info('Check indexes...')
|
||||||
for cache in self.lookyloo.sorted_capture_cache(cached_captures_only=False):
|
for cache in self.lookyloo.sorted_capture_cache(cached_captures_only=False):
|
||||||
if self.lookyloo.is_public_instance and cache.no_index:
|
if self.lookyloo.is_public_instance and cache.no_index:
|
||||||
# Capture unindexed
|
# Capture unindexed
|
||||||
|
@ -118,6 +123,7 @@ class BackgroundIndexer(AbstractManager):
|
||||||
self.lookyloo.indexing.index_http_headers_hashes_capture(ct)
|
self.lookyloo.indexing.index_http_headers_hashes_capture(ct)
|
||||||
# NOTE: categories aren't taken in account here, should be fixed(?)
|
# NOTE: categories aren't taken in account here, should be fixed(?)
|
||||||
# see indexing.index_categories_capture(capture_uuid, categories)
|
# see indexing.index_categories_capture(capture_uuid, categories)
|
||||||
|
index_redis.delete('ongoing_indexing')
|
||||||
self.logger.info('... done.')
|
self.logger.info('... done.')
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -269,7 +269,7 @@ def is_locked(locked_dir_path: Path, /) -> bool:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
# The lockfile is here for a good reason.
|
# The lockfile is here for a good reason.
|
||||||
logger.info(f'Directory locked by {pid}.')
|
logger.debug(f'Directory locked by {pid}.')
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue