From 0daff9ef7751a2d60479c54e959830b6db98a98c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 11 Oct 2023 15:02:11 +0200 Subject: [PATCH] chg: settings tweaks, logging --- bin/background_processing.py | 2 ++ bin/start_website.py | 2 +- lookyloo/capturecache.py | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/bin/background_processing.py b/bin/background_processing.py index 64978188..cdc23582 100755 --- a/bin/background_processing.py +++ b/bin/background_processing.py @@ -85,11 +85,13 @@ class Processing(AbstractManager): while retry > 0: if self.lookyloo.lacus.get_capture_status(uuid) not in [CaptureStatusPy.UNKNOWN, CaptureStatusCore.UNKNOWN]: # Was a race condition, the UUID is being processed by Lacus + self.logger.info(f'UUID {uuid} was only temporary unknown') break retry -= 1 time.sleep(3) else: # UUID is still unknown + self.logger.info(f'UUID {uuid} is still unknown') try_reenqueue = True if not try_reenqueue: continue diff --git a/bin/start_website.py b/bin/start_website.py index f6e20840..b8d2a6ae 100755 --- a/bin/start_website.py +++ b/bin/start_website.py @@ -27,7 +27,7 @@ class Website(AbstractManager): '--graceful-timeout', '2', '--timeout', '300', '-b', f'{ip}:{port}', '--log-level', 'info', - '--max-requests', '1000', + '--max-requests', '10000', 'web:app'], cwd=website_dir) diff --git a/lookyloo/capturecache.py b/lookyloo/capturecache.py index 04ad709b..3ae8f8aa 100644 --- a/lookyloo/capturecache.py +++ b/lookyloo/capturecache.py @@ -107,7 +107,7 @@ def remove_pickle_tree(capture_dir: Path) -> None: pickle_file_gz.unlink() -@lru_cache(maxsize=256) +@lru_cache(maxsize=64) def load_pickle_tree(capture_dir: Path, last_mod_time: int, logger: Logger) -> CrawledTree: pickle_file = capture_dir / 'tree.pickle' pickle_file_gz = capture_dir / 'tree.pickle.gz'