chg: settings tweaks, logging

bin/background_processing.py

@@ -85,11 +85,13 @@ class Processing(AbstractManager):
                 while retry > 0:
                     if self.lookyloo.lacus.get_capture_status(uuid) not in [CaptureStatusPy.UNKNOWN, CaptureStatusCore.UNKNOWN]:
                         # Was a race condition, the UUID is being processed by Lacus
+                        self.logger.info(f'UUID {uuid} was only temporary unknown')
                         break
                     retry -= 1
                     time.sleep(3)
                 else:
                     # UUID is still unknown
+                    self.logger.info(f'UUID {uuid} is still unknown')
                     try_reenqueue = True
             if not try_reenqueue:
                 continue

bin/start_website.py

@@ -27,7 +27,7 @@ class Website(AbstractManager):
                       '--graceful-timeout', '2', '--timeout', '300',
                       '-b', f'{ip}:{port}',
                       '--log-level', 'info',
-                      '--max-requests', '1000',
+                      '--max-requests', '10000',
                       'web:app'],
                      cwd=website_dir)
 

lookyloo/capturecache.py

@@ -107,7 +107,7 @@ def remove_pickle_tree(capture_dir: Path) -> None:
         pickle_file_gz.unlink()
 
 
-@lru_cache(maxsize=256)
+@lru_cache(maxsize=64)
 def load_pickle_tree(capture_dir: Path, last_mod_time: int, logger: Logger) -> CrawledTree:
     pickle_file = capture_dir / 'tree.pickle'
     pickle_file_gz = capture_dir / 'tree.pickle.gz'