From 1220f5926d1691e2f34113b13e2327cd0970d844 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= <raphael@vinot.info>
Date: Fri, 29 Sep 2023 15:00:40 +0200
Subject: [PATCH] fix: reduce calls to stat on archived dirs, improve logging
---
bin/archiver.py | 12 ++++++------
lookyloo/lookyloo.py | 8 ++++++--
2 files changed, 12 insertions(+), 8 deletions(-)
diff --git a/bin/archiver.py b/bin/archiver.py
index eefcf80a..9c7480a7 100755
--- a/bin/archiver.py
+++ b/bin/archiver.py
@@ -61,16 +61,12 @@ class Archiver(AbstractManager):
self.logger.debug(f'Updating index for {root_dir}')
index_file = root_dir / 'index'
- existing_captures_names = {existing_capture.name for existing_capture in root_dir.iterdir()
- if existing_capture.is_dir()}
if index_file.exists():
# Skip index if the directory has been archived.
try:
with index_file.open('r') as _f:
current_index = {uuid: dirname for uuid, dirname in csv.reader(_f)
- if uuid
- and dirname
- and dirname in existing_captures_names}
+ if uuid and dirname}
except Exception as e:
# the index file is broken, it will be recreated.
self.logger.warning(f'Index for {root_dir} broken, recreating it: {e}')
@@ -78,7 +74,11 @@ class Archiver(AbstractManager):
if not current_index:
index_file.unlink()
- if set(current_index.values()) == existing_captures_names:
+ curent_index_dirs = set(current_index.values())
+
+ existing_captures_names = {existing_capture.name for existing_capture in root_dir.iterdir()
+ if (existing_capture.name in curent_index_dirs) or existing_capture.is_dir()}
+ if curent_index_dirs == existing_captures_names:
# No new captures, quitting
self.logger.debug(f'No new captures in {root_dir}.')
return
diff --git a/lookyloo/lookyloo.py b/lookyloo/lookyloo.py
index 6d548822..7b7cb074 100644
--- a/lookyloo/lookyloo.py
+++ b/lookyloo/lookyloo.py
@@ -916,8 +916,12 @@ class Lookyloo():
# We might have a direct download link, and no screenshot. Assign the thumbnail accordingly.
try:
filename, data = self.get_data(capture_uuid)
- self.logger.info(f'{capture_uuid} is is a download link, set thumbnail.')
- error_img = get_homedir() / 'website' / 'web' / 'static' / 'download.png'
+ if filename:
+ self.logger.info(f'{capture_uuid} is is a download link, set thumbnail.')
+ error_img = get_homedir() / 'website' / 'web' / 'static' / 'download.png'
+ else:
+ # No screenshot and no data, it is probably because the capture failed.
+ error_img = get_homedir() / 'website' / 'web' / 'static' / 'error_screenshot.png'
except Exception:
# The capture probably doesn't have a screenshot at all, no need to log that as a warning.
self.logger.debug(f'Unable to generate the screenshot thumbnail of {capture_uuid}: {e}.')