CIRCL
/
lookyloo
mirror of https://github.com/CIRCL/lookyloo
2
0
Fork 0
Code Issues Releases Wiki Activity

fix: reduce calls to stat on archived dirs, improve logging

pull/793/head
Raphaël Vinot 2023-09-29 15:00:40 +02:00
parent db9e6edcdb
commit 1220f5926d
2 changed files with 12 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
bin/archiver.py
View File

@ -61,16 +61,12 @@ class Archiver(AbstractManager):
self.logger.debug(f'Updating index for {root_dir}')
index_file = root_dir / 'index'
existing_captures_names = {existing_capture.name for existing_capture in root_dir.iterdir()
if existing_capture.is_dir()}
if index_file.exists():
# Skip index if the directory has been archived.
try:
with index_file.open('r') as _f:
current_index = {uuid: dirname for uuid, dirname in csv.reader(_f)
if uuid
and dirname
and dirname in existing_captures_names}
if uuid and dirname}
except Exception as e:
# the index file is broken, it will be recreated.
self.logger.warning(f'Index for {root_dir} broken, recreating it: {e}')
@ -78,7 +74,11 @@ class Archiver(AbstractManager):
if not current_index:
index_file.unlink()
if set(current_index.values()) == existing_captures_names:
curent_index_dirs = set(current_index.values())
existing_captures_names = {existing_capture.name for existing_capture in root_dir.iterdir()
if (existing_capture.name in curent_index_dirs) or existing_capture.is_dir()}
if curent_index_dirs == existing_captures_names:
# No new captures, quitting
self.logger.debug(f'No new captures in {root_dir}.')
return

8
lookyloo/lookyloo.py
View File

@ -916,8 +916,12 @@ class Lookyloo():
# We might have a direct download link, and no screenshot. Assign the thumbnail accordingly.
try:
filename, data = self.get_data(capture_uuid)
self.logger.info(f'{capture_uuid} is is a download link, set thumbnail.')
error_img = get_homedir() / 'website' / 'web' / 'static' / 'download.png'
if filename:
self.logger.info(f'{capture_uuid} is is a download link, set thumbnail.')
error_img = get_homedir() / 'website' / 'web' / 'static' / 'download.png'
else:
# No screenshot and no data, it is probably because the capture failed.
error_img = get_homedir() / 'website' / 'web' / 'static' / 'error_screenshot.png'
except Exception:
# The capture probably doesn't have a screenshot at all, no need to log that as a warning.
self.logger.debug(f'Unable to generate the screenshot thumbnail of {capture_uuid}: {e}.')