CIRCL
/
lookyloo
mirror of https://github.com/CIRCL/lookyloo
2
0
Fork 0
Code Issues Releases Wiki Activity

chg: Proper use of shutil.move, speedup initialization of CaptureCache

pull/752/head
Raphaël Vinot 2023-08-08 12:41:21 +02:00
parent fa1d353834
commit e256a7fe6b
2 changed files with 6 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
bin/archiver.py
View File

@ -82,17 +82,17 @@ class Archiver(AbstractManager):
uuid_file = capture_dir / 'uuid'
if not uuid_file.exists():
self.logger.warning(f'No UUID file in {capture_dir}.')
shutil.move(str(capture_dir), str(get_homedir() / 'discarded_captures' / capture_dir.name))
shutil.move(str(capture_dir), str(get_homedir() / 'discarded_captures'))
continue
with uuid_file.open() as _f:
uuid = _f.read().strip()
if not uuid:
self.logger.warning(f'{uuid_file} is empty')
shutil.move(str(capture_dir), str(get_homedir() / 'discarded_captures' / capture_dir.name))
shutil.move(str(capture_dir), str(get_homedir() / 'discarded_captures'))
continue
if uuid in current_index:
self.logger.warning(f'Duplicate UUID ({uuid}) in {current_index[uuid]} and {uuid_file.parent.name}')
shutil.move(str(capture_dir), str(get_homedir() / 'discarded_captures' / capture_dir.name))
shutil.move(str(capture_dir), str(get_homedir() / 'discarded_captures'))
continue
current_index[uuid] = uuid_file.parent.name
@ -188,7 +188,7 @@ class Archiver(AbstractManager):
p.delete(str(capture_path))
(capture_path / 'tree.pickle').unlink(missing_ok=True)
(capture_path / 'tree.pickle.gz').unlink(missing_ok=True)
shutil.move(str(capture_path), str(dest_dir / capture_path.name))
shutil.move(str(capture_path), str(dest_dir))
p.execute()
self.logger.info('Archiving done.')

4
lookyloo/capturecache.py
View File

@ -58,8 +58,6 @@ class CaptureCache():
self.logger = LookylooCacheLogAdapter(logger, {'uuid': self.uuid})
self.capture_dir: Path = Path(cache_entry['capture_dir'])
if not self.capture_dir.exists():
raise MissingCaptureDirectory(f'The capture {self.uuid} does not exists in {self.capture_dir}.')
if url := cache_entry.get('url'):
# This entry *should* be present even if there is an error.
@ -95,6 +93,8 @@ class CaptureCache():
@property
def tree(self) -> CrawledTree:
if not self.capture_dir.exists():
raise MissingCaptureDirectory(f'The capture {self.uuid} does not exists in {self.capture_dir}.')
return load_pickle_tree(self.capture_dir, self.capture_dir.stat().st_mtime, self.logger)