CIRCL
/
lookyloo
mirror of https://github.com/CIRCL/lookyloo
2
0
Fork 0
Code Issues Releases Wiki Activity

chg: Update indexes only when needed

pull/860/head
Raphaël Vinot 2024-01-08 16:27:12 +01:00
parent 67b0dc0b7e
commit d60a4e56db
1 changed files with 10 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
bin/archiver.py
View File

@ -111,17 +111,16 @@ class Archiver(AbstractManager):
continue continue
dir_on_disk = root_dir / entry.rsplit('/', 1)[-1] dir_on_disk = root_dir / entry.rsplit('/', 1)[-1]
if dir_on_disk.name.isdigit(): if dir_on_disk.name.isdigit():
# got a day directory that contains captures if sub_index := self._update_index(dir_on_disk, s3fs_parent_dir=s3fs_dir):
sub_index = self._update_index(dir_on_disk, s3fs_parent_dir=s3fs_dir) # got a day directory that contains captures
if sub_index:
sub_indexes.append(sub_index) sub_indexes.append(sub_index)
else:
continue
else: else:
# got a capture # got a capture
if str(dir_on_disk) not in current_index_dirs: if str(dir_on_disk) not in current_index_dirs:
new_captures.add(dir_on_disk) new_captures.add(dir_on_disk)
else: current_dirs.add(dir_on_disk.name)
current_dirs.add(str(dir_on_disk))
current_dirs.add(dir_on_disk.name)
else: else:
with os.scandir(root_dir) as it: with os.scandir(root_dir) as it:
@ -132,16 +131,16 @@ class Archiver(AbstractManager):
continue continue
dir_on_disk = Path(entry) dir_on_disk = Path(entry)
if dir_on_disk.name.isdigit(): if dir_on_disk.name.isdigit():
sub_index = self._update_index(dir_on_disk) if sub_index := self._update_index(dir_on_disk):
if sub_index: # got a day directory that contains captures
sub_indexes.append(sub_index) sub_indexes.append(sub_index)
else:
continue
else: else:
# isoformat # isoformat
if str(dir_on_disk) not in current_index_dirs: if str(dir_on_disk) not in current_index_dirs:
new_captures.add(dir_on_disk) new_captures.add(dir_on_disk)
else: current_dirs.add(dir_on_disk.name)
current_dirs.add(str(dir_on_disk))
current_dirs.add(dir_on_disk.name)
# Check if all the directories in current_dirs (that we got by listing the directory) # Check if all the directories in current_dirs (that we got by listing the directory)
# are the same as the one in the index. If they're not, we pop the UUID before writing the index # are the same as the one in the index. If they're not, we pop the UUID before writing the index