CIRCL
/
lookyloo
mirror of https://github.com/CIRCL/lookyloo
2
0
Fork 0
Code Issues Releases Wiki Activity

new: set to keep track of the un-indexable captures

pull/937/head
Raphaël Vinot 2024-08-29 16:18:13 +02:00
parent 92f927bdb1
commit 55b437b74d
1 changed files with 9 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
lookyloo/indexing.py
View File

@ -99,6 +99,9 @@ class Indexing():
return tuple(to_return) # type: ignore[return-value] return tuple(to_return) # type: ignore[return-value]
def index_capture(self, uuid_to_index: str, directory: Path) -> None: def index_capture(self, uuid_to_index: str, directory: Path) -> None:
if self.redis.sismember('nothing_to_index', uuid_to_index):
# No HAR file in the capture, break immediately.
return
if not self.can_index(uuid_to_index): if not self.can_index(uuid_to_index):
self.logger.info(f'Indexing on {uuid_to_index} ongoing, skipping. ') self.logger.info(f'Indexing on {uuid_to_index} ongoing, skipping. ')
return return
@ -108,9 +111,14 @@ class Indexing():
if all(indexed): if all(indexed):
return return
if not list(directory.rglob('*.har.gz')) and not list(directory.rglob('*.har')):
self.logger.debug(f'No harfile in {uuid_to_index} - {directory}, nothing to index. ')
self.redis.sadd('nothing_to_index', uuid_to_index)
return
if not any((directory / pickle_name).exists() if not any((directory / pickle_name).exists()
for pickle_name in ['tree.pickle.gz', 'tree.pickle']): for pickle_name in ['tree.pickle.gz', 'tree.pickle']):
self.logger.warning(f'No pickle for {uuid_to_index}, skipping. ') self.logger.warning(f'No pickle for {uuid_to_index} - {directory}, skipping. ')
return return
# do the indexing # do the indexing