From dab2c5326938aeabb339f414d33a932bb73ecee1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 8 Jul 2020 18:28:07 +0200 Subject: [PATCH] chg: More reasonable rebuild cache --- bin/rebuild_caches.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/bin/rebuild_caches.py b/bin/rebuild_caches.py index 1e7338c..83ccafb 100755 --- a/bin/rebuild_caches.py +++ b/bin/rebuild_caches.py @@ -4,7 +4,6 @@ import argparse import logging -from lookyloo.helpers import load_pickle_tree from lookyloo.lookyloo import Lookyloo, Indexing logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s:%(message)s', @@ -23,16 +22,12 @@ if __name__ == '__main__': indexing = Indexing() indexing.clear_indexes() - for capture_dir in lookyloo.capture_dirs: + for capture_uuid in lookyloo.capture_uuids: try: - tree = load_pickle_tree(capture_dir) + tree = lookyloo.get_crawled_tree(capture_uuid) except Exception as e: - print(capture_dir, e) - if tree: - indexing.index_cookies_capture(tree) - indexing.index_body_hashes_capture(tree) - else: - try: - lookyloo.cache_tree(capture_dir=capture_dir) - except Exception as e: - print(capture_dir, e) + print(capture_uuid, e) + continue + # NOTE: these two methods do nothing if we just generated the pickle + indexing.index_cookies_capture(tree) + indexing.index_body_hashes_capture(tree)