CIRCL
/
lookyloo
mirror of https://github.com/CIRCL/lookyloo
2
0
Fork 0
Code Issues Releases Wiki Activity

chg: More reasonable rebuild cache

pull/78/head
Raphaël Vinot 2020-07-08 18:28:07 +02:00
parent db0d5b3f2d
commit dab2c53269
1 changed files with 7 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
bin/rebuild_caches.py
View File

@ -4,7 +4,6 @@
import argparse import argparse
import logging import logging
from lookyloo.helpers import load_pickle_tree
from lookyloo.lookyloo import Lookyloo, Indexing from lookyloo.lookyloo import Lookyloo, Indexing
logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s:%(message)s', logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s:%(message)s',
@ -23,16 +22,12 @@ if __name__ == '__main__':
indexing = Indexing() indexing = Indexing()
indexing.clear_indexes() indexing.clear_indexes()
for capture_dir in lookyloo.capture_dirs: for capture_uuid in lookyloo.capture_uuids:
try: try:
tree = load_pickle_tree(capture_dir) tree = lookyloo.get_crawled_tree(capture_uuid)
except Exception as e: except Exception as e:
print(capture_dir, e) print(capture_uuid, e)
if tree: continue
# NOTE: these two methods do nothing if we just generated the pickle
indexing.index_cookies_capture(tree) indexing.index_cookies_capture(tree)
indexing.index_body_hashes_capture(tree) indexing.index_body_hashes_capture(tree)
else:
try:
lookyloo.cache_tree(capture_dir=capture_dir)
except Exception as e:
print(capture_dir, e)