CIRCL
/
lookyloo
mirror of https://github.com/CIRCL/lookyloo
2
0
Fork 0
Code Issues Releases Wiki Activity

fix: Attempt to fix the stats page

pull/476/head
Raphaël Vinot 2022-07-29 11:02:51 +02:00
parent 9b4d56a0b6
commit b1d30cadcf
2 changed files with 18 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
lookyloo/capturecache.py
View File

@ -117,6 +117,7 @@ class CapturesIndex(Mapping):
and not cc.incomplete_redirects): and not cc.incomplete_redirects):
self.__cache[uuid] = cc self.__cache[uuid] = cc
return self.__cache[uuid] return self.__cache[uuid]
# The tree isn't cached yet
try: try:
tree = load_pickle_tree(capture_dir, capture_dir.stat().st_mtime) tree = load_pickle_tree(capture_dir, capture_dir.stat().st_mtime)
except TreeNeedsRebuild: except TreeNeedsRebuild:
@ -230,7 +231,7 @@ class CapturesIndex(Mapping):
lock_file.unlink(missing_ok=True) lock_file.unlink(missing_ok=True)
return tree return tree
def _set_capture_cache(self, capture_dir: Path, tree: Optional[CrawledTree]=None) -> CaptureCache: def _set_capture_cache(self, capture_dir: Path, tree: CrawledTree) -> CaptureCache:
'''Populate the redis cache for a capture. Mostly used on the index page. '''Populate the redis cache for a capture. Mostly used on the index page.
NOTE: Doesn't require the pickle.''' NOTE: Doesn't require the pickle.'''
with (capture_dir / 'uuid').open() as f: with (capture_dir / 'uuid').open() as f:
@ -257,17 +258,9 @@ class CapturesIndex(Mapping):
cache['timestamp'] = har.initial_start_time cache['timestamp'] = har.initial_start_time
cache['url'] = har.root_url cache['url'] = har.root_url
if har.initial_redirects and har.need_tree_redirects: if har.initial_redirects and har.need_tree_redirects:
if not tree:
# try to load tree from disk
tree = load_pickle_tree(capture_dir, capture_dir.stat().st_mtime)
# get redirects # get redirects
if tree: cache['redirects'] = json.dumps(tree.redirects)
cache['redirects'] = json.dumps(tree.redirects) cache['incomplete_redirects'] = 0
cache['incomplete_redirects'] = 0
else:
# Pickle not available
cache['redirects'] = json.dumps(har.initial_redirects)
cache['incomplete_redirects'] = 1
else: else:
cache['redirects'] = json.dumps(har.initial_redirects) cache['redirects'] = json.dumps(har.initial_redirects)
cache['incomplete_redirects'] = 0 cache['incomplete_redirects'] = 0

15
lookyloo/lookyloo.py
View File

@ -968,7 +968,20 @@ class Lookyloo():
stats: Dict[int, Dict[int, Dict[str, Any]]] = {} stats: Dict[int, Dict[int, Dict[str, Any]]] = {}
weeks_stats: Dict[int, Dict] = {} weeks_stats: Dict[int, Dict] = {}
for cache in self.sorted_capture_cache(): # Load the archived captures from redis
archived: List[CaptureCache] = []
p = self.redis.pipeline()
for directory in self.redis.hvals('lookup_dirs_archived'):
p.hgetall(directory)
for cache in p.execute():
if not cache:
continue
try:
archived.append(CaptureCache(cache))
except Exception:
continue
for cache in self.sorted_capture_cache() + archived:
date_submission: datetime = cache.timestamp date_submission: datetime = cache.timestamp
if date_submission.year not in stats: if date_submission.year not in stats: