diff --git a/lookyloo/lookyloo.py b/lookyloo/lookyloo.py index 393c61a0..3231cdc0 100644 --- a/lookyloo/lookyloo.py +++ b/lookyloo/lookyloo.py @@ -18,6 +18,7 @@ from typing import Union, Dict, List, Tuple, Optional, Any, MutableMapping, Set, from urllib.parse import urlsplit from uuid import uuid4 from zipfile import ZipFile +import operator import dns.resolver import dns.rdatatype @@ -448,6 +449,18 @@ class Lookyloo(): def capture_uuids(self): return self.redis.hkeys('lookup_dirs') + @property + def sorted_cache(self): + all_cache = [] + for capture_uuid in self.capture_uuids: + try: + cache = self.capture_cache(capture_uuid) + if cache and 'timestamp' in cache: + all_cache.append(cache) + except Exception: + pass + return sorted(all_cache, key=operator.itemgetter('timestamp'), reverse=True) + def capture_cache(self, capture_uuid: str) -> Dict[str, Union[str, Path]]: capture_dir = self.lookup_capture_dir(capture_uuid) if not capture_dir: diff --git a/website/web/__init__.py b/website/web/__init__.py index 4e4b5ae5..d723d917 100644 --- a/website/web/__init__.py +++ b/website/web/__init__.py @@ -378,8 +378,7 @@ def index_generic(show_hidden: bool=False): cut_time = datetime.now() - timedelta(**time_delta_on_index) else: cut_time = None # type: ignore - for capture_uuid in lookyloo.capture_uuids: - cached = lookyloo.capture_cache(capture_uuid) + for cached in lookyloo.sorted_cache: if not cached: continue if show_hidden: @@ -388,9 +387,6 @@ def index_generic(show_hidden: bool=False): continue elif 'no_index' in cached: continue - if 'timestamp' not in cached: - # this is a buggy capture, skip - continue if cut_time and datetime.fromisoformat(cached['timestamp'][:-1]) < cut_time: # type: ignore continue titles.append((cached['uuid'], cached['title'], cached['timestamp'], cached['url'],