CIRCL
/
lookyloo
mirror of https://github.com/CIRCL/lookyloo
2
0
Fork 0
Code Issues Releases Wiki Activity

fix: Speedup quick cache init

pull/920/head
Raphaël Vinot 2024-05-25 12:29:01 +02:00
parent 0fef524105
commit 0f6d5c1649
3 changed files with 10 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
bin/async_capture.py
View File

@ -30,7 +30,7 @@ class AsyncCapture(AbstractManager):
self.script_name = 'async_capture' self.script_name = 'async_capture'
self.only_global_lookups: bool = get_config('generic', 'only_global_lookups') self.only_global_lookups: bool = get_config('generic', 'only_global_lookups')
self.capture_dir: Path = get_captures_dir() self.capture_dir: Path = get_captures_dir()
self.lookyloo = Lookyloo() self.lookyloo = Lookyloo(cache_max_size=1)
self.captures: set[asyncio.Task] = set() # type: ignore[type-arg] self.captures: set[asyncio.Task] = set() # type: ignore[type-arg]

10
lookyloo/capturecache.py
View File

@ -28,7 +28,7 @@ from pyipasnhistory import IPASNHistory # type: ignore[attr-defined]
from redis import Redis from redis import Redis
from .context import Context from .context import Context
from .helpers import get_captures_dir, is_locked, make_ts_from_dirname from .helpers import get_captures_dir, is_locked
from .indexing import Indexing from .indexing import Indexing
from .default import LookylooException, try_make_file, get_config from .default import LookylooException, try_make_file, get_config
from .exceptions import MissingCaptureDirectory, NoValidHarFile, MissingUUID, TreeNeedsRebuild from .exceptions import MissingCaptureDirectory, NoValidHarFile, MissingUUID, TreeNeedsRebuild
@ -187,6 +187,7 @@ class CapturesIndex(Mapping): # type: ignore[type-arg]
self.ipasnhistory: IPASNHistory | None = IPASNHistory() self.ipasnhistory: IPASNHistory | None = IPASNHistory()
if not self.ipasnhistory.is_up: if not self.ipasnhistory.is_up:
self.ipasnhistory = None self.ipasnhistory = None
self.logger.info('IPASN History ready')
except Exception as e: except Exception as e:
# Unable to setup IPASN History # Unable to setup IPASN History
self.logger.warning(f'Unable to setup IPASN History: {e}') self.logger.warning(f'Unable to setup IPASN History: {e}')
@ -195,6 +196,7 @@ class CapturesIndex(Mapping): # type: ignore[type-arg]
self.cloudflare: Cloudflare | None = Cloudflare() self.cloudflare: Cloudflare | None = Cloudflare()
if not self.cloudflare.available: if not self.cloudflare.available:
self.cloudflare = None self.cloudflare = None
self.logger.info('Cloudflare ready')
except Exception as e: except Exception as e:
self.logger.warning(f'Unable to setup Cloudflare: {e}') self.logger.warning(f'Unable to setup Cloudflare: {e}')
self.cloudflare = None self.cloudflare = None
@ -266,7 +268,6 @@ class CapturesIndex(Mapping): # type: ignore[type-arg]
continue continue
has_new_cached_captures = True has_new_cached_captures = True
p.hgetall(directory) p.hgetall(directory)
recent_captures[uuid] = make_ts_from_dirname(directory.rsplit('/', 1)[-1]).timestamp()
if not has_new_cached_captures: if not has_new_cached_captures:
return return
for cache in p.execute(): for cache in p.execute():
@ -278,7 +279,10 @@ class CapturesIndex(Mapping): # type: ignore[type-arg]
self.logger.warning(f'Unable to initialize the cache: {e}') self.logger.warning(f'Unable to initialize the cache: {e}')
continue continue
self.__cache[cc.uuid] = cc self.__cache[cc.uuid] = cc
self.redis.zadd('recent_captures', recent_captures) if hasattr(cc, 'timestamp'):
recent_captures[uuid] = cc.timestamp.timestamp()
if recent_captures:
self.redis.zadd('recent_captures', recent_captures)
def _get_capture_dir(self, uuid: str) -> str: def _get_capture_dir(self, uuid: str) -> str:
# Try to get from the recent captures cache in redis # Try to get from the recent captures cache in redis

4
website/web/__init__.py
View File

@ -80,14 +80,14 @@ user_agents = UserAgents()
if get_config('generic', 'index_is_capture'): if get_config('generic', 'index_is_capture'):
@app.route('/', methods=['GET']) @app.route('/', methods=['GET'])
def landing_page() -> WerkzeugResponse: def landing_page() -> WerkzeugResponse | str:
if request.method == 'HEAD': if request.method == 'HEAD':
# Just returns ack if the webserver is running # Just returns ack if the webserver is running
return 'Ack' return 'Ack'
return redirect(url_for('capture_web')) return redirect(url_for('capture_web'))
else: else:
@app.route('/', methods=['GET']) @app.route('/', methods=['GET'])
def landing_page() -> WerkzeugResponse: def landing_page() -> WerkzeugResponse | str:
if request.method == 'HEAD': if request.method == 'HEAD':
# Just returns ack if the webserver is running # Just returns ack if the webserver is running
return 'Ack' return 'Ack'