From 0718c6cb39096845671f4501d553c97e52a4ff0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 21 Nov 2024 15:51:22 +0100 Subject: [PATCH] fix: Trigger autoreport once the tree is built --- bin/async_capture.py | 14 ++------------ bin/background_build_captures.py | 24 ++++++++++++++++++++++++ lookyloo/helpers.py | 2 +- lookyloo/lookyloo.py | 18 +++++++++++++----- website/web/__init__.py | 1 - website/web/templates/capture.html | 8 -------- 6 files changed, 40 insertions(+), 27 deletions(-) diff --git a/bin/async_capture.py b/bin/async_capture.py index 070c5471..f50a92be 100755 --- a/bin/async_capture.py +++ b/bin/async_capture.py @@ -98,20 +98,10 @@ class AsyncCapture(AbstractManager): last_redirected_url=entries.get('last_redirected_url'), cookies=entries.get('cookies'), capture_settings=to_capture, - potential_favicons=entries.get('potential_favicons') + potential_favicons=entries.get('potential_favicons'), + auto_report=to_capture.auto_report, ) - if to_capture.auto_report: - send_report = True - settings = {} - if isinstance(to_capture.auto_report, dict): - settings = to_capture.auto_report - - if send_report: - self.lookyloo.send_mail(uuid, email=settings.get('email', ''), - comment=settings.get('comment'), - recipient_mail=settings.get("recipient_mail")) - lazy_cleanup = self.lookyloo.redis.pipeline() if queue and self.lookyloo.redis.zscore('queues', queue): lazy_cleanup.zincrby('queues', -1, queue) diff --git a/bin/background_build_captures.py b/bin/background_build_captures.py index abf27cee..2c19ff77 100755 --- a/bin/background_build_captures.py +++ b/bin/background_build_captures.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json import logging import logging.config import shutil @@ -34,6 +35,24 @@ class BackgroundBuildCaptures(AbstractManager): # Redis connector so we don't use the one from Lookyloo self.redis = Redis(unix_socket_path=get_socket_path('cache'), decode_responses=True) + def __auto_report(self, path: Path) -> None: + with (path / 'uuid').open() as f: + capture_uuid = f.read() + self.logger.info(f'Triggering autoreport for {capture_uuid}...') + settings = {} + with (path / 'auto_report').open() as f: + if ar := f.read(): + # could be an empty file. + settings = json.loads(ar) + try: + self.lookyloo.send_mail(capture_uuid, email=settings.get('email', ''), + comment=settings.get('comment')) + (path / 'auto_report').unlink() + except Exception as e: + self.logger.warning(f'Unable to send auto report for {capture_uuid}: {e}') + else: + self.logger.info(f'Auto report for {capture_uuid} sent.') + def _to_run_forever(self) -> None: self._build_missing_pickles() # Don't need the cache in this class. @@ -59,6 +78,9 @@ class BackgroundBuildCaptures(AbstractManager): if ((path / 'tree.pickle.gz').exists() or (path / 'tree.pickle').exists()): # We already have a pickle file self.logger.debug(f'{path} has a pickle.') + if (path / 'auto_report').exists(): + # the pickle was built somewhere else, trigger report. + self.__auto_report(path) continue if not list(path.rglob('*.har.gz')) and not list(path.rglob('*.har')): # No HAR file @@ -102,6 +124,8 @@ class BackgroundBuildCaptures(AbstractManager): self.logger.info(f'Pickle for {uuid} built.') got_new_captures = True max_captures -= 1 + if (path / 'auto_report').exists(): + self.__auto_report(path) except MissingUUID: self.logger.warning(f'Unable to find {uuid}. That should not happen.') except NoValidHarFile as e: diff --git a/lookyloo/helpers.py b/lookyloo/helpers.py index 25f4a29a..67f657c5 100644 --- a/lookyloo/helpers.py +++ b/lookyloo/helpers.py @@ -408,7 +408,7 @@ class CaptureSettings(LacuscoreCaptureSettings): '''The capture settings that can be passed to Lookyloo''' listing: bool = get_config('generic', 'default_public') not_queued: bool = False - auto_report: bool | dict[str, str] | None = None # {'email': , 'comment': , 'recipient_mail':} + auto_report: bool | dict[str, str] | None = None # {'email': , 'comment':} dnt: str | None = None browser_name: str | None = None os: str | None = None diff --git a/lookyloo/lookyloo.py b/lookyloo/lookyloo.py index 0c285d51..d43bc8eb 100644 --- a/lookyloo/lookyloo.py +++ b/lookyloo/lookyloo.py @@ -884,12 +884,11 @@ class Lookyloo(): # unable to run the query, probably an invalid key pass if len(modules) == 0: - return "Capture does not seem to be malicious" + return "URL captured doesn't appear in malicious databases." return f"Malicious capture according to {len(modules)} module(s): {', '.join(modules)}" - def send_mail(self, capture_uuid: str, /, email: str='', comment: str | None=None, - recipient_mail: str | None = None) -> bool | dict[str, Any]: + def send_mail(self, capture_uuid: str, /, email: str | None=None, comment: str | None=None) -> bool | dict[str, Any]: '''Send an email notification regarding a specific capture''' if not get_config('generic', 'enable_mail_notification'): return {"error": "Unable to send mail: mail notification disabled"} @@ -938,7 +937,7 @@ class Lookyloo(): msg['From'] = email_config['from'] if email: msg['Reply-To'] = email - msg['To'] = email_config['to'] if not recipient_mail else recipient_mail + msg['To'] = email_config['to'] msg['Subject'] = email_config['subject'] body = get_email_template() body = body.format( @@ -1519,7 +1518,8 @@ class Lookyloo(): last_redirected_url: str | None=None, cookies: list[Cookie] | list[dict[str, str]] | None=None, capture_settings: CaptureSettings | None=None, - potential_favicons: set[bytes] | None=None + potential_favicons: set[bytes] | None=None, + auto_report: bool | dict[str, str] | None = None ) -> None: now = datetime.now() @@ -1595,5 +1595,13 @@ class Lookyloo(): with (dirpath / f'{f_id}.potential_favicons.ico').open('wb') as _fw: _fw.write(favicon) + if auto_report: + # autoreport needs to be triggered once the tree is build + if isinstance(auto_report, bool): + (dirpath / 'auto_report').touch() + else: + with (dirpath / 'auto_report').open('w') as _ar: + json.dump(auto_report, _ar) + self.redis.hset('lookup_dirs', uuid, str(dirpath)) self.redis.zadd('recent_captures', {uuid: now.timestamp()}) diff --git a/website/web/__init__.py b/website/web/__init__.py index 272a58c3..2df7d221 100644 --- a/website/web/__init__.py +++ b/website/web/__init__.py @@ -1659,7 +1659,6 @@ def capture_web() -> str | Response | WerkzeugResponse: capture_query['auto_report'] = { 'email': request.form.get('email', ""), 'comment': request.form.get('comment', ""), - 'recipient_mail': request.form.get('recipient-mail', "") } if request.form.get('url'): diff --git a/website/web/templates/capture.html b/website/web/templates/capture.html index b625f92c..fc1c5432 100644 --- a/website/web/templates/capture.html +++ b/website/web/templates/capture.html @@ -454,14 +454,6 @@ name="email" placeholder="myself@example.com"> -
- -
- -
-