diff --git a/lookyloo/helpers.py b/lookyloo/helpers.py index ef9adb5..87026ea 100644 --- a/lookyloo/helpers.py +++ b/lookyloo/helpers.py @@ -2,7 +2,6 @@ import hashlib import json import logging -import os import pkg_resources from datetime import datetime, timedelta @@ -10,20 +9,18 @@ from enum import IntEnum, unique from functools import lru_cache from io import BufferedIOBase from pathlib import Path -from typing import Any, Dict, List, Optional, Set, Tuple, Union -from urllib.parse import urljoin, urlparse +from typing import Any, Dict, List, Optional, Set, Union +from urllib.parse import urlparse -import requests from har2tree import CrawledTree, HostNode, URLNode from publicsuffix2 import PublicSuffixList, fetch # type: ignore from pytaxonomies import Taxonomies -from requests.exceptions import HTTPError from ua_parser import user_agent_parser # type: ignore from werkzeug.user_agent import UserAgent from werkzeug.utils import cached_property -from .default import get_homedir, safe_create_dir, get_config +from .default import get_homedir, safe_create_dir logger = logging.getLogger('Lookyloo - Helpers') @@ -135,7 +132,7 @@ def load_cookies(cookie_pseudofile: Optional[Union[BufferedIOBase, str]]=None) - 'value': cookie['Content raw'] } else: - # Cookie from lookyloo/splash + # Cookie from lookyloo/playwright to_add = cookie to_return.append(to_add) except Exception as e: @@ -157,30 +154,6 @@ def get_useragent_for_requests(): return f'Lookyloo / {version}' -@lru_cache(64) -def get_splash_url() -> str: - if os.environ.get('SPLASH_URL_DOCKER'): - # In order to have a working default for the docker image, it is easier to use an environment variable - return os.environ['SPLASH_URL_DOCKER'] - else: - return get_config('generic', 'splash_url') - - -def splash_status() -> Tuple[bool, str]: - try: - splash_status = requests.get(urljoin(get_splash_url(), '_ping')) - splash_status.raise_for_status() - json_status = splash_status.json() - if json_status['status'] == 'ok': - return True, 'Splash is up' - else: - return False, str(json_status) - except HTTPError as http_err: - return False, f'HTTP error occurred: {http_err}' - except Exception as err: - return False, f'Other error occurred: {err}' - - def get_cache_directory(root: Path, identifier: str, namespace: Optional[str] = None) -> Path: m = hashlib.md5() m.update(identifier.encode()) diff --git a/lookyloo/lookyloo.py b/lookyloo/lookyloo.py index e3b204b..2a02c85 100644 --- a/lookyloo/lookyloo.py +++ b/lookyloo/lookyloo.py @@ -28,7 +28,7 @@ from .default import LookylooException, get_homedir, get_config, get_socket_path from .exceptions import (MissingCaptureDirectory, MissingUUID, TreeNeedsRebuild, NoValidHarFile) from .helpers import (CaptureStatus, get_captures_dir, get_email_template, - get_resources_hashes, get_splash_url, get_taxonomies, + get_resources_hashes, get_taxonomies, uniq_domains, ParsedUserAgent) from .indexing import Indexing from .modules import (MISP, PhishingInitiative, UniversalWhois, @@ -48,7 +48,6 @@ class Lookyloo(): self.redis_pool: ConnectionPool = ConnectionPool(connection_class=UnixDomainSocketConnection, path=get_socket_path('cache'), decode_responses=True) self.capture_dir: Path = get_captures_dir() - self.splash_url: str = get_splash_url() self._priority = get_config('generic', 'priority') diff --git a/tools/monitoring.py b/tools/monitoring.py index 347d9ef..e98ae15 100755 --- a/tools/monitoring.py +++ b/tools/monitoring.py @@ -11,7 +11,6 @@ from rich.console import Console from rich.padding import Padding from lookyloo.default import get_socket_path, AbstractManager -from lookyloo.helpers import splash_status # NOTE: run with watch: # watch --color tools/monitoring.py @@ -80,12 +79,6 @@ class Monitoring(): if __name__ == '__main__': - status, message = splash_status() - if status: - console.print(f'[green]{message}[/green]') - else: - console.print('Splash is [bold red]down[/bold red]: ', message) - m = Monitoring() backend_up = m.backend_status if not backend_up: diff --git a/website/web/genericapi.py b/website/web/genericapi.py index 31372c3..040e323 100644 --- a/website/web/genericapi.py +++ b/website/web/genericapi.py @@ -10,7 +10,6 @@ from flask import request, send_file from flask_restx import Namespace, Resource, abort, fields # type: ignore from werkzeug.security import check_password_hash -from lookyloo.helpers import splash_status from lookyloo.lookyloo import Lookyloo from .helpers import build_users_table, load_user_from_request, src_request_ip @@ -58,14 +57,6 @@ class AuthToken(Resource): return {'error': 'User/Password invalid.'}, 401 -@api.route('/json/splash_status') -@api.doc(description='Get status of splash.') -class SplashStatus(Resource): - def get(self): - status, info = splash_status() - return {'is_up': status, 'info': info} - - @api.route('/json//status') @api.doc(description='Get the status of a capture', params={'capture_uuid': 'The UUID of the capture'})