diff --git a/bin/archiver.py b/bin/archiver.py index 1987788..01d320a 100755 --- a/bin/archiver.py +++ b/bin/archiver.py @@ -10,9 +10,8 @@ from typing import Dict, List from redis import Redis -from lookyloo.abstractmanager import AbstractManager -from lookyloo.helpers import (get_captures_dir, get_config, get_homedir, - get_socket_path) +from lookyloo.default import AbstractManager, get_config, get_homedir, get_socket_path +from lookyloo.helpers import get_captures_dir logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s:%(message)s', level=logging.INFO) diff --git a/bin/async_capture.py b/bin/async_capture.py index c4f7154..361519e 100755 --- a/bin/async_capture.py +++ b/bin/async_capture.py @@ -16,10 +16,8 @@ from defang import refang # type: ignore from redis import Redis from scrapysplashwrapper import crawl -from lookyloo.abstractmanager import AbstractManager -from lookyloo.helpers import (get_captures_dir, get_config, get_socket_path, - get_splash_url, load_cookies, safe_create_dir, - splash_status) +from lookyloo.default import AbstractManager, get_config, get_socket_path, safe_create_dir +from lookyloo.helpers import get_captures_dir, get_splash_url, load_cookies, splash_status logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s:%(message)s', level=logging.INFO) diff --git a/bin/background_indexer.py b/bin/background_indexer.py index 45566b0..ab428b8 100755 --- a/bin/background_indexer.py +++ b/bin/background_indexer.py @@ -5,7 +5,7 @@ import logging import os from datetime import datetime, timedelta -from lookyloo.abstractmanager import AbstractManager +from lookyloo.default import AbstractManager from lookyloo.exceptions import MissingUUID, NoValidHarFile from lookyloo.lookyloo import Lookyloo diff --git a/bin/background_processing.py b/bin/background_processing.py index 11f0fb6..d699e2d 100755 --- a/bin/background_processing.py +++ b/bin/background_processing.py @@ -10,9 +10,7 @@ from typing import Any, Dict from redis import Redis from werkzeug.useragents import UserAgent -from lookyloo.abstractmanager import AbstractManager -from lookyloo.helpers import (get_config, get_homedir, get_socket_path, - safe_create_dir) +from lookyloo.default import AbstractManager, get_config, get_homedir, get_socket_path, safe_create_dir logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s:%(message)s', level=logging.INFO) diff --git a/bin/run_backend.py b/bin/run_backend.py index b6ed280..a48a5fb 100755 --- a/bin/run_backend.py +++ b/bin/run_backend.py @@ -11,7 +11,7 @@ from typing import List, Optional, Union from redis import Redis from redis.exceptions import ConnectionError -from lookyloo.helpers import get_homedir, get_socket_path +from lookyloo.default import get_homedir, get_socket_path def check_running(name: str) -> bool: diff --git a/bin/shutdown.py b/bin/shutdown.py index 2dab6f7..21367d0 100755 --- a/bin/shutdown.py +++ b/bin/shutdown.py @@ -3,7 +3,7 @@ import time -from lookyloo.abstractmanager import AbstractManager +from lookyloo.default import AbstractManager def main(): diff --git a/bin/start.py b/bin/start.py index fb8bab1..0df27d4 100755 --- a/bin/start.py +++ b/bin/start.py @@ -3,7 +3,7 @@ from subprocess import Popen, run -from lookyloo.helpers import get_config, get_homedir +from lookyloo.default import get_config, get_homedir def main(): diff --git a/bin/start_website.py b/bin/start_website.py index 3962716..6924e1b 100755 --- a/bin/start_website.py +++ b/bin/start_website.py @@ -4,8 +4,7 @@ import logging from subprocess import Popen -from lookyloo.abstractmanager import AbstractManager -from lookyloo.helpers import get_config, get_homedir +from lookyloo.default import get_config, get_homedir, AbstractManager logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s:%(message)s', level=logging.INFO) diff --git a/bin/stop.py b/bin/stop.py index 9bf7c1f..b45f3aa 100755 --- a/bin/stop.py +++ b/bin/stop.py @@ -5,7 +5,7 @@ from subprocess import Popen from redis import Redis -from lookyloo.helpers import get_homedir, get_socket_path +from lookyloo.default import get_homedir, get_socket_path def main(): diff --git a/bin/update.py b/bin/update.py index 6f3c232..4d7bb86 100755 --- a/bin/update.py +++ b/bin/update.py @@ -10,7 +10,7 @@ import subprocess import sys from pathlib import Path -from lookyloo.helpers import get_homedir +from lookyloo.default import get_homedir logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s:%(message)s', level=logging.INFO) diff --git a/config/generic.json.sample b/config/generic.json.sample index 49a321e..ab75bd1 100644 --- a/config/generic.json.sample +++ b/config/generic.json.sample @@ -6,6 +6,7 @@ "public_domain": "lookyloo.myorg.local", "website_listen_ip": "0.0.0.0", "website_listen_port": 5100, + "systemd_service_name": "lookyloo", "splash_url": "http://127.0.0.1:8050", "default_user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36", "users": {}, @@ -52,6 +53,7 @@ "public_domain": "Domain where the instance can be reached. Used for permalinks (e-mail, MISP export).", "website_listen_ip": "IP Flask will listen on. Defaults to 0.0.0.0, meaning all interfaces.", "website_listen_port": "Port Flask will listen on.", + "systemd_service_name": "(Optional) Name of the systemd service if your project has one.", "splash_url": "URL to connect to splash", "default_user_agent": "Ultimate fallback if the capture form, or the asynchronous submission, doesn't provide a user agent.", "users": "It is some kind of an admin accounts. Format: {username: password}", diff --git a/lookyloo/capturecache.py b/lookyloo/capturecache.py index 5cf1dc0..8a3d22b 100644 --- a/lookyloo/capturecache.py +++ b/lookyloo/capturecache.py @@ -20,9 +20,8 @@ from redis import Redis from .context import Context from .indexing import Indexing -from .exceptions import (LookylooException, MissingCaptureDirectory, NoValidHarFile, - MissingUUID, TreeNeedsRebuild) -from .helpers import try_make_file, get_config +from .default import LookylooException, try_make_file, get_config +from .exceptions import MissingCaptureDirectory, NoValidHarFile, MissingUUID, TreeNeedsRebuild class CaptureCache(): diff --git a/lookyloo/context.py b/lookyloo/context.py index 655796c..6023c20 100644 --- a/lookyloo/context.py +++ b/lookyloo/context.py @@ -10,8 +10,8 @@ from urllib.parse import urlsplit from har2tree import CrawledTree, HostNode, URLNode from redis import Redis -from .helpers import (get_config, get_homedir, get_resources_hashes, - get_socket_path, load_known_content, serialize_to_json) +from .default import get_config, get_homedir, get_socket_path +from .helpers import get_resources_hashes, load_known_content, serialize_to_json from .modules import SaneJavaScript diff --git a/lookyloo/default/__init__.py b/lookyloo/default/__init__.py new file mode 100644 index 0000000..169577b --- /dev/null +++ b/lookyloo/default/__init__.py @@ -0,0 +1,14 @@ +env_global_name: str = 'LOOKYLOO_HOME' + +from .exceptions import LookylooException # noqa + +# NOTE: the imports below are there to avoid too long paths when importing the +# classes/methods in the rest of the project while keeping all that in a subdirectory +# and allow to update them easily. +# You should not have to change anything in this file below this line. + +from .abstractmanager import AbstractManager # noqa + +from .exceptions import MissingEnv, CreateDirectoryException, ConfigError # noqa + +from .helpers import get_homedir, load_configs, get_config, safe_create_dir, get_socket_path, try_make_file # noqa diff --git a/lookyloo/abstractmanager.py b/lookyloo/default/abstractmanager.py similarity index 87% rename from lookyloo/abstractmanager.py rename to lookyloo/default/abstractmanager.py index cfbd4b5..f33a6f6 100644 --- a/lookyloo/abstractmanager.py +++ b/lookyloo/default/abstractmanager.py @@ -29,13 +29,20 @@ class AbstractManager(ABC): @staticmethod def is_running() -> List[Tuple[str, float]]: - r = Redis(unix_socket_path=get_socket_path('cache'), db=1, decode_responses=True) - return r.zrangebyscore('running', '-inf', '+inf', withscores=True) + try: + r = Redis(unix_socket_path=get_socket_path('cache'), db=1, decode_responses=True) + return r.zrangebyscore('running', '-inf', '+inf', withscores=True) + except ConnectionError: + print('Unable to connect to redis, the system is down.') + return [] @staticmethod def force_shutdown(): - r = Redis(unix_socket_path=get_socket_path('cache'), db=1, decode_responses=True) - r.set('shutdown', 1) + try: + r = Redis(unix_socket_path=get_socket_path('cache'), db=1, decode_responses=True) + r.set('shutdown', 1) + except ConnectionError: + print('Unable to connect to redis, the system is down.') def set_running(self) -> None: self.__redis.zincrby('running', 1, self.script_name) diff --git a/lookyloo/default/exceptions.py b/lookyloo/default/exceptions.py new file mode 100644 index 0000000..17a24c7 --- /dev/null +++ b/lookyloo/default/exceptions.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + + +class LookylooException(Exception): + pass + + +class MissingEnv(LookylooException): + pass + + +class CreateDirectoryException(LookylooException): + pass + + +class ConfigError(LookylooException): + pass diff --git a/lookyloo/default/helpers.py b/lookyloo/default/helpers.py new file mode 100644 index 0000000..031e054 --- /dev/null +++ b/lookyloo/default/helpers.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +import json +import logging +import os +from functools import lru_cache +from pathlib import Path +from typing import Any, Dict, Optional, Union + +from . import env_global_name +from .exceptions import ConfigError, CreateDirectoryException, MissingEnv + +configs: Dict[str, Dict[str, Any]] = {} +logger = logging.getLogger('Helpers') + + +@lru_cache(64) +def get_homedir() -> Path: + if not os.environ.get(env_global_name): + # Try to open a .env file in the home directory if it exists. + if (Path(__file__).resolve().parent.parent.parent / '.env').exists(): + with (Path(__file__).resolve().parent.parent.parent / '.env').open() as f: + for line in f: + key, value = line.strip().split('=', 1) + if value[0] in ['"', "'"]: + value = value[1:-1] + os.environ[key] = value + + if not os.environ.get(env_global_name): + guessed_home = Path(__file__).resolve().parent.parent.parent + raise MissingEnv(f"{env_global_name} is missing. \ +Run the following command (assuming you run the code from the clonned repository):\ + export {env_global_name}='{guessed_home}'") + return Path(os.environ[env_global_name]) + + +@lru_cache(64) +def load_configs(path_to_config_files: Optional[Union[str, Path]]=None): + global configs + if configs: + return + if path_to_config_files: + if isinstance(path_to_config_files, str): + config_path = Path(path_to_config_files) + else: + config_path = path_to_config_files + else: + config_path = get_homedir() / 'config' + if not config_path.exists(): + raise ConfigError(f'Configuration directory {config_path} does not exists.') + elif not config_path.is_dir(): + raise ConfigError(f'Configuration directory {config_path} is not a directory.') + + configs = {} + for path in config_path.glob('*.json'): + with path.open() as _c: + configs[path.stem] = json.load(_c) + + +@lru_cache(64) +def get_config(config_type: str, entry: str, quiet: bool=False) -> Any: + """Get an entry from the given config_type file. Automatic fallback to the sample file""" + global configs + if not configs: + load_configs() + if config_type in configs: + if entry in configs[config_type]: + return configs[config_type][entry] + else: + if not quiet: + logger.warning(f'Unable to find {entry} in config file.') + else: + if not quiet: + logger.warning(f'No {config_type} config file available.') + if not quiet: + logger.warning(f'Falling back on sample config, please initialize the {config_type} config file.') + with (get_homedir() / 'config' / f'{config_type}.json.sample').open() as _c: + sample_config = json.load(_c) + return sample_config[entry] + + +def safe_create_dir(to_create: Path) -> None: + if to_create.exists() and not to_create.is_dir(): + raise CreateDirectoryException(f'The path {to_create} already exists and is not a directory') + to_create.mkdir(parents=True, exist_ok=True) + + +def get_socket_path(name: str) -> str: + mapping = { + 'cache': Path('cache', 'cache.sock'), + 'indexing': Path('indexing', 'indexing.sock'), + } + return str(get_homedir() / mapping[name]) + + +def try_make_file(filename: Path): + try: + filename.touch(exist_ok=False) + return True + except FileExistsError: + return False diff --git a/lookyloo/exceptions.py b/lookyloo/exceptions.py index d6da41b..d48ccad 100644 --- a/lookyloo/exceptions.py +++ b/lookyloo/exceptions.py @@ -1,27 +1,13 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- - -class LookylooException(Exception): - pass - - -class MissingEnv(LookylooException): - pass +from .default import LookylooException class NoValidHarFile(LookylooException): pass -class CreateDirectoryException(LookylooException): - pass - - -class ConfigError(LookylooException): - pass - - class MissingUUID(LookylooException): pass diff --git a/lookyloo/helpers.py b/lookyloo/helpers.py index 5b551ec..6e10bc3 100644 --- a/lookyloo/helpers.py +++ b/lookyloo/helpers.py @@ -18,9 +18,8 @@ from publicsuffix2 import PublicSuffixList, fetch # type: ignore from pytaxonomies import Taxonomies from requests.exceptions import HTTPError -from .exceptions import ConfigError, CreateDirectoryException, MissingEnv +from .default import get_homedir, safe_create_dir, get_config -configs: Dict[str, Dict[str, Any]] = {} logger = logging.getLogger('Lookyloo - Helpers') @@ -71,26 +70,6 @@ def get_public_suffix_list(): return psl -@lru_cache(64) -def get_homedir() -> Path: - if not os.environ.get('LOOKYLOO_HOME'): - # Try to open a .env file in the home directory if it exists. - if (Path(__file__).resolve().parent.parent / '.env').exists(): - with (Path(__file__).resolve().parent.parent / '.env').open() as f: - for line in f: - key, value = line.strip().split('=', 1) - if value[0] in ['"', "'"]: - value = value[1:-1] - os.environ[key] = value - - if not os.environ.get('LOOKYLOO_HOME'): - guessed_home = Path(__file__).resolve().parent.parent - raise MissingEnv(f"LOOKYLOO_HOME is missing. \ -Run the following command (assuming you run the code from the clonned repository):\ - export LOOKYLOO_HOME='{guessed_home}'") - return Path(os.environ['LOOKYLOO_HOME']) - - @lru_cache(64) def get_captures_dir() -> Path: capture_dir = get_homedir() / 'scraped' @@ -104,66 +83,6 @@ def get_email_template() -> str: return f.read() -@lru_cache(64) -def load_configs(path_to_config_files: Optional[Union[str, Path]]=None): - global configs - if configs: - return - if path_to_config_files: - if isinstance(path_to_config_files, str): - config_path = Path(path_to_config_files) - else: - config_path = path_to_config_files - else: - config_path = get_homedir() / 'config' - if not config_path.exists(): - raise ConfigError(f'Configuration directory {config_path} does not exists.') - elif not config_path.is_dir(): - raise ConfigError(f'Configuration directory {config_path} is not a directory.') - - configs = {} - for path in config_path.glob('*.json'): - with path.open() as _c: - configs[path.stem] = json.load(_c) - - -@lru_cache(64) -def get_config(config_type: str, entry: str, quiet: bool=False) -> Any: - """Get an entry from the given config_type file. Automatic fallback to the sample file""" - global configs - if not configs: - load_configs() - if config_type in configs: - if entry in configs[config_type]: - return configs[config_type][entry] - else: - if not quiet: - logger.warning(f'Unable to find {entry} in config file.') - else: - if not quiet: - logger.warning(f'No {config_type} config file available.') - if not quiet: - logger.warning(f'Falling back on sample config, please initialize the {config_type} config file.') - with (get_homedir() / 'config' / f'{config_type}.json.sample').open() as _c: - sample_config = json.load(_c) - return sample_config[entry] - - -def safe_create_dir(to_create: Path) -> None: - if to_create.exists() and not to_create.is_dir(): - raise CreateDirectoryException(f'The path {to_create} already exists and is not a directory') - to_create.mkdir(parents=True, exist_ok=True) - - -def get_socket_path(name: str) -> str: - mapping = { - 'cache': Path('cache', 'cache.sock'), - 'indexing': Path('indexing', 'indexing.sock'), - 'storage': Path('storage', 'storage.sock'), - } - return str(get_homedir() / mapping[name]) - - def get_user_agents(directory: str='user_agents') -> Dict[str, Any]: ua_files_path = sorted((get_homedir() / directory).glob('**/*.json'), reverse=True) with ua_files_path[0].open() as f: @@ -223,14 +142,6 @@ def uniq_domains(uniq_urls): return domains -def try_make_file(filename: Path): - try: - filename.touch(exist_ok=False) - return True - except FileExistsError: - return False - - @lru_cache(64) def get_useragent_for_requests(): version = pkg_resources.get_distribution('lookyloo').version diff --git a/lookyloo/indexing.py b/lookyloo/indexing.py index 6600922..52f0073 100644 --- a/lookyloo/indexing.py +++ b/lookyloo/indexing.py @@ -12,7 +12,8 @@ from har2tree import CrawledTree from redis import ConnectionPool, Redis from redis.connection import UnixDomainSocketConnection -from .helpers import get_public_suffix_list, get_socket_path, get_config +from .default import get_socket_path, get_config +from .helpers import get_public_suffix_list class Indexing(): diff --git a/lookyloo/lookyloo.py b/lookyloo/lookyloo.py index 7f2a8ad..88ef294 100644 --- a/lookyloo/lookyloo.py +++ b/lookyloo/lookyloo.py @@ -25,11 +25,11 @@ from werkzeug.useragents import UserAgent from .capturecache import CaptureCache, CapturesIndex from .context import Context -from .exceptions import (LookylooException, MissingCaptureDirectory, +from .default import LookylooException, get_homedir, get_config, get_socket_path +from .exceptions import (MissingCaptureDirectory, MissingUUID, TreeNeedsRebuild, NoValidHarFile) -from .helpers import (CaptureStatus, get_captures_dir, get_config, - get_email_template, get_homedir, get_resources_hashes, - get_socket_path, get_splash_url, get_taxonomies, uniq_domains) +from .helpers import (CaptureStatus, get_captures_dir, get_email_template, + get_resources_hashes, get_splash_url, get_taxonomies, uniq_domains) from .indexing import Indexing from .modules import (MISP, PhishingInitiative, UniversalWhois, UrlScan, VirusTotal, Phishtank) diff --git a/lookyloo/modules/misp.py b/lookyloo/modules/misp.py index 82e6741..e316153 100644 --- a/lookyloo/modules/misp.py +++ b/lookyloo/modules/misp.py @@ -11,7 +11,8 @@ from har2tree import HostNode, URLNode, Har2TreeError from pymisp import MISPAttribute, MISPEvent, PyMISP from pymisp.tools import FileObject, URLObject -from ..helpers import get_config, get_homedir, get_public_suffix_list +from ..default import get_config, get_homedir +from ..helpers import get_public_suffix_list if TYPE_CHECKING: from ..capturecache import CaptureCache diff --git a/lookyloo/modules/phishtank.py b/lookyloo/modules/phishtank.py index 13a4c4f..5fe5a3c 100644 --- a/lookyloo/modules/phishtank.py +++ b/lookyloo/modules/phishtank.py @@ -10,8 +10,7 @@ from typing import Any, Dict, Optional, List from har2tree import CrawledTree from pyphishtanklookup import PhishtankLookup -from ..exceptions import ConfigError -from ..helpers import get_homedir +from ..default import ConfigError, get_homedir # Note: stop doing requests 48 after the capture was intially done. diff --git a/lookyloo/modules/pi.py b/lookyloo/modules/pi.py index 116ccea..6abfa3a 100644 --- a/lookyloo/modules/pi.py +++ b/lookyloo/modules/pi.py @@ -11,8 +11,7 @@ from typing import Any, Dict, Optional from har2tree import CrawledTree from pyeupi import PyEUPI -from ..exceptions import ConfigError -from ..helpers import get_homedir +from ..default import ConfigError, get_homedir class PhishingInitiative(): diff --git a/lookyloo/modules/sanejs.py b/lookyloo/modules/sanejs.py index 26054ab..1eed3f5 100644 --- a/lookyloo/modules/sanejs.py +++ b/lookyloo/modules/sanejs.py @@ -8,7 +8,7 @@ from typing import Any, Dict, Iterable, List, Union from pysanejs import SaneJS -from ..helpers import get_config, get_homedir +from ..default import get_config, get_homedir class SaneJavaScript(): diff --git a/lookyloo/modules/urlscan.py b/lookyloo/modules/urlscan.py index c049468..7384d30 100644 --- a/lookyloo/modules/urlscan.py +++ b/lookyloo/modules/urlscan.py @@ -10,8 +10,8 @@ from typing import Any, Dict import requests -from ..exceptions import ConfigError -from ..helpers import get_config, get_homedir, get_useragent_for_requests +from ..default import ConfigError, get_config, get_homedir +from ..helpers import get_useragent_for_requests class UrlScan(): diff --git a/lookyloo/modules/uwhois.py b/lookyloo/modules/uwhois.py index 1e56f09..49f3e1c 100644 --- a/lookyloo/modules/uwhois.py +++ b/lookyloo/modules/uwhois.py @@ -7,7 +7,7 @@ from typing import Any, Dict from har2tree import CrawledTree, Har2TreeError, HostNode -from ..helpers import get_config +from ..default import get_config class UniversalWhois(): diff --git a/lookyloo/modules/vt.py b/lookyloo/modules/vt.py index 09eb90f..443b463 100644 --- a/lookyloo/modules/vt.py +++ b/lookyloo/modules/vt.py @@ -12,8 +12,7 @@ import vt # type: ignore from har2tree import CrawledTree from vt.error import APIError # type: ignore -from ..exceptions import ConfigError -from ..helpers import get_homedir +from ..default import ConfigError, get_homedir class VirusTotal(): diff --git a/tools/3rdparty.py b/tools/3rdparty.py index e13d674..43f5a8e 100755 --- a/tools/3rdparty.py +++ b/tools/3rdparty.py @@ -3,7 +3,7 @@ import requests -from lookyloo.helpers import get_homedir +from lookyloo.default import get_homedir d3js_version = '7' datatables_version = "1.11.3" diff --git a/tools/change_captures_dir.py b/tools/change_captures_dir.py index 88a5510..0c3ec56 100755 --- a/tools/change_captures_dir.py +++ b/tools/change_captures_dir.py @@ -6,7 +6,8 @@ from pathlib import Path from redis import Redis -from lookyloo.helpers import get_captures_dir, safe_create_dir, get_socket_path +from lookyloo.default import safe_create_dir, get_socket_path +from lookyloo.helpers import get_captures_dir def rename_captures(): diff --git a/tools/generate_sri.py b/tools/generate_sri.py index a0ae8a7..0179eda 100755 --- a/tools/generate_sri.py +++ b/tools/generate_sri.py @@ -7,7 +7,7 @@ import json from typing import Dict -from lookyloo.helpers import get_homedir +from lookyloo.default import get_homedir if __name__ == '__main__': dest_dir = get_homedir() / 'website' / 'web' diff --git a/tools/manual_parse_ua_list.py b/tools/manual_parse_ua_list.py index 4c6dcbb..9001c79 100644 --- a/tools/manual_parse_ua_list.py +++ b/tools/manual_parse_ua_list.py @@ -14,7 +14,7 @@ try: except ImportError: HAS_CF = False -from lookyloo.helpers import get_homedir, safe_create_dir +from lookyloo.default import get_homedir, safe_create_dir def update_user_agents() -> None: @@ -63,6 +63,7 @@ def ua_parser(html_content: str) -> Dict[str, Any]: to_store['by_frequency'].append({'os': os, 'browser': browser, 'useragent': ua['useragent']}) return to_store + def main(): to_parse = Path('Most Common User Agents - Tech Blog (wh).html') @@ -77,5 +78,6 @@ def main(): with open(ua_file_name, 'w') as f: json.dump(to_store, f, indent=2) + if __name__ == '__main__': main() diff --git a/tools/monitoring.py b/tools/monitoring.py index 73d4e61..a193b00 100755 --- a/tools/monitoring.py +++ b/tools/monitoring.py @@ -11,8 +11,8 @@ from redis.exceptions import ConnectionError from rich.console import Console from rich.padding import Padding -from lookyloo.helpers import get_socket_path, splash_status -from lookyloo.abstractmanager import AbstractManager +from lookyloo.default import get_socket_path, AbstractManager +from lookyloo.helpers import splash_status # NOTE: run with watch: # watch --color tools/monitoring.py diff --git a/tools/validate_config_files.py b/tools/validate_config_files.py index 82e11c5..c637b19 100755 --- a/tools/validate_config_files.py +++ b/tools/validate_config_files.py @@ -5,7 +5,7 @@ import json import logging import argparse -from lookyloo.helpers import get_homedir +from lookyloo.default import get_homedir def validate_generic_config_file(): diff --git a/website/web/__init__.py b/website/web/__init__.py index a002dee..b5434a3 100644 --- a/website/web/__init__.py +++ b/website/web/__init__.py @@ -21,8 +21,9 @@ from flask_restx import Api # type: ignore from pymisp import MISPEvent, MISPServerError from werkzeug.security import check_password_hash +from lookyloo.default import get_config from lookyloo.exceptions import MissingUUID, NoValidHarFile -from lookyloo.helpers import (CaptureStatus, get_config, get_taxonomies, +from lookyloo.helpers import (CaptureStatus, get_taxonomies, get_user_agents, load_cookies, splash_status) from lookyloo.lookyloo import Indexing, Lookyloo diff --git a/website/web/helpers.py b/website/web/helpers.py index a2243c9..2c2e38f 100644 --- a/website/web/helpers.py +++ b/website/web/helpers.py @@ -11,7 +11,7 @@ from typing import Dict, List, Union import flask_login # type: ignore from werkzeug.security import generate_password_hash -from lookyloo.helpers import get_config, get_homedir +from lookyloo.default import get_config, get_homedir def src_request_ip(request) -> str: