From 00370291ac798c8d0b6c9edc2211b478196057f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 23 Nov 2022 15:54:22 +0100 Subject: [PATCH] new: Logging config in file --- bin/archiver.py | 4 ++-- bin/async_capture.py | 8 +++----- bin/background_indexer.py | 7 ++++--- bin/background_processing.py | 4 ++-- bin/start_website.py | 4 ++-- bin/update.py | 6 +++--- config/logging.json.sample | 33 +++++++++++++++++++++++++++++++++ logs/.keepdir | 0 lookyloo/default/helpers.py | 17 +++++++++++------ website/web/__init__.py | 2 -- 10 files changed, 60 insertions(+), 25 deletions(-) create mode 100644 config/logging.json.sample create mode 100644 logs/.keepdir diff --git a/bin/archiver.py b/bin/archiver.py index fac00b04..18a91182 100755 --- a/bin/archiver.py +++ b/bin/archiver.py @@ -3,6 +3,7 @@ import csv import gzip import logging +import logging.config import shutil from collections import defaultdict @@ -16,8 +17,7 @@ from redis import Redis from lookyloo.default import AbstractManager, get_config, get_homedir, get_socket_path from lookyloo.helpers import get_captures_dir -logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s:%(message)s', - level=logging.INFO) +logging.config.dictConfig(get_config('logging')) class Archiver(AbstractManager): diff --git a/bin/async_capture.py b/bin/async_capture.py index 19456c4a..beb0fe26 100755 --- a/bin/async_capture.py +++ b/bin/async_capture.py @@ -1,12 +1,11 @@ #!/usr/bin/env python3 import asyncio -import json import logging +import logging.config import signal import time -from datetime import datetime from pathlib import Path from typing import Dict, Optional, Union @@ -14,13 +13,12 @@ from lacuscore import LacusCore, CaptureStatus as CaptureStatusCore, CaptureResp from pylacus import CaptureStatus as CaptureStatusPy, CaptureResponse as CaptureResponsePy from lookyloo.lookyloo import Lookyloo -from lookyloo.default import AbstractManager, get_config, safe_create_dir +from lookyloo.default import AbstractManager, get_config from lookyloo.helpers import get_captures_dir from lookyloo.modules import FOX -logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s:%(message)s', - level=logging.INFO) +logging.config.dictConfig(get_config('logging')) class AsyncCapture(AbstractManager): diff --git a/bin/background_indexer.py b/bin/background_indexer.py index 638542d7..ef945646 100755 --- a/bin/background_indexer.py +++ b/bin/background_indexer.py @@ -1,16 +1,17 @@ #!/usr/bin/env python3 import logging +import logging.config import os from datetime import datetime, timedelta import shutil -from lookyloo.default import AbstractManager +from lookyloo.default import AbstractManager, get_config from lookyloo.exceptions import MissingUUID, NoValidHarFile from lookyloo.lookyloo import Lookyloo -logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s:%(message)s', - level=logging.INFO) + +logging.config.dictConfig(get_config('logging')) class BackgroundIndexer(AbstractManager): diff --git a/bin/background_processing.py b/bin/background_processing.py index a44c5bd7..7ea15e07 100755 --- a/bin/background_processing.py +++ b/bin/background_processing.py @@ -2,6 +2,7 @@ import json import logging +import logging.config from collections import Counter from datetime import date, timedelta from typing import Any, Dict @@ -10,8 +11,7 @@ from lookyloo.lookyloo import Lookyloo from lookyloo.default import AbstractManager, get_config, get_homedir, safe_create_dir from lookyloo.helpers import ParsedUserAgent, serialize_to_json -logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s:%(message)s', - level=logging.INFO) +logging.config.dictConfig(get_config('logging')) class Processing(AbstractManager): diff --git a/bin/start_website.py b/bin/start_website.py index 231c389f..4aa4d409 100755 --- a/bin/start_website.py +++ b/bin/start_website.py @@ -1,12 +1,12 @@ #!/usr/bin/env python3 import logging +import logging.config from subprocess import Popen from lookyloo.default import get_config, get_homedir, AbstractManager -logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s:%(message)s', - level=logging.INFO) +logging.config.dictConfig(get_config('logging')) class Website(AbstractManager): diff --git a/bin/update.py b/bin/update.py index bbda8332..d7a8c363 100755 --- a/bin/update.py +++ b/bin/update.py @@ -3,16 +3,16 @@ import argparse import hashlib import logging +import logging.config import platform import shlex import subprocess import sys from pathlib import Path -from lookyloo.default import get_homedir +from lookyloo.default import get_homedir, get_config -logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s:%(message)s', - level=logging.INFO) +logging.config.dictConfig(get_config('logging')) def compute_hash_self(): diff --git a/config/logging.json.sample b/config/logging.json.sample new file mode 100644 index 00000000..633b8107 --- /dev/null +++ b/config/logging.json.sample @@ -0,0 +1,33 @@ +{ + "version": 1, + "disable_existing_loggers": false, + "formatters": { + "simple": { + "format": "%(asctime)s %(name)s %(levelname)s:%(message)s" + } + }, + "handlers": { + "stdout": { + "class": "logging.StreamHandler", + "level": "INFO", + "formatter": "simple", + "stream": "ext://sys.stdout" + }, + "file": { + "class": "logging.handlers.RotatingFileHandler", + "level": "WARNING", + "formatter": "simple", + "filename": "logs/warning.log", + "mode": "a", + "maxBytes": 1000000, + "backupCount": 5 + } + }, + "root": { + "level": "DEBUG", + "handlers": [ + "stdout", + "file" + ] + } +} diff --git a/logs/.keepdir b/logs/.keepdir new file mode 100644 index 00000000..e69de29b diff --git a/lookyloo/default/helpers.py b/lookyloo/default/helpers.py index db6de009..25e35bb2 100644 --- a/lookyloo/default/helpers.py +++ b/lookyloo/default/helpers.py @@ -57,17 +57,20 @@ def load_configs(path_to_config_files: Optional[Union[str, Path]]=None): @lru_cache(64) -def get_config(config_type: str, entry: str, quiet: bool=False) -> Any: +def get_config(config_type: str, entry: Optional[str]=None, quiet: bool=False) -> Any: """Get an entry from the given config_type file. Automatic fallback to the sample file""" global configs if not configs: load_configs() if config_type in configs: - if entry in configs[config_type]: - return configs[config_type][entry] + if entry: + if entry in configs[config_type]: + return configs[config_type][entry] + else: + if not quiet: + logger.warning(f'Unable to find {entry} in config file.') else: - if not quiet: - logger.warning(f'Unable to find {entry} in config file.') + return configs[config_type] else: if not quiet: logger.warning(f'No {config_type} config file available.') @@ -75,7 +78,9 @@ def get_config(config_type: str, entry: str, quiet: bool=False) -> Any: logger.warning(f'Falling back on sample config, please initialize the {config_type} config file.') with (get_homedir() / 'config' / f'{config_type}.json.sample').open() as _c: sample_config = json.load(_c) - return sample_config[entry] + if entry: + return sample_config[entry] + return sample_config def safe_create_dir(to_create: Path) -> None: diff --git a/website/web/__init__.py b/website/web/__init__.py index 47059fd5..20768ae5 100644 --- a/website/web/__init__.py +++ b/website/web/__init__.py @@ -123,8 +123,6 @@ enable_bookmark = get_config('generic', 'enable_bookmark') auto_trigger_modules = get_config('generic', 'auto_trigger_modules') hide_captures_with_error = get_config('generic', 'hide_captures_with_error') -logging.basicConfig(level=get_config('generic', 'loglevel')) - # ##### Global methods passed to jinja