new: Logging config in file

bin/archiver.py

@@ -3,6 +3,7 @@
 import csv
 import gzip
 import logging
+import logging.config
 import shutil
 
 from collections import defaultdict
@@ -16,8 +17,7 @@ from redis import Redis
 from lookyloo.default import AbstractManager, get_config, get_homedir, get_socket_path
 from lookyloo.helpers import get_captures_dir
 
-logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s:%(message)s',
-                    level=logging.INFO)
+logging.config.dictConfig(get_config('logging'))
 
 
 class Archiver(AbstractManager):

bin/async_capture.py

@@ -1,12 +1,11 @@
 #!/usr/bin/env python3
 
 import asyncio
-import json
 import logging
+import logging.config
 import signal
 import time
 
-from datetime import datetime
 from pathlib import Path
 from typing import Dict, Optional, Union
 
@@ -14,13 +13,12 @@ from lacuscore import LacusCore, CaptureStatus as CaptureStatusCore, CaptureResp
 from pylacus import CaptureStatus as CaptureStatusPy, CaptureResponse as CaptureResponsePy
 
 from lookyloo.lookyloo import Lookyloo
-from lookyloo.default import AbstractManager, get_config, safe_create_dir
+from lookyloo.default import AbstractManager, get_config
 from lookyloo.helpers import get_captures_dir
 
 from lookyloo.modules import FOX
 
-logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s:%(message)s',
-                    level=logging.INFO)
+logging.config.dictConfig(get_config('logging'))
 
 
 class AsyncCapture(AbstractManager):

bin/background_indexer.py

@@ -1,16 +1,17 @@
 #!/usr/bin/env python3
 
 import logging
+import logging.config
 import os
 from datetime import datetime, timedelta
 import shutil
 
-from lookyloo.default import AbstractManager
+from lookyloo.default import AbstractManager, get_config
 from lookyloo.exceptions import MissingUUID, NoValidHarFile
 from lookyloo.lookyloo import Lookyloo
 
-logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s:%(message)s',
-                    level=logging.INFO)
+
+logging.config.dictConfig(get_config('logging'))
 
 
 class BackgroundIndexer(AbstractManager):

bin/background_processing.py

@@ -2,6 +2,7 @@
 
 import json
 import logging
+import logging.config
 from collections import Counter
 from datetime import date, timedelta
 from typing import Any, Dict
@@ -10,8 +11,7 @@ from lookyloo.lookyloo import Lookyloo
 from lookyloo.default import AbstractManager, get_config, get_homedir, safe_create_dir
 from lookyloo.helpers import ParsedUserAgent, serialize_to_json
 
-logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s:%(message)s',
-                    level=logging.INFO)
+logging.config.dictConfig(get_config('logging'))
 
 
 class Processing(AbstractManager):

bin/start_website.py

@@ -1,12 +1,12 @@
 #!/usr/bin/env python3
 
 import logging
+import logging.config
 from subprocess import Popen
 
 from lookyloo.default import get_config, get_homedir, AbstractManager
 
-logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s:%(message)s',
-                    level=logging.INFO)
+logging.config.dictConfig(get_config('logging'))
 
 
 class Website(AbstractManager):

bin/update.py

@@ -3,16 +3,16 @@
 import argparse
 import hashlib
 import logging
+import logging.config
 import platform
 import shlex
 import subprocess
 import sys
 from pathlib import Path
 
-from lookyloo.default import get_homedir
+from lookyloo.default import get_homedir, get_config
 
-logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s:%(message)s',
-                    level=logging.INFO)
+logging.config.dictConfig(get_config('logging'))
 
 
 def compute_hash_self():

config/logging.json.sample

@@ -0,0 +1,33 @@
+{
+    "version": 1,
+    "disable_existing_loggers": false,
+    "formatters": {
+        "simple": {
+            "format": "%(asctime)s %(name)s %(levelname)s:%(message)s"
+        }
+    },
+    "handlers": {
+        "stdout": {
+            "class": "logging.StreamHandler",
+            "level": "INFO",
+            "formatter": "simple",
+            "stream": "ext://sys.stdout"
+        },
+        "file": {
+            "class": "logging.handlers.RotatingFileHandler",
+			"level": "WARNING",
+            "formatter": "simple",
+            "filename": "logs/warning.log",
+            "mode": "a",
+            "maxBytes": 1000000,
+            "backupCount": 5
+        }
+    },
+    "root": {
+        "level": "DEBUG",
+        "handlers": [
+            "stdout",
+            "file"
+        ]
+    }
+}

lookyloo/default/helpers.py

@@ -57,17 +57,20 @@ def load_configs(path_to_config_files: Optional[Union[str, Path]]=None):
 
 
 @lru_cache(64)
-def get_config(config_type: str, entry: str, quiet: bool=False) -> Any:
+def get_config(config_type: str, entry: Optional[str]=None, quiet: bool=False) -> Any:
     """Get an entry from the given config_type file. Automatic fallback to the sample file"""
     global configs
     if not configs:
         load_configs()
     if config_type in configs:
-        if entry in configs[config_type]:
-            return configs[config_type][entry]
+        if entry:
+            if entry in configs[config_type]:
+                return configs[config_type][entry]
+            else:
+                if not quiet:
+                    logger.warning(f'Unable to find {entry} in config file.')
         else:
-            if not quiet:
-                logger.warning(f'Unable to find {entry} in config file.')
+            return configs[config_type]
     else:
         if not quiet:
             logger.warning(f'No {config_type} config file available.')
@@ -75,7 +78,9 @@ def get_config(config_type: str, entry: str, quiet: bool=False) -> Any:
         logger.warning(f'Falling back on sample config, please initialize the {config_type} config file.')
     with (get_homedir() / 'config' / f'{config_type}.json.sample').open() as _c:
         sample_config = json.load(_c)
-    return sample_config[entry]
+    if entry:
+        return sample_config[entry]
+    return sample_config
 
 
 def safe_create_dir(to_create: Path) -> None:

website/web/__init__.py

@@ -123,8 +123,6 @@ enable_bookmark = get_config('generic', 'enable_bookmark')
 auto_trigger_modules = get_config('generic', 'auto_trigger_modules')
 hide_captures_with_error = get_config('generic', 'hide_captures_with_error')
 
-logging.basicConfig(level=get_config('generic', 'loglevel'))
-
 
 # ##### Global methods passed to jinja