From 25d5beaaaf6df1f8600e822e663940ef7731ec18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 1 Sep 2021 12:45:36 +0200 Subject: [PATCH] chg: Improve monitoring --- poetry.lock | 39 ++++++++++++++++++++++++++++++++-- pyproject.toml | 1 + tools/monitoring.py | 51 +++++++++++++++++++++++++-------------------- 3 files changed, 66 insertions(+), 25 deletions(-) diff --git a/poetry.lock b/poetry.lock index d78f622a..5e9086c8 100644 --- a/poetry.lock +++ b/poetry.lock @@ -176,6 +176,17 @@ category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +[[package]] +name = "commonmark" +version = "0.9.1" +description = "Python parser for the CommonMark Markdown spec" +category = "main" +optional = false +python-versions = "*" + +[package.extras] +test = ["flake8 (==3.7.8)", "hypothesis (==3.55.3)"] + [[package]] name = "constantly" version = "15.1.0" @@ -787,7 +798,7 @@ python-versions = "*" name = "pygments" version = "2.10.0" description = "Pygments is a syntax highlighting package written in Python." -category = "dev" +category = "main" optional = false python-versions = ">=3.5" @@ -949,6 +960,22 @@ urllib3 = ">=1.21.1,<1.27" socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"] use_chardet_on_py3 = ["chardet (>=3.0.2,<5)"] +[[package]] +name = "rich" +version = "10.9.0" +description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" +category = "main" +optional = false +python-versions = ">=3.6,<4.0" + +[package.dependencies] +colorama = ">=0.4.0,<0.5.0" +commonmark = ">=0.9.0,<0.10.0" +pygments = ">=2.6.0,<3.0.0" + +[package.extras] +jupyter = ["ipywidgets (>=7.5.1,<8.0.0)"] + [[package]] name = "scrapy" version = "2.5.0" @@ -1273,7 +1300,7 @@ misp = ["python-magic", "pydeep"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "7c93a6b1c98b1b595c4b1e653c0e72401bafd96c4d611a41fdcc3a414393bd9f" +content-hash = "45847baf6fb2e8963b6cea41ad681434dff4fb0948fd48864778f4865a14bd50" [metadata.files] aiohttp = [ @@ -1440,6 +1467,10 @@ click = [ colorama = [ {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, ] +commonmark = [ + {file = "commonmark-0.9.1-py2.py3-none-any.whl", hash = "sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9"}, + {file = "commonmark-0.9.1.tar.gz", hash = "sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60"}, +] constantly = [ {file = "constantly-15.1.0-py2.py3-none-any.whl", hash = "sha256:dd2fa9d6b1a51a83f0d7dd76293d734046aa176e384bf6e33b7e44880eb37c5d"}, {file = "constantly-15.1.0.tar.gz", hash = "sha256:586372eb92059873e29eba4f9dec8381541b4d3834660707faf8ba59146dfc35"}, @@ -2002,6 +2033,10 @@ requests = [ {file = "requests-2.26.0-py2.py3-none-any.whl", hash = "sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24"}, {file = "requests-2.26.0.tar.gz", hash = "sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7"}, ] +rich = [ + {file = "rich-10.9.0-py3-none-any.whl", hash = "sha256:2c84d9b3459c16bf413fe0f9644c7ae1791971e0bb944dfae56e7c7634b187ab"}, + {file = "rich-10.9.0.tar.gz", hash = "sha256:ba285f1c519519490034284e6a9d2e6e3f16dc7690f2de3d9140737d81304d22"}, +] scrapy = [ {file = "Scrapy-2.5.0-py2.py3-none-any.whl", hash = "sha256:5f590fdc84b496e5a4bb5ef99836b0aa688a07cfcb4bc3bb7290f66486f27424"}, {file = "Scrapy-2.5.0.tar.gz", hash = "sha256:0a68ed41f7173679f160c4cef2db05288548c21e7164170552adae8b13cefaab"}, diff --git a/pyproject.toml b/pyproject.toml index 5d3722d6..3ce305c9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,6 +62,7 @@ lief = "^0.11.4" Flask-Login = "^0.5.0" flask-restx = "^0.5.0" hiredis = "^2.0.0" +rich = "^10.9.0" [tool.poetry.extras] misp = ['python-magic', 'pydeep'] diff --git a/tools/monitoring.py b/tools/monitoring.py index 407439df..f85d66bf 100755 --- a/tools/monitoring.py +++ b/tools/monitoring.py @@ -1,8 +1,6 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -import json -import logging import os import sys @@ -10,17 +8,21 @@ from typing import List, Tuple from redis import Redis from redis.exceptions import ConnectionError +from rich.console import Console +from rich.padding import Padding -from lookyloo.helpers import get_config, get_socket_path, splash_status +from lookyloo.helpers import get_socket_path, splash_status from lookyloo.abstractmanager import AbstractManager +# NOTE: run with watch: +# watch --color tools/monitoring.py + +console = Console(color_system="256") + class Monitoring(): def __init__(self) -> None: - self.logger = logging.getLogger(f'{self.__class__.__name__}') - self.logger.setLevel(get_config('generic', 'loglevel')) - self.redis_cache: Redis = Redis(unix_socket_path=get_socket_path('cache'), decode_responses=True) self.redis_indexing: Redis = Redis(unix_socket_path=get_socket_path('indexing'), decode_responses=True) @@ -30,27 +32,27 @@ class Monitoring(): socket_path_index = get_socket_path('indexing') backend_up = True if not os.path.exists(socket_path_cache): - print(f'Socket path for the cache redis DB does not exists ({socket_path_cache}).') + console.print(f'Socket path for the [blue]cache[/blue] redis DB [red]does not exists[/red] ({socket_path_cache}).') backend_up = False if not os.path.exists(socket_path_index): - print(f'Socket path for the indexing redis DB does not exists ({socket_path_index}).') + console.print(f'Socket path for the [blue]indexing[/blue] redis DB [red]does not exists[/red] ({socket_path_index}).') backend_up = False if backend_up: try: cache_reachable = True if self.redis_cache.ping() else False if not cache_reachable: - print('Unable to ping the redis cache db.') + console.print('Unable to ping the redis cache db.') backend_up = False except ConnectionError: - print('Unable to connect to the redis cache db.') + console.print('Unable to connect to the redis cache db.') backend_up = False try: indexing_reachable = True if self.redis_indexing.ping() else False if not indexing_reachable: - print('Unable to ping the redis indexing db.') + console.print('Unable to ping the redis indexing db.') backend_up = False except ConnectionError: - print('Unable to connect to the redis indexing db.') + console.print('Unable to connect to the redis indexing db.') backend_up = False return backend_up @@ -77,28 +79,31 @@ if __name__ == '__main__': status, message = splash_status() if status: - print(message) + console.print(f'[green]{message}[/green]') else: - print('Splash is down: ', message) + console.print('Splash is [bold red]down[/bold red]: ', message) m = Monitoring() backend_up = m.backend_status if not backend_up: - print('Backend not up, breaking.') + console.print('[bold red]Backend not up, breaking.[/bold red]') sys.exit() - print('Services currently running:') + console.print('Services currently running:') running = AbstractManager.is_running() for service, number in running: - print(service, f'({int(number)} service(s))') + s = Padding(f'{service} ({int(number)} service(s))', (0, 2)) + console.print(s) - print('Current queues:') + console.print('Current queues:') for q, priority in m.queues: - print(q, f'Priority: {int(priority)}') + s = Padding(f'{q} Priority: {int(priority)}', (0, 2)) + console.print(s) # ------------------ - print('Captures details:') + console.print('Captures details:') captures = m.ongoing_captures - print('Queue length', len(captures)) + console.print(f'Queue length: [yellow]{len(captures)}[/yellow]') for uuid, rank, d in captures: - print(uuid, f'Rank: {int(rank)}') - print(json.dumps(d, indent=2)) + a = Padding(f'{uuid} Rank: {int(rank)}', (0, 2)) + console.print(a) + console.print(d)