From a50c5c6fdba919a4a7ed270e8a7565152c7c7d97 Mon Sep 17 00:00:00 2001 From: mokaddem Date: Fri, 14 Jun 2019 16:15:07 +0200 Subject: [PATCH] new: Started dev of diagnostic tool - WiP --- diagnostic.py | 220 +++++++++++++++++++++++++++++++++++++++++++++ diagnostic_util.py | 20 +++++ 2 files changed, 240 insertions(+) create mode 100755 diagnostic.py create mode 100644 diagnostic_util.py diff --git a/diagnostic.py b/diagnostic.py new file mode 100755 index 0000000..60215a4 --- /dev/null +++ b/diagnostic.py @@ -0,0 +1,220 @@ +#!/usr/bin/env python3 +import os +import sys +import stat +import time +import functools +import configparser +import diagnostic_util +import redis +import zmq +from halo import Halo +from pprint import pprint + +''' +Steps: +- check if virtualenv exists +- check if configuration is update-to-date + - check file permission + - check if redis is running and responding + - check if able to connect to zmq + - check zmq_dispatcher processing queue + - check queue status: being filled up / being filled down +- check if subscriber responding +- check if dispatcher responding +- check if server listening +- check log static endpoint +- check log dynamic endpoint +''' + +configuration_file = {} + + +def humanize(name, isResult=False): + words = name.split('_') + if isResult: + words = words[1:] + words[0] = words[0][0].upper() + words[0][1:] + else: + words[0] = words[0][0].upper() + words[0][1:] + 'ing' + return ' '.join(words) + + +def add_spinner(_func=None, name='dots'): + def decorator_add_spinner(func): + @functools.wraps(func) + def wrapper_add_spinner(*args, **kwargs): + human_func_name = humanize(func.__name__) + human_func_result = humanize(func.__name__, isResult=True) + flag_skip = False + + with Halo(text=human_func_name, spinner=name) as spinner: + result = func(spinner, *args, **kwargs) + if isinstance(result, tuple): + status, output = result + elif isinstance(result, list): + status = result[0] + output = result[1] + elif isinstance(result, bool): + status = result + output = None + else: + status = False + flag_skip = True + spinner.fail(f'{human_func_name} - Function return unexpected result: {str(result)}') + + if not flag_skip: + text = human_func_result + if output is not None and len(output) > 0: + text += f': {output}' + + if isinstance(status, bool) and status: + spinner.succeed(text) + elif isinstance(status, bool) and not status: + spinner.fail(text) + else: + spinner.warn(text) + return status + return wrapper_add_spinner + + if _func is None: + return decorator_add_spinner + else: + return decorator_add_spinner(_func) + + +@add_spinner +def check_virtual_environment(spinner): + result = os.environ.get('VIRTUAL_ENV') + if result is None: + return (False, 'This diagnostic tool should be started inside a virtual environment.') + else: + return (True, '') + + +@add_spinner +def check_configuration(spinner): + global configuration_file + configfile = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'config/config.cfg') + cfg = configparser.ConfigParser() + cfg.read(configfile) + configuration_file = cfg + cfg = {s: dict(cfg.items(s)) for s in cfg.sections()} + configfile = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'config/config.cfg.default') + cfg_default = configparser.ConfigParser() + cfg_default.read(configfile) + cfg_default = {s: dict(cfg_default.items(s)) for s in cfg_default.sections()} + + # Check if all fields from config.default exists in config + result, faulties = diagnostic_util.dict_compare(cfg_default, cfg) + faulties = [item for sublist in faulties for item in sublist] + if result: + return (True, '') + else: + return (False, f'''Configuration incomplete. +\tUpdate your configuration file `config.cfg`.\n\t➥ Faulty fields: {", ".join(faulties)}''') + + +@add_spinner(name='dot') +def check_file_permission(spinner): + max_mind_database_path = configuration_file.get('RedisMap', 'pathmaxminddb') + st = os.stat(max_mind_database_path) + all_read_perm = bool(st.st_mode & stat.S_IROTH) # FIXME: permission may be changed + if all_read_perm: + return (True, '') + else: + return (False, 'Maxmin GeoDB might have incorrect read file permission') + + +@add_spinner +def check_redis(spinner): + redis_server = redis.StrictRedis( + host=configuration_file.get('RedisGlobal', 'host'), + port=configuration_file.getint('RedisGlobal', 'port'), + db=configuration_file.getint('RedisLog', 'db')) + if redis_server.ping(): + return (True, '') + else: + return (False, '''Can\'t reach Redis server. +\t➥ Make sure it is running and adapt your configuration accordingly''') + + +@add_spinner +def check_zmq(spinner): + context = zmq.Context() + socket = context.socket(zmq.SUB) + socket.connect(configuration_file.get('RedisGlobal', 'zmq_url')) + socket.setsockopt_string(zmq.SUBSCRIBE, '') + poller = zmq.Poller() + + poller.register(socket, zmq.POLLIN) + socks = dict(poller.poll(timeout=15000)) + if len(socks) > 0: + if socket in socks and socks[socket] == zmq.POLLIN: + rcv_string = socket.recv() + if rcv_string.startswith(b'misp_json'): + return (True, '') + else: + return (False, '''Can\'t connect to the ZMQ stream. +\t➥ Make sure the MISP ZMQ is running: `/servers/serverSettings/diagnostics` +\t➥ Make sure your network infrastucture allows you to connect to the ZMQ''') + + +@add_spinner +def check_buffer_queue(spinner): + redis_server = redis.StrictRedis( + host=configuration_file.get('RedisGlobal', 'host'), + port=configuration_file.getint('RedisGlobal', 'port'), + db=configuration_file.getint('RedisLIST', 'db')) + elements_in_list = redis_server.llen(configuration_file.get('RedisLIST', 'listName')) + if elements_in_list > 100: + return ('warning', f'Currently {elements_in_list} in the buffer') + else: + return (True, f'Currently {elements_in_list} in the buffer') + + +@add_spinner +def check_subscriber_status(spinner): + return (False, '') + + +@add_spinner +def check_dispatcher_status(spinner): + return (False, '') + + +@add_spinner +def check_server_listening(spinner): + return (False, '') + + +@add_spinner +def check_static_endpoint(spinner): + return (False, '') + + +@add_spinner +def check_dynamic_enpoint(spinner): + return (False, '') + + +def start_diagnostic(): + if not (check_virtual_environment() and check_configuration()): + return + check_file_permission() + check_redis() + check_zmq() + check_buffer_queue() + check_subscriber_status() + check_dispatcher_status() + check_server_listening() + check_static_endpoint() + check_dynamic_enpoint() + + +def main(): + start_diagnostic() + + +if __name__ == '__main__': + main() diff --git a/diagnostic_util.py b/diagnostic_util.py new file mode 100644 index 0000000..d6838dd --- /dev/null +++ b/diagnostic_util.py @@ -0,0 +1,20 @@ +def dict_compare(dict1, dict2): + dict1_keys = set(dict1.keys()) + dict2_keys = set(dict2.keys()) + intersection = dict1_keys.difference(dict2_keys) + if len(intersection) > 0: + return (False, list(intersection)) + + flag_no_error = True + faulties = [] + for k, v in dict1.items(): + if (isinstance(v, dict)): + status, faulty = dict_compare(v, dict2[k]) + flag_no_error = flag_no_error and status + faulties.append(faulty) + else: + (True, []) + if flag_no_error: + return (True, []) + else: + return (False, faulties)