From 6f58e862cc095dada95848486f5ec8a2dcf37c3e Mon Sep 17 00:00:00 2001 From: Terrtia Date: Mon, 25 Nov 2019 16:28:20 +0100 Subject: [PATCH] chg: [core] add redis server in config --- server/LAUNCH.sh | 12 +++-- server/configs/6380.conf | 6 +-- server/configs/server.conf.sample | 25 ++++++++++ server/documentation/Database.md | 7 +++ server/lib/ConfigLoader.py | 54 ++++++++++++++++++++ server/server.py | 39 ++++++--------- server/web/Flask_server.py | 55 +++++---------------- server/web/Role_Manager.py | 17 ++++--- server/web/blueprints/restApi.py | 28 +++-------- server/web/blueprints/settings.py | 22 ++++----- server/web/create_default_user.py | 16 +++--- server/workers/workers_1/file_compressor.py | 30 ++++------- server/workers/workers_1/worker.py | 36 +++----------- server/workers/workers_1/workers_manager.py | 14 +++--- server/workers/workers_2/file_compressor.py | 30 ++++------- server/workers/workers_2/worker.py | 22 +++------ server/workers/workers_2/workers_manager.py | 16 +++--- server/workers/workers_4/worker.py | 20 ++++---- server/workers/workers_4/workers_manager.py | 14 +++--- server/workers/workers_8/worker.py | 39 ++++----------- server/workers/workers_8/workers_manager.py | 14 +++--- 21 files changed, 240 insertions(+), 276 deletions(-) create mode 100755 server/lib/ConfigLoader.py diff --git a/server/LAUNCH.sh b/server/LAUNCH.sh index 22ed83f..ce146b1 100755 --- a/server/LAUNCH.sh +++ b/server/LAUNCH.sh @@ -45,6 +45,10 @@ function helptext { " } +CONFIG=$D4_HOME/configs/server.conf +redis_stream=`sed -nr '/\[Redis_STREAM\]/,/\[/{/port/p}' ${CONFIG} | awk -F= '/port/{print $2}' | sed 's/ //g'` +redis_metadata=`sed -nr '/\[Redis_METADATA\]/,/\[/{/port/p}' ${CONFIG} | awk -F= '/port/{print $2}' | sed 's/ //g'` + function launching_redis { conf_dir="${D4_HOME}/configs/" redis_dir="${D4_HOME}/redis/src/" @@ -84,22 +88,22 @@ function launching_workers { function shutting_down_redis { redis_dir=${D4_HOME}/redis/src/ - bash -c $redis_dir'redis-cli -p 6379 SHUTDOWN' + bash -c $redis_dir'redis-cli -p '$redis_stream' SHUTDOWN' sleep 0.1 - bash -c $redis_dir'redis-cli -p 6380 SHUTDOWN' + bash -c $redis_dir'redis-cli -p '$redis_metadata' SHUTDOWN' sleep 0.1 } function checking_redis { flag_redis=0 redis_dir=${D4_HOME}/redis/src/ - bash -c $redis_dir'redis-cli -p 6379 PING | grep "PONG" &> /dev/null' + bash -c $redis_dir'redis-cli -p '$redis_stream' PING | grep "PONG" &> /dev/null' if [ ! $? == 0 ]; then echo -e $RED"\t6379 not ready"$DEFAULT flag_redis=1 fi sleep 0.1 - bash -c $redis_dir'redis-cli -p 6380 PING | grep "PONG" &> /dev/null' + bash -c $redis_dir'redis-cli -p '$redis_metadata' PING | grep "PONG" &> /dev/null' if [ ! $? == 0 ]; then echo -e $RED"\t6380 not ready"$DEFAULT flag_redis=1 diff --git a/server/configs/6380.conf b/server/configs/6380.conf index b225ebd..98b7d80 100644 --- a/server/configs/6380.conf +++ b/server/configs/6380.conf @@ -155,7 +155,7 @@ supervised no # # Creating a pid file is best effort: if Redis is not able to create it # nothing bad happens, the server will start and run normally. -pidfile /var/run/redis_6379.pid +pidfile /var/run/redis_6380.pid # Specify the server verbosity level. # This can be one of: @@ -843,7 +843,7 @@ lua-time-limit 5000 # Make sure that instances running in the same system do not have # overlapping cluster configuration file names. # -# cluster-config-file nodes-6379.conf +# cluster-config-file nodes-6380.conf # Cluster node timeout is the amount of milliseconds a node must be unreachable # for it to be considered in failure state. @@ -971,7 +971,7 @@ lua-time-limit 5000 # Example: # # cluster-announce-ip 10.1.1.5 -# cluster-announce-port 6379 +# cluster-announce-port 6380 # cluster-announce-bus-port 6380 ################################## SLOW LOG ################################### diff --git a/server/configs/server.conf.sample b/server/configs/server.conf.sample index a937c36..4ef7eaf 100644 --- a/server/configs/server.conf.sample +++ b/server/configs/server.conf.sample @@ -6,3 +6,28 @@ save_directory = None [D4_Server] # registration or shared-secret server_mode = registration + +[Redis_STREAM] +host = localhost +port = 6379 +db = 0 + +[Redis_METADATA] +host = localhost +port = 6380 +db = 0 + +[Redis_SERV] +host = localhost +port = 6380 +db = 1 + +[Redis_ANALYZER] +host = localhost +port = 6380 +db = 2 + +[Redis_CACHE] +host = localhost +port = 6380 +db = 3 diff --git a/server/documentation/Database.md b/server/documentation/Database.md index 56a727a..be0f24c 100644 --- a/server/documentation/Database.md +++ b/server/documentation/Database.md @@ -48,6 +48,13 @@ sensor registrations, management of decoding protocols and dispatching to adequa | server:accepted_type | **accepted type** | | server:accepted_extended_type | **accepted extended type** | +###### Server Mode +| Set Key | Value | +| --- | --- | +| blacklist_ip | **blacklisted ip** | +| blacklist_ip_by_uuid | **uuidv4** | +| blacklist_uuid | **uuidv4** | + ###### Connection Manager | Set Key | Value | | --- | --- | diff --git a/server/lib/ConfigLoader.py b/server/lib/ConfigLoader.py new file mode 100755 index 0000000..3db061f --- /dev/null +++ b/server/lib/ConfigLoader.py @@ -0,0 +1,54 @@ +#!/usr/bin/python3 + +""" +The ``ConfigLoader`` +=================== + + +""" + +import os +import sys +import time +import redis +import configparser + +# Get Config file +config_dir = os.path.join(os.environ['D4_HOME'], 'configs') +config_file = os.path.join(config_dir, 'server.conf') +if not os.path.exists(config_file): + raise Exception('Unable to find the configuration file. \ + Did you set environment variables? \ + Or activate the virtualenv.') + + # # TODO: create sphinx doc + + # # TODO: add config_field to reload + +class ConfigLoader(object): + """docstring for Config_Loader.""" + + def __init__(self): + self.cfg = configparser.ConfigParser() + self.cfg.read(config_file) + + def get_redis_conn(self, redis_name, decode_responses=True): ## TODO: verify redis name + return redis.StrictRedis( host=self.cfg.get(redis_name, "host"), + port=self.cfg.getint(redis_name, "port"), + db=self.cfg.getint(redis_name, "db"), + decode_responses=decode_responses ) + + def get_config_str(self, section, key_name): + return self.cfg.get(section, key_name) + + def get_config_int(self, section, key_name): + return self.cfg.getint(section, key_name) + + def get_config_boolean(self, section, key_name): + return self.cfg.getboolean(section, key_name) + + def has_option(self, section, key_name): + return self.cfg.has_option(section, key_name) + + def has_section(self, section): + return self.cfg.has_section(section) diff --git a/server/server.py b/server/server.py index 917e689..2d66d57 100755 --- a/server/server.py +++ b/server/server.py @@ -12,7 +12,6 @@ import datetime import argparse import logging import logging.handlers -import configparser from twisted.internet import ssl, task, protocol, endpoints, defer from twisted.python import log @@ -21,6 +20,9 @@ from twisted.python.modules import getModule from twisted.internet.protocol import Protocol from twisted.protocols.policies import TimeoutMixin +sys.path.append(os.path.join(os.environ['D4_HOME'], 'lib/')) +import ConfigLoader + hmac_reset = bytearray(32) hmac_key = os.getenv('D4_HMAC_KEY', b'private key to change') @@ -36,35 +38,29 @@ header_size = 62 data_default_size_limit = 1000000 default_max_entries_by_stream = 10000 -host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost") -port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379)) +### Config ### +config_loader = ConfigLoader.ConfigLoader() -host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost") -port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380)) +# REDIS # +redis_server_stream = config_loader.get_redis_conn("Redis_STREAM", decode_responses=False) +redis_server_metadata = config_loader.get_redis_conn("Redis_METADATA", decode_responses=False) +# get server_mode +server_mode = config_loader.get_config_str("D4_Server", "server_mode") -### REDIS ### - -redis_server_stream = redis.StrictRedis( - host=host_redis_stream, - port=port_redis_stream, - db=0) - -redis_server_metadata = redis.StrictRedis( - host=host_redis_metadata, - port=port_redis_metadata, - db=0) +config_loader = None +### ### try: redis_server_stream.ping() except redis.exceptions.ConnectionError: - print('Error: Redis server {}:{}, ConnectionError'.format(host_redis_stream, port_redis_stream)) + print('Error: Redis server Redis_STREAM, ConnectionError') sys.exit(1) try: redis_server_metadata.ping() except redis.exceptions.ConnectionError: - print('Error: Redis server {}:{}, ConnectionError'.format(host_redis_metadata, port_redis_metadata)) + print('Error: Redis server Redis_METADATA, ConnectionError') sys.exit(1) ### REDIS ### @@ -584,14 +580,7 @@ if __name__ == "__main__": logger.addHandler(handler_log) logger.setLevel(args.verbose) - - # get file config - config_file_server = os.path.join(os.environ['D4_HOME'], 'configs/server.conf') - config_server = configparser.ConfigParser() - config_server.read(config_file_server) - # get server_mode - server_mode = config_server['D4_Server'].get('server_mode') if server_mode not in all_server_modes: print('Error: incorrect server_mode') logger.critical('Error: incorrect server_mode') diff --git a/server/web/Flask_server.py b/server/web/Flask_server.py index 43efd91..6557b16 100755 --- a/server/web/Flask_server.py +++ b/server/web/Flask_server.py @@ -13,7 +13,6 @@ import redis import random import datetime import ipaddress -import configparser import subprocess @@ -29,6 +28,7 @@ from Role_Manager import login_user_basic, login_admin sys.path.append(os.path.join(os.environ['D4_HOME'], 'lib')) from User import User import Sensor +import ConfigLoader # Import Blueprint from blueprints.restApi import restApi @@ -40,9 +40,6 @@ if baseUrl != '': all_server_modes = ('registration', 'shared-secret') -host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost") -port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379)) - default_max_entries_by_stream = 10000 analyzer_list_max_default_size = 10000 @@ -50,55 +47,29 @@ default_analyzer_max_line_len = 3000 json_type_description_path = os.path.join(os.environ['D4_HOME'], 'web/static/json/type.json') -# get file config -config_file_server = os.path.join(os.environ['D4_HOME'], 'configs/server.conf') -config_server = configparser.ConfigParser() -config_server.read(config_file_server) +### Config ### +config_loader = ConfigLoader.ConfigLoader() # get data directory -use_default_save_directory = config_server['Save_Directories'].getboolean('use_default_save_directory') +use_default_save_directory = config_loader.get_config_boolean("Save_Directories", "use_default_save_directory") # check if field is None if use_default_save_directory: data_directory = os.path.join(os.environ['D4_HOME'], 'data') else: - data_directory = config_server['Save_Directories'].get('save_directory') + data_directory = config_loader.get_config_str("Save_Directories", "save_directory") -server_mode = config_server['D4_Server'].get('server_mode') +server_mode = config_loader.get_config_str("D4_Server", "server_mode") if server_mode not in all_server_modes: print('Error: incorrect server_mode') -redis_server_stream = redis.StrictRedis( - host=host_redis_stream, - port=port_redis_stream, - db=0, - decode_responses=True) +redis_server_stream = config_loader.get_redis_conn("Redis_STREAM") +redis_server_metadata = config_loader.get_redis_conn("Redis_METADATA") +redis_users = config_loader.get_redis_conn("Redis_SERV") +redis_server_analyzer = config_loader.get_redis_conn("Redis_ANALYZER") +r_cache = config_loader.get_redis_conn("Redis_CACHE") -host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost") -port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380)) - -redis_server_metadata = redis.StrictRedis( - host=host_redis_metadata, - port=port_redis_metadata, - db=0, - decode_responses=True) - -redis_users = redis.StrictRedis( - host=host_redis_metadata, - port=port_redis_metadata, - db=1, - decode_responses=True) - -redis_server_analyzer = redis.StrictRedis( - host=host_redis_metadata, - port=port_redis_metadata, - db=2, - decode_responses=True) - -r_cache = redis.StrictRedis( - host=host_redis_metadata, - port=port_redis_metadata, - db=3, - decode_responses=True) +config_loader = None +### ### with open(json_type_description_path, 'r') as f: json_type = json.loads(f.read()) diff --git a/server/web/Role_Manager.py b/server/web/Role_Manager.py index 4772be8..a772e99 100644 --- a/server/web/Role_Manager.py +++ b/server/web/Role_Manager.py @@ -3,6 +3,7 @@ import os import re +import sys import redis import bcrypt @@ -11,17 +12,17 @@ from flask_login import LoginManager, current_user, login_user, logout_user, log from flask import request, current_app +sys.path.append(os.path.join(os.environ['D4_HOME'], 'lib/')) +import ConfigLoader + login_manager = LoginManager() login_manager.login_view = 'role' -host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost") -port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380)) - -r_serv_db = redis.StrictRedis( - host=host_redis_metadata, - port=port_redis_metadata, - db=1, - decode_responses=True) +### Config ### +config_loader = ConfigLoader.ConfigLoader() +r_serv_db = config_loader.get_redis_conn("Redis_SERV") +config_loader = None +### ### default_passwd_file = os.path.join(os.environ['D4_HOME'], 'DEFAULT_PASSWORD') diff --git a/server/web/blueprints/restApi.py b/server/web/blueprints/restApi.py index ce2a97c..9b67140 100644 --- a/server/web/blueprints/restApi.py +++ b/server/web/blueprints/restApi.py @@ -22,6 +22,7 @@ from functools import wraps sys.path.append(os.path.join(os.environ['D4_HOME'], 'lib')) import Sensor +import ConfigLoader # ============ BLUEPRINT ============ @@ -29,26 +30,13 @@ restApi = Blueprint('restApi', __name__, template_folder='templates') # ============ VARIABLES ============ -host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost") -port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380)) - -r_serv_metadata = redis.StrictRedis( - host=host_redis_metadata, - port=port_redis_metadata, - db=0, - decode_responses=True) - -r_serv_db = redis.StrictRedis( - host=host_redis_metadata, - port=port_redis_metadata, - db=1, - decode_responses=True) - -r_cache = redis.StrictRedis( - host=host_redis_metadata, - port=port_redis_metadata, - db=3, - decode_responses=True) +### Config ### +config_loader = ConfigLoader.ConfigLoader() +r_serv_metadata = config_loader.get_redis_conn("Redis_METADATA") +r_serv_db = config_loader.get_redis_conn("Redis_SERV") +r_cache = config_loader.get_redis_conn("Redis_CACHE") +config_loader = None +### ### # ============ AUTH FUNCTIONS ============ diff --git a/server/web/blueprints/settings.py b/server/web/blueprints/settings.py index 8ba071f..32b4360 100644 --- a/server/web/blueprints/settings.py +++ b/server/web/blueprints/settings.py @@ -10,6 +10,9 @@ import re import sys import redis +sys.path.append(os.path.join(os.environ['D4_HOME'], 'lib')) +import ConfigLoader + from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response from flask_login import login_required, current_user @@ -24,20 +27,13 @@ settings = Blueprint('settings', __name__, template_folder='templates') email_regex = r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}' email_regex = re.compile(email_regex) -host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost") -port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380)) -r_serv_metadata = redis.StrictRedis( - host=host_redis_metadata, - port=port_redis_metadata, - db=0, - decode_responses=True) - -r_serv_db = redis.StrictRedis( - host=host_redis_metadata, - port=port_redis_metadata, - db=1, - decode_responses=True) +### Config ### +config_loader = ConfigLoader.ConfigLoader() +r_serv_metadata = config_loader.get_redis_conn("Redis_METADATA") +r_serv_db = config_loader.get_redis_conn("Redis_SERV") +config_loader = None +### ### # ============ FUNCTIONS ============ diff --git a/server/web/create_default_user.py b/server/web/create_default_user.py index 68fe248..a6c971a 100755 --- a/server/web/create_default_user.py +++ b/server/web/create_default_user.py @@ -4,21 +4,17 @@ import os import sys import redis -import configparser sys.path.append(os.path.join(os.environ['D4_HOME'], 'lib')) +import ConfigLoader from Role_Manager import create_user_db, edit_user_db, get_default_admin_token, gen_password -host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost") -port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_HOST', 6380)) - -r_serv = redis.StrictRedis( - host=host_redis_metadata, - port=port_redis_metadata, - db=1, - decode_responses=True) - +### Config ### +config_loader = ConfigLoader.ConfigLoader() +r_serv = config_loader.get_redis_conn("Redis_SERV", decode_responses=False) +config_loader = None +### ### if __name__ == "__main__": diff --git a/server/workers/workers_1/file_compressor.py b/server/workers/workers_1/file_compressor.py index cf812bc..86804f3 100755 --- a/server/workers/workers_1/file_compressor.py +++ b/server/workers/workers_1/file_compressor.py @@ -10,6 +10,9 @@ import datetime import signal +sys.path.append(os.path.join(os.environ['D4_HOME'], 'lib/')) +import ConfigLoader + class GracefulKiller: kill_now = False def __init__(self): @@ -45,27 +48,12 @@ def compress_file(file_full_path, session_uuid,i=0): analyser_queue_max_size = analyzer_list_max_default_size redis_server_analyzer.ltrim('analyzer:{}:{}'.format(type, analyzer_uuid), 0, analyser_queue_max_size) - -host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost") -port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379)) - -host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost") -port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380)) - -redis_server_stream = redis.StrictRedis( - host=host_redis_stream, - port=port_redis_stream, - db=0) - -redis_server_metadata = redis.StrictRedis( - host=host_redis_metadata, - port=port_redis_metadata, - db=0) - -redis_server_analyzer = redis.StrictRedis( - host=host_redis_metadata, - port=port_redis_metadata, - db=2) +### Config ### +config_loader = ConfigLoader.ConfigLoader() +redis_server_stream = config_loader.get_redis_conn("Redis_STREAM", decode_responses=False) +redis_server_metadata = config_loader.get_redis_conn("Redis_METADATA", decode_responses=False) +redis_server_analyzer = config_loader.get_redis_conn("Redis_ANALYZER", decode_responses=False) +config_loader = None type = 1 sleep_time = 300 diff --git a/server/workers/workers_1/worker.py b/server/workers/workers_1/worker.py index 0994924..4b96689 100755 --- a/server/workers/workers_1/worker.py +++ b/server/workers/workers_1/worker.py @@ -8,7 +8,9 @@ import redis import shutil import datetime import subprocess -import configparser + +sys.path.append(os.path.join(os.environ['D4_HOME'], 'lib/')) +import ConfigLoader def data_incorrect_format(stream_name, session_uuid, uuid): redis_server_stream.sadd('Error:IncorrectType', session_uuid) @@ -46,39 +48,17 @@ def compress_file(file_full_path, i=0): analyser_queue_max_size = analyzer_list_max_default_size redis_server_analyzer.ltrim('analyzer:{}:{}'.format(type, analyzer_uuid), 0, analyser_queue_max_size) -host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost") -port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379)) - -host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost") -port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380)) - -redis_server_stream = redis.StrictRedis( - host=host_redis_stream, - port=port_redis_stream, - db=0) - -redis_server_metadata = redis.StrictRedis( - host=host_redis_metadata, - port=port_redis_metadata, - db=0) - -redis_server_analyzer = redis.StrictRedis( - host=host_redis_metadata, - port=port_redis_metadata, - db=2) - -# get file config -config_file_server = os.path.join(os.environ['D4_HOME'], 'configs/server.conf') -config_server = configparser.ConfigParser() -config_server.read(config_file_server) +redis_server_stream = config_loader.get_redis_conn("Redis_STREAM", decode_responses=False) +redis_server_metadata = config_loader.get_redis_conn("Redis_METADATA", decode_responses=False) +redis_server_analyzer = config_loader.get_redis_conn("Redis_ANALYZER", decode_responses=False) # get data directory -use_default_save_directory = config_server['Save_Directories'].getboolean('use_default_save_directory') +use_default_save_directory = config_loader.get_config_boolean("Save_Directories", "use_default_save_directory") # check if field is None if use_default_save_directory: data_directory = os.path.join(os.environ['D4_HOME'], 'data') else: - data_directory = config_server['Save_Directories'].get('save_directory') + data_directory = config_loader.get_config_str("Save_Directories", "save_directory") type = 1 diff --git a/server/workers/workers_1/workers_manager.py b/server/workers/workers_1/workers_manager.py index 0e90289..5492c95 100755 --- a/server/workers/workers_1/workers_manager.py +++ b/server/workers/workers_1/workers_manager.py @@ -6,19 +6,19 @@ import time import redis import subprocess -host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost") -port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379)) +sys.path.append(os.path.join(os.environ['D4_HOME'], 'lib/')) +import ConfigLoader + +config_loader = ConfigLoader.ConfigLoader() +redis_server_stream = config_loader.get_redis_conn("Redis_STREAM", decode_responses=False) +config_loader = None -redis_server_stream = redis.StrictRedis( - host=host_redis_stream, - port=port_redis_stream, - db=0) type = 1 try: redis_server_stream.ping() except redis.exceptions.ConnectionError: - print('Error: Redis server {}:{}, ConnectionError'.format(host_redis_stream, port_redis_stream)) + print('Error: Redis server {}, ConnectionError'.format("Redis_STREAM")) sys.exit(1) if __name__ == "__main__": diff --git a/server/workers/workers_2/file_compressor.py b/server/workers/workers_2/file_compressor.py index cf812bc..85ebdd7 100755 --- a/server/workers/workers_2/file_compressor.py +++ b/server/workers/workers_2/file_compressor.py @@ -10,6 +10,9 @@ import datetime import signal +sys.path.append(os.path.join(os.environ['D4_HOME'], 'lib/')) +import ConfigLoader + class GracefulKiller: kill_now = False def __init__(self): @@ -45,27 +48,14 @@ def compress_file(file_full_path, session_uuid,i=0): analyser_queue_max_size = analyzer_list_max_default_size redis_server_analyzer.ltrim('analyzer:{}:{}'.format(type, analyzer_uuid), 0, analyser_queue_max_size) +### Config ### +config_loader = ConfigLoader.ConfigLoader() -host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost") -port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379)) - -host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost") -port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380)) - -redis_server_stream = redis.StrictRedis( - host=host_redis_stream, - port=port_redis_stream, - db=0) - -redis_server_metadata = redis.StrictRedis( - host=host_redis_metadata, - port=port_redis_metadata, - db=0) - -redis_server_analyzer = redis.StrictRedis( - host=host_redis_metadata, - port=port_redis_metadata, - db=2) +redis_server_stream = config_loader.get_redis_conn("Redis_STREAM", decode_responses=False) +redis_server_metadata = config_loader.get_redis_conn("Redis_METADATA", decode_responses=False) +redis_server_analyzer = config_loader.get_redis_conn("Redis_ANALYZER", decode_responses=False) +config_loader = None +### ### type = 1 sleep_time = 300 diff --git a/server/workers/workers_2/worker.py b/server/workers/workers_2/worker.py index 18b1b5f..edf02da 100755 --- a/server/workers/workers_2/worker.py +++ b/server/workers/workers_2/worker.py @@ -10,21 +10,15 @@ import datetime from meta_types_modules import MetaTypesDefault -host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost") -port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379)) +sys.path.append(os.path.join(os.environ['D4_HOME'], 'lib/')) +import ConfigLoader -redis_server_stream = redis.StrictRedis( - host=host_redis_stream, - port=port_redis_stream, - db=0) - -host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost") -port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380)) - -redis_server_metadata = redis.StrictRedis( - host=host_redis_metadata, - port=port_redis_metadata, - db=0) +### Config ### +config_loader = ConfigLoader.ConfigLoader() +redis_server_stream = config_loader.get_redis_conn("Redis_STREAM", decode_responses=False) +redis_server_metadata = config_loader.get_redis_conn("Redis_METADATA", decode_responses=False) +config_loader = None +### ### type_meta_header = 2 type_defined = 254 diff --git a/server/workers/workers_2/workers_manager.py b/server/workers/workers_2/workers_manager.py index 92b9a08..e57fabc 100755 --- a/server/workers/workers_2/workers_manager.py +++ b/server/workers/workers_2/workers_manager.py @@ -6,19 +6,21 @@ import time import redis import subprocess -host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost") -port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379)) +sys.path.append(os.path.join(os.environ['D4_HOME'], 'lib/')) +import ConfigLoader + +### Config ### +config_loader = ConfigLoader.ConfigLoader() +redis_server_stream = config_loader.get_redis_conn("Redis_STREAM", decode_responses=False) +config_loader = None +### ### -redis_server_stream = redis.StrictRedis( - host=host_redis_stream, - port=port_redis_stream, - db=0) type = 2 try: redis_server_stream.ping() except redis.exceptions.ConnectionError: - print('Error: Redis server {}:{}, ConnectionError'.format(host_redis_stream, port_redis_stream)) + print('Error: Redis server: Redis_STREAM, ConnectionError') sys.exit(1) if __name__ == "__main__": diff --git a/server/workers/workers_4/worker.py b/server/workers/workers_4/worker.py index 5384459..275bc43 100755 --- a/server/workers/workers_4/worker.py +++ b/server/workers/workers_4/worker.py @@ -6,19 +6,17 @@ import time import redis import datetime -import configparser + +sys.path.append(os.path.join(os.environ['D4_HOME'], 'lib/')) +import ConfigLoader def data_incorrect_format(session_uuid): print('Incorrect format') sys.exit(1) -host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost") -port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379)) - -redis_server_stream = redis.StrictRedis( - host=host_redis_stream, - port=port_redis_stream, - db=0) +config_loader = ConfigLoader.ConfigLoader() +redis_server_stream = config_loader.get_redis_conn("Redis_STREAM", decode_responses=False) +config_loader = None # get file config config_file_server = os.path.join(os.environ['D4_HOME'], 'configs/server.conf') @@ -26,13 +24,13 @@ config_server = configparser.ConfigParser() config_server.read(config_file_server) # get data directory -use_default_save_directory = config_server['Save_Directories'].getboolean('use_default_save_directory') +use_default_save_directory = config_loader.get_config_boolean("Save_Directories", "use_default_save_directory") # check if field is None if use_default_save_directory: data_directory = os.path.join(os.environ['D4_HOME'], 'data') else: - data_directory = config_server['Save_Directories'].get('save_directory') - + data_directory = get_config_str.get_config_boolean("Save_Directories", "save_directory") +config_loader = None type = 4 rotation_save_cycle = 300 #seconds diff --git a/server/workers/workers_4/workers_manager.py b/server/workers/workers_4/workers_manager.py index ddbbd7d..d6f9112 100755 --- a/server/workers/workers_4/workers_manager.py +++ b/server/workers/workers_4/workers_manager.py @@ -6,19 +6,19 @@ import time import redis import subprocess -host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost") -port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379)) +sys.path.append(os.path.join(os.environ['D4_HOME'], 'lib/')) +import ConfigLoader + +config_loader = ConfigLoader.ConfigLoader() +redis_server_stream = config_loader.get_redis_conn("Redis_STREAM", decode_responses=False) +config_loader = None -redis_server_stream = redis.StrictRedis( - host=host_redis_stream, - port=port_redis_stream, - db=0) type = 4 try: redis_server_stream.ping() except redis.exceptions.ConnectionError: - print('Error: Redis server {}:{}, ConnectionError'.format(host_redis_stream, port_redis_stream)) + print('Error: Redis server {}, ConnectionError'.format("Redis_STREAM")) sys.exit(1) if __name__ == "__main__": diff --git a/server/workers/workers_8/worker.py b/server/workers/workers_8/worker.py index 3fcfcfd..8ee2373 100755 --- a/server/workers/workers_8/worker.py +++ b/server/workers/workers_8/worker.py @@ -8,46 +8,27 @@ import redis import shutil import datetime -import configparser + +sys.path.append(os.path.join(os.environ['D4_HOME'], 'lib/')) +import ConfigLoader def data_incorrect_format(session_uuid): print('Incorrect format') sys.exit(1) -host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost") -port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379)) - -redis_server_stream = redis.StrictRedis( - host=host_redis_stream, - port=port_redis_stream, - db=0) - -host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost") -port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380)) - -redis_server_metadata = redis.StrictRedis( - host=host_redis_metadata, - port=port_redis_metadata, - db=0) - -redis_server_analyzer = redis.StrictRedis( - host=host_redis_metadata, - port=port_redis_metadata, - db=2) - -# get file config -config_file_server = os.path.join(os.environ['D4_HOME'], 'configs/server.conf') -config_server = configparser.ConfigParser() -config_server.read(config_file_server) +config_loader = ConfigLoader.ConfigLoader() +redis_server_stream = config_loader.get_redis_conn("Redis_STREAM", decode_responses=False) +redis_server_analyzer = config_loader.get_redis_conn("Redis_ANALYZER", decode_responses=False) +config_loader = None # get data directory -use_default_save_directory = config_server['Save_Directories'].getboolean('use_default_save_directory') +use_default_save_directory = config_loader.get_config_boolean("Save_Directories", "use_default_save_directory") # check if field is None if use_default_save_directory: data_directory = os.path.join(os.environ['D4_HOME'], 'data') else: - data_directory = config_server['Save_Directories'].get('save_directory') - + data_directory = config_loader.get_config_str("Save_Directories", "save_directory") +config_loader = None type = 8 rotation_save_cycle = 300 #seconds diff --git a/server/workers/workers_8/workers_manager.py b/server/workers/workers_8/workers_manager.py index 8b99c50..d498a80 100755 --- a/server/workers/workers_8/workers_manager.py +++ b/server/workers/workers_8/workers_manager.py @@ -6,19 +6,19 @@ import time import redis import subprocess -host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost") -port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379)) +sys.path.append(os.path.join(os.environ['D4_HOME'], 'lib/')) +import ConfigLoader + +config_loader = ConfigLoader.ConfigLoader() +redis_server_stream = config_loader.get_redis_conn("Redis_STREAM", decode_responses=False) +config_loader = None -redis_server_stream = redis.StrictRedis( - host=host_redis_stream, - port=port_redis_stream, - db=0) type = 8 try: redis_server_stream.ping() except redis.exceptions.ConnectionError: - print('Error: Redis server {}:{}, ConnectionError'.format(host_redis_stream, port_redis_stream)) + print('Error: Redis server {}, ConnectionError'.format("Redis_STREAM")) sys.exit(1) if __name__ == "__main__":