chg: [core] add redis server in config

server/LAUNCH.sh

@@ -45,6 +45,10 @@ function helptext {
     "
 }
 
+CONFIG=$D4_HOME/configs/server.conf
+redis_stream=`sed -nr '/\[Redis_STREAM\]/,/\[/{/port/p}' ${CONFIG} | awk -F= '/port/{print $2}' | sed 's/ //g'`
+redis_metadata=`sed -nr '/\[Redis_METADATA\]/,/\[/{/port/p}' ${CONFIG} | awk -F= '/port/{print $2}' | sed 's/ //g'`
+
 function launching_redis {
     conf_dir="${D4_HOME}/configs/"
     redis_dir="${D4_HOME}/redis/src/"
@@ -84,22 +88,22 @@ function launching_workers {
 
 function shutting_down_redis {
     redis_dir=${D4_HOME}/redis/src/
-    bash -c $redis_dir'redis-cli -p 6379 SHUTDOWN'
+    bash -c $redis_dir'redis-cli -p '$redis_stream' SHUTDOWN'
     sleep 0.1
-    bash -c $redis_dir'redis-cli -p 6380 SHUTDOWN'
+    bash -c $redis_dir'redis-cli -p '$redis_metadata' SHUTDOWN'
     sleep 0.1
 }
 
 function checking_redis {
     flag_redis=0
     redis_dir=${D4_HOME}/redis/src/
-    bash -c $redis_dir'redis-cli -p 6379 PING | grep "PONG" &> /dev/null'
+    bash -c $redis_dir'redis-cli -p '$redis_stream' PING | grep "PONG" &> /dev/null'
     if [ ! $? == 0 ]; then
        echo -e $RED"\t6379 not ready"$DEFAULT
        flag_redis=1
     fi
     sleep 0.1
-    bash -c $redis_dir'redis-cli -p 6380 PING | grep "PONG" &> /dev/null'
+    bash -c $redis_dir'redis-cli -p '$redis_metadata' PING | grep "PONG" &> /dev/null'
     if [ ! $? == 0 ]; then
        echo -e $RED"\t6380 not ready"$DEFAULT
        flag_redis=1

server/configs/6380.conf

@@ -155,7 +155,7 @@ supervised no
 #
 # Creating a pid file is best effort: if Redis is not able to create it
 # nothing bad happens, the server will start and run normally.
-pidfile /var/run/redis_6379.pid
+pidfile /var/run/redis_6380.pid
 
 # Specify the server verbosity level.
 # This can be one of:
@@ -843,7 +843,7 @@ lua-time-limit 5000
 # Make sure that instances running in the same system do not have
 # overlapping cluster configuration file names.
 #
-# cluster-config-file nodes-6379.conf
+# cluster-config-file nodes-6380.conf
 
 # Cluster node timeout is the amount of milliseconds a node must be unreachable
 # for it to be considered in failure state.
@@ -971,7 +971,7 @@ lua-time-limit 5000
 # Example:
 #
 # cluster-announce-ip 10.1.1.5
-# cluster-announce-port 6379
+# cluster-announce-port 6380
 # cluster-announce-bus-port 6380
 
 ################################## SLOW LOG ###################################

server/configs/server.conf.sample

@@ -6,3 +6,28 @@ save_directory = None
 [D4_Server]
 # registration or shared-secret
 server_mode = registration
+
+[Redis_STREAM]
+host = localhost
+port = 6379
+db = 0
+
+[Redis_METADATA]
+host = localhost
+port = 6380
+db = 0
+
+[Redis_SERV]
+host = localhost
+port = 6380
+db = 1
+
+[Redis_ANALYZER]
+host = localhost
+port = 6380
+db = 2
+
+[Redis_CACHE]
+host = localhost
+port = 6380
+db = 3

server/documentation/Database.md

@@ -48,6 +48,13 @@ sensor registrations, management of decoding protocols and dispatching to adequa
 | server:accepted_type          | **accepted type** |
 | server:accepted_extended_type | **accepted extended type** |
 
+###### Server Mode
+| Set Key | Value |
+| --- | --- |
+| blacklist_ip          | **blacklisted ip** |
+| blacklist_ip_by_uuid | **uuidv4** |
+| blacklist_uuid | **uuidv4** |
+
 ###### Connection Manager
 | Set Key | Value |
 | --- | --- |

server/lib/ConfigLoader.py

@@ -0,0 +1,54 @@
+#!/usr/bin/python3
+
+"""
+The ``ConfigLoader``
+===================
+
+
+"""
+
+import os
+import sys
+import time
+import redis
+import configparser
+
+# Get Config file
+config_dir = os.path.join(os.environ['D4_HOME'], 'configs')
+config_file = os.path.join(config_dir, 'server.conf')
+if not os.path.exists(config_file):
+    raise Exception('Unable to find the configuration file. \
+                    Did you set environment variables? \
+                    Or activate the virtualenv.')
+
+ # # TODO: create sphinx doc
+
+ # # TODO: add config_field to reload
+
+class ConfigLoader(object):
+    """docstring for Config_Loader."""
+
+    def __init__(self):
+        self.cfg = configparser.ConfigParser()
+        self.cfg.read(config_file)
+
+    def get_redis_conn(self, redis_name, decode_responses=True): ## TODO: verify redis name
+        return redis.StrictRedis( host=self.cfg.get(redis_name, "host"),
+                                  port=self.cfg.getint(redis_name, "port"),
+                                  db=self.cfg.getint(redis_name, "db"),
+                                  decode_responses=decode_responses )
+
+    def get_config_str(self, section, key_name):
+        return self.cfg.get(section, key_name)
+
+    def get_config_int(self, section, key_name):
+        return self.cfg.getint(section, key_name)
+
+    def get_config_boolean(self, section, key_name):
+        return self.cfg.getboolean(section, key_name)
+
+    def has_option(self, section, key_name):
+        return self.cfg.has_option(section, key_name)
+
+    def has_section(self, section):
+        return self.cfg.has_section(section)

server/server.py

@@ -12,7 +12,6 @@ import datetime
 import argparse
 import logging
 import logging.handlers
-import configparser
 
 from twisted.internet import ssl, task, protocol, endpoints, defer
 from twisted.python import log
@@ -21,6 +20,9 @@ from twisted.python.modules import getModule
 from twisted.internet.protocol import Protocol
 from twisted.protocols.policies import TimeoutMixin
 
+sys.path.append(os.path.join(os.environ['D4_HOME'], 'lib/'))
+import ConfigLoader
+
 hmac_reset = bytearray(32)
 hmac_key = os.getenv('D4_HMAC_KEY', b'private key to change')
 
@@ -36,35 +38,29 @@ header_size = 62
 data_default_size_limit = 1000000
 default_max_entries_by_stream = 10000
 
-host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost")
-port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379))
+### Config ###
+config_loader = ConfigLoader.ConfigLoader()
 
-host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost")
-port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380))
+# REDIS #
+redis_server_stream = config_loader.get_redis_conn("Redis_STREAM", decode_responses=False)
+redis_server_metadata = config_loader.get_redis_conn("Redis_METADATA", decode_responses=False)
 
+# get server_mode
+server_mode = config_loader.get_config_str("D4_Server", "server_mode")
 
-### REDIS ###
-
-redis_server_stream = redis.StrictRedis(
-                    host=host_redis_stream,
-                    port=port_redis_stream,
-                    db=0)
-
-redis_server_metadata = redis.StrictRedis(
-                    host=host_redis_metadata,
-                    port=port_redis_metadata,
-                    db=0)
+config_loader = None
+###  ###
 
 try:
     redis_server_stream.ping()
 except redis.exceptions.ConnectionError:
-    print('Error: Redis server {}:{}, ConnectionError'.format(host_redis_stream, port_redis_stream))
+    print('Error: Redis server Redis_STREAM, ConnectionError')
     sys.exit(1)
 
 try:
     redis_server_metadata.ping()
 except redis.exceptions.ConnectionError:
-    print('Error: Redis server {}:{}, ConnectionError'.format(host_redis_metadata, port_redis_metadata))
+    print('Error: Redis server Redis_METADATA, ConnectionError')
     sys.exit(1)
 
 ### REDIS ###
@@ -584,14 +580,7 @@ if __name__ == "__main__":
     logger.addHandler(handler_log)
     logger.setLevel(args.verbose)
 
-
-    # get file config
-    config_file_server = os.path.join(os.environ['D4_HOME'], 'configs/server.conf')
-    config_server = configparser.ConfigParser()
-    config_server.read(config_file_server)
-
     # get server_mode
-    server_mode = config_server['D4_Server'].get('server_mode')
     if server_mode not in all_server_modes:
         print('Error: incorrect server_mode')
         logger.critical('Error: incorrect server_mode')

server/web/Flask_server.py

@@ -13,7 +13,6 @@ import redis
 import random
 import datetime
 import ipaddress
-import configparser
 
 import subprocess
 
@@ -29,6 +28,7 @@ from Role_Manager import login_user_basic, login_admin
 sys.path.append(os.path.join(os.environ['D4_HOME'], 'lib'))
 from User import User
 import Sensor
+import ConfigLoader
 
 # Import Blueprint
 from blueprints.restApi import restApi
@@ -40,9 +40,6 @@ if baseUrl != '':
 
 all_server_modes = ('registration', 'shared-secret')
 
-host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost")
-port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379))
-
 default_max_entries_by_stream = 10000
 analyzer_list_max_default_size = 10000
 
@@ -50,55 +47,29 @@ default_analyzer_max_line_len = 3000
 
 json_type_description_path = os.path.join(os.environ['D4_HOME'], 'web/static/json/type.json')
 
-# get file config
-config_file_server = os.path.join(os.environ['D4_HOME'], 'configs/server.conf')
-config_server = configparser.ConfigParser()
-config_server.read(config_file_server)
+### Config ###
+config_loader = ConfigLoader.ConfigLoader()
 
 # get data directory
-use_default_save_directory = config_server['Save_Directories'].getboolean('use_default_save_directory')
+use_default_save_directory = config_loader.get_config_boolean("Save_Directories", "use_default_save_directory")
 # check if field is None
 if use_default_save_directory:
     data_directory = os.path.join(os.environ['D4_HOME'], 'data')
 else:
-    data_directory = config_server['Save_Directories'].get('save_directory')
+    data_directory = config_loader.get_config_str("Save_Directories", "save_directory")
 
-server_mode = config_server['D4_Server'].get('server_mode')
+server_mode = config_loader.get_config_str("D4_Server", "server_mode")
 if server_mode not in all_server_modes:
     print('Error: incorrect server_mode')
 
-redis_server_stream = redis.StrictRedis(
-                    host=host_redis_stream,
-                    port=port_redis_stream,
-                    db=0,
-                    decode_responses=True)
+redis_server_stream = config_loader.get_redis_conn("Redis_STREAM")
+redis_server_metadata = config_loader.get_redis_conn("Redis_METADATA")
+redis_users = config_loader.get_redis_conn("Redis_SERV")
+redis_server_analyzer = config_loader.get_redis_conn("Redis_ANALYZER")
+r_cache = config_loader.get_redis_conn("Redis_CACHE")
 
-host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost")
-port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380))
-
-redis_server_metadata = redis.StrictRedis(
-                    host=host_redis_metadata,
-                    port=port_redis_metadata,
-                    db=0,
-                    decode_responses=True)
-
-redis_users = redis.StrictRedis(
-                    host=host_redis_metadata,
-                    port=port_redis_metadata,
-                    db=1,
-                    decode_responses=True)
-
-redis_server_analyzer = redis.StrictRedis(
-                    host=host_redis_metadata,
-                    port=port_redis_metadata,
-                    db=2,
-                    decode_responses=True)
-
-r_cache = redis.StrictRedis(
-                host=host_redis_metadata,
-                port=port_redis_metadata,
-                db=3,
-                decode_responses=True)
+config_loader = None
+###  ###
 
 with open(json_type_description_path, 'r') as f:
     json_type = json.loads(f.read())

server/web/Role_Manager.py

@@ -3,6 +3,7 @@
 
 import os
 import re
+import sys
 import redis
 import bcrypt
 
@@ -11,17 +12,17 @@ from flask_login import LoginManager, current_user, login_user, logout_user, log
 
 from flask import request, current_app
 
+sys.path.append(os.path.join(os.environ['D4_HOME'], 'lib/'))
+import ConfigLoader
+
 login_manager = LoginManager()
 login_manager.login_view = 'role'
 
-host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost")
-port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380))
-
-r_serv_db = redis.StrictRedis(
-                    host=host_redis_metadata,
-                    port=port_redis_metadata,
-                    db=1,
-                    decode_responses=True)
+### Config ###
+config_loader = ConfigLoader.ConfigLoader()
+r_serv_db = config_loader.get_redis_conn("Redis_SERV")
+config_loader = None
+###  ###
 
 default_passwd_file = os.path.join(os.environ['D4_HOME'], 'DEFAULT_PASSWORD')
 

server/web/blueprints/restApi.py

@@ -22,6 +22,7 @@ from functools import wraps
 
 sys.path.append(os.path.join(os.environ['D4_HOME'], 'lib'))
 import Sensor
+import ConfigLoader
 
 # ============ BLUEPRINT ============
 
@@ -29,26 +30,13 @@ restApi = Blueprint('restApi', __name__, template_folder='templates')
 
 # ============ VARIABLES ============
 
-host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost")
-port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380))
-
-r_serv_metadata = redis.StrictRedis(
-                host=host_redis_metadata,
-                port=port_redis_metadata,
-                db=0,
-                decode_responses=True)
-
-r_serv_db = redis.StrictRedis(
-                host=host_redis_metadata,
-                port=port_redis_metadata,
-                db=1,
-                decode_responses=True)
-
-r_cache = redis.StrictRedis(
-                host=host_redis_metadata,
-                port=port_redis_metadata,
-                db=3,
-                decode_responses=True)
+### Config ###
+config_loader = ConfigLoader.ConfigLoader()
+r_serv_metadata = config_loader.get_redis_conn("Redis_METADATA")
+r_serv_db = config_loader.get_redis_conn("Redis_SERV")
+r_cache = config_loader.get_redis_conn("Redis_CACHE")
+config_loader = None
+###  ###
 
 # ============ AUTH FUNCTIONS ============
 

server/web/blueprints/settings.py

@@ -10,6 +10,9 @@ import re
 import sys
 import redis
 
+sys.path.append(os.path.join(os.environ['D4_HOME'], 'lib'))
+import ConfigLoader
+
 from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response
 from flask_login import login_required, current_user
 
@@ -24,20 +27,13 @@ settings = Blueprint('settings', __name__, template_folder='templates')
 
 email_regex = r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}'
 email_regex = re.compile(email_regex)
-host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost")
-port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380))
 
-r_serv_metadata = redis.StrictRedis(
-                host=host_redis_metadata,
-                port=port_redis_metadata,
-                db=0,
-                decode_responses=True)
-
-r_serv_db = redis.StrictRedis(
-                host=host_redis_metadata,
-                port=port_redis_metadata,
-                db=1,
-                decode_responses=True)
+### Config ###
+config_loader = ConfigLoader.ConfigLoader()
+r_serv_metadata = config_loader.get_redis_conn("Redis_METADATA")
+r_serv_db = config_loader.get_redis_conn("Redis_SERV")
+config_loader = None
+###  ###
 
 # ============ FUNCTIONS ============
 

server/web/create_default_user.py

@@ -4,21 +4,17 @@
 import os
 import sys
 import redis
-import configparser
 
 sys.path.append(os.path.join(os.environ['D4_HOME'], 'lib'))
+import ConfigLoader
 
 from Role_Manager import create_user_db, edit_user_db, get_default_admin_token, gen_password
 
-host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost")
-port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_HOST', 6380))
-
-r_serv = redis.StrictRedis(
-    host=host_redis_metadata,
-    port=port_redis_metadata,
-    db=1,
-    decode_responses=True)
-
+### Config ###
+config_loader = ConfigLoader.ConfigLoader()
+r_serv = config_loader.get_redis_conn("Redis_SERV", decode_responses=False)
+config_loader = None
+###  ###
 
 if __name__ == "__main__":
 

server/workers/workers_1/file_compressor.py

@@ -10,6 +10,9 @@ import datetime
 
 import signal
 
+sys.path.append(os.path.join(os.environ['D4_HOME'], 'lib/'))
+import ConfigLoader
+
 class GracefulKiller:
   kill_now = False
   def __init__(self):
@@ -45,27 +48,12 @@ def compress_file(file_full_path, session_uuid,i=0):
                 analyser_queue_max_size = analyzer_list_max_default_size
             redis_server_analyzer.ltrim('analyzer:{}:{}'.format(type, analyzer_uuid), 0, analyser_queue_max_size)
 
-
-host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost")
-port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379))
-
-host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost")
-port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380))
-
-redis_server_stream = redis.StrictRedis(
-                    host=host_redis_stream,
-                    port=port_redis_stream,
-                    db=0)
-
-redis_server_metadata = redis.StrictRedis(
-                    host=host_redis_metadata,
-                    port=port_redis_metadata,
-                    db=0)
-
-redis_server_analyzer = redis.StrictRedis(
-                    host=host_redis_metadata,
-                    port=port_redis_metadata,
-                    db=2)
+### Config ###
+config_loader = ConfigLoader.ConfigLoader()
+redis_server_stream = config_loader.get_redis_conn("Redis_STREAM", decode_responses=False)
+redis_server_metadata = config_loader.get_redis_conn("Redis_METADATA", decode_responses=False)
+redis_server_analyzer = config_loader.get_redis_conn("Redis_ANALYZER", decode_responses=False)
+config_loader = None
 
 type = 1
 sleep_time = 300

server/workers/workers_1/worker.py

@@ -8,7 +8,9 @@ import redis
 import shutil
 import datetime
 import subprocess
-import configparser
+
+sys.path.append(os.path.join(os.environ['D4_HOME'], 'lib/'))
+import ConfigLoader
 
 def data_incorrect_format(stream_name, session_uuid, uuid):
     redis_server_stream.sadd('Error:IncorrectType', session_uuid)
@@ -46,39 +48,17 @@ def compress_file(file_full_path, i=0):
                 analyser_queue_max_size = analyzer_list_max_default_size
             redis_server_analyzer.ltrim('analyzer:{}:{}'.format(type, analyzer_uuid), 0, analyser_queue_max_size)
 
-host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost")
-port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379))
-
-host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost")
-port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380))
-
-redis_server_stream = redis.StrictRedis(
-                    host=host_redis_stream,
-                    port=port_redis_stream,
-                    db=0)
-
-redis_server_metadata = redis.StrictRedis(
-                    host=host_redis_metadata,
-                    port=port_redis_metadata,
-                    db=0)
-
-redis_server_analyzer = redis.StrictRedis(
-                    host=host_redis_metadata,
-                    port=port_redis_metadata,
-                    db=2)
-
-# get file config
-config_file_server = os.path.join(os.environ['D4_HOME'], 'configs/server.conf')
-config_server = configparser.ConfigParser()
-config_server.read(config_file_server)
+redis_server_stream = config_loader.get_redis_conn("Redis_STREAM", decode_responses=False)
+redis_server_metadata = config_loader.get_redis_conn("Redis_METADATA", decode_responses=False)
+redis_server_analyzer = config_loader.get_redis_conn("Redis_ANALYZER", decode_responses=False)
 
 # get data directory
-use_default_save_directory = config_server['Save_Directories'].getboolean('use_default_save_directory')
+use_default_save_directory = config_loader.get_config_boolean("Save_Directories", "use_default_save_directory")
 # check if field is None
 if use_default_save_directory:
     data_directory = os.path.join(os.environ['D4_HOME'], 'data')
 else:
-    data_directory = config_server['Save_Directories'].get('save_directory')
+    data_directory = config_loader.get_config_str("Save_Directories", "save_directory")
 
 
 type = 1

server/workers/workers_1/workers_manager.py

@@ -6,19 +6,19 @@ import time
 import redis
 import subprocess
 
-host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost")
-port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379))
+sys.path.append(os.path.join(os.environ['D4_HOME'], 'lib/'))
+import ConfigLoader
+
+config_loader = ConfigLoader.ConfigLoader()
+redis_server_stream = config_loader.get_redis_conn("Redis_STREAM", decode_responses=False)
+config_loader = None
 
-redis_server_stream = redis.StrictRedis(
-                    host=host_redis_stream,
-                    port=port_redis_stream,
-                    db=0)
 type = 1
 
 try:
     redis_server_stream.ping()
 except redis.exceptions.ConnectionError:
-    print('Error: Redis server {}:{}, ConnectionError'.format(host_redis_stream, port_redis_stream))
+    print('Error: Redis server {}, ConnectionError'.format("Redis_STREAM"))
     sys.exit(1)
 
 if __name__ == "__main__":

server/workers/workers_2/file_compressor.py

@@ -10,6 +10,9 @@ import datetime
 
 import signal
 
+sys.path.append(os.path.join(os.environ['D4_HOME'], 'lib/'))
+import ConfigLoader
+
 class GracefulKiller:
   kill_now = False
   def __init__(self):
@@ -45,27 +48,14 @@ def compress_file(file_full_path, session_uuid,i=0):
                 analyser_queue_max_size = analyzer_list_max_default_size
             redis_server_analyzer.ltrim('analyzer:{}:{}'.format(type, analyzer_uuid), 0, analyser_queue_max_size)
 
+### Config ###
+config_loader = ConfigLoader.ConfigLoader()
 
-host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost")
-port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379))
-
-host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost")
-port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380))
-
-redis_server_stream = redis.StrictRedis(
-                    host=host_redis_stream,
-                    port=port_redis_stream,
-                    db=0)
-
-redis_server_metadata = redis.StrictRedis(
-                    host=host_redis_metadata,
-                    port=port_redis_metadata,
-                    db=0)
-
-redis_server_analyzer = redis.StrictRedis(
-                    host=host_redis_metadata,
-                    port=port_redis_metadata,
-                    db=2)
+redis_server_stream = config_loader.get_redis_conn("Redis_STREAM", decode_responses=False)
+redis_server_metadata = config_loader.get_redis_conn("Redis_METADATA", decode_responses=False)
+redis_server_analyzer = config_loader.get_redis_conn("Redis_ANALYZER", decode_responses=False)
+config_loader = None
+###  ###
 
 type = 1
 sleep_time = 300

server/workers/workers_2/worker.py

@@ -10,21 +10,15 @@ import datetime
 
 from meta_types_modules import MetaTypesDefault
 
-host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost")
-port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379))
+sys.path.append(os.path.join(os.environ['D4_HOME'], 'lib/'))
+import ConfigLoader
 
-redis_server_stream = redis.StrictRedis(
-                    host=host_redis_stream,
-                    port=port_redis_stream,
-                    db=0)
-
-host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost")
-port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380))
-
-redis_server_metadata = redis.StrictRedis(
-                    host=host_redis_metadata,
-                    port=port_redis_metadata,
-                    db=0)
+### Config ###
+config_loader = ConfigLoader.ConfigLoader()
+redis_server_stream = config_loader.get_redis_conn("Redis_STREAM", decode_responses=False)
+redis_server_metadata = config_loader.get_redis_conn("Redis_METADATA", decode_responses=False)
+config_loader = None
+###  ###
 
 type_meta_header = 2
 type_defined = 254

server/workers/workers_2/workers_manager.py

@@ -6,19 +6,21 @@ import time
 import redis
 import subprocess
 
-host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost")
-port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379))
+sys.path.append(os.path.join(os.environ['D4_HOME'], 'lib/'))
+import ConfigLoader
+
+### Config ###
+config_loader = ConfigLoader.ConfigLoader()
+redis_server_stream = config_loader.get_redis_conn("Redis_STREAM", decode_responses=False)
+config_loader = None
+###  ###
 
-redis_server_stream = redis.StrictRedis(
-                    host=host_redis_stream,
-                    port=port_redis_stream,
-                    db=0)
 type = 2
 
 try:
     redis_server_stream.ping()
 except redis.exceptions.ConnectionError:
-    print('Error: Redis server {}:{}, ConnectionError'.format(host_redis_stream, port_redis_stream))
+    print('Error: Redis server: Redis_STREAM, ConnectionError')
     sys.exit(1)
 
 if __name__ == "__main__":

server/workers/workers_4/worker.py

@@ -6,19 +6,17 @@ import time
 import redis
 
 import datetime
-import configparser
+
+sys.path.append(os.path.join(os.environ['D4_HOME'], 'lib/'))
+import ConfigLoader
 
 def data_incorrect_format(session_uuid):
     print('Incorrect format')
     sys.exit(1)
 
-host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost")
-port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379))
-
-redis_server_stream = redis.StrictRedis(
-                    host=host_redis_stream,
-                    port=port_redis_stream,
-                    db=0)
+config_loader = ConfigLoader.ConfigLoader()
+redis_server_stream = config_loader.get_redis_conn("Redis_STREAM", decode_responses=False)
+config_loader = None
 
 # get file config
 config_file_server = os.path.join(os.environ['D4_HOME'], 'configs/server.conf')
@@ -26,13 +24,13 @@ config_server = configparser.ConfigParser()
 config_server.read(config_file_server)
 
 # get data directory
-use_default_save_directory = config_server['Save_Directories'].getboolean('use_default_save_directory')
+use_default_save_directory = config_loader.get_config_boolean("Save_Directories", "use_default_save_directory")
 # check if field is None
 if use_default_save_directory:
     data_directory = os.path.join(os.environ['D4_HOME'], 'data')
 else:
-    data_directory = config_server['Save_Directories'].get('save_directory')
-
+    data_directory = get_config_str.get_config_boolean("Save_Directories", "save_directory")
+config_loader = None
 
 type = 4
 rotation_save_cycle = 300 #seconds

server/workers/workers_4/workers_manager.py

@@ -6,19 +6,19 @@ import time
 import redis
 import subprocess
 
-host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost")
-port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379))
+sys.path.append(os.path.join(os.environ['D4_HOME'], 'lib/'))
+import ConfigLoader
+
+config_loader = ConfigLoader.ConfigLoader()
+redis_server_stream = config_loader.get_redis_conn("Redis_STREAM", decode_responses=False)
+config_loader = None
 
-redis_server_stream = redis.StrictRedis(
-                    host=host_redis_stream,
-                    port=port_redis_stream,
-                    db=0)
 type = 4
 
 try:
     redis_server_stream.ping()
 except redis.exceptions.ConnectionError:
-    print('Error: Redis server {}:{}, ConnectionError'.format(host_redis_stream, port_redis_stream))
+    print('Error: Redis server {}, ConnectionError'.format("Redis_STREAM"))
     sys.exit(1)
 
 if __name__ == "__main__":

server/workers/workers_8/worker.py

@@ -8,46 +8,27 @@ import redis
 
 import shutil
 import datetime
-import configparser
+
+sys.path.append(os.path.join(os.environ['D4_HOME'], 'lib/'))
+import ConfigLoader
 
 def data_incorrect_format(session_uuid):
     print('Incorrect format')
     sys.exit(1)
 
-host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost")
-port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379))
-
-redis_server_stream = redis.StrictRedis(
-                    host=host_redis_stream,
-                    port=port_redis_stream,
-                    db=0)
-
-host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost")
-port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380))
-
-redis_server_metadata = redis.StrictRedis(
-                    host=host_redis_metadata,
-                    port=port_redis_metadata,
-                    db=0)
-
-redis_server_analyzer = redis.StrictRedis(
-                    host=host_redis_metadata,
-                    port=port_redis_metadata,
-                    db=2)
-
-# get file config
-config_file_server = os.path.join(os.environ['D4_HOME'], 'configs/server.conf')
-config_server = configparser.ConfigParser()
-config_server.read(config_file_server)
+config_loader = ConfigLoader.ConfigLoader()
+redis_server_stream = config_loader.get_redis_conn("Redis_STREAM", decode_responses=False)
+redis_server_analyzer = config_loader.get_redis_conn("Redis_ANALYZER", decode_responses=False)
+config_loader = None
 
 # get data directory
-use_default_save_directory = config_server['Save_Directories'].getboolean('use_default_save_directory')
+use_default_save_directory = config_loader.get_config_boolean("Save_Directories", "use_default_save_directory")
 # check if field is None
 if use_default_save_directory:
     data_directory = os.path.join(os.environ['D4_HOME'], 'data')
 else:
-    data_directory = config_server['Save_Directories'].get('save_directory')
-
+    data_directory = config_loader.get_config_str("Save_Directories", "save_directory")
+config_loader = None
 
 type = 8
 rotation_save_cycle = 300 #seconds

server/workers/workers_8/workers_manager.py

@@ -6,19 +6,19 @@ import time
 import redis
 import subprocess
 
-host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost")
-port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379))
+sys.path.append(os.path.join(os.environ['D4_HOME'], 'lib/'))
+import ConfigLoader
+
+config_loader = ConfigLoader.ConfigLoader()
+redis_server_stream = config_loader.get_redis_conn("Redis_STREAM", decode_responses=False)
+config_loader = None
 
-redis_server_stream = redis.StrictRedis(
-                    host=host_redis_stream,
-                    port=port_redis_stream,
-                    db=0)
 type = 8
 
 try:
     redis_server_stream.ping()
 except redis.exceptions.ConnectionError:
-    print('Error: Redis server {}:{}, ConnectionError'.format(host_redis_stream, port_redis_stream))
+    print('Error: Redis server {}, ConnectionError'.format("Redis_STREAM"))
     sys.exit(1)
 
 if __name__ == "__main__":