diff --git a/server/LAUNCH.sh b/server/LAUNCH.sh index 121981c..80c7044 100755 --- a/server/LAUNCH.sh +++ b/server/LAUNCH.sh @@ -205,9 +205,15 @@ function update_web { fi } +function update_config { + echo -e $GREEN"\t* Updating Config File"$DEFAULT + bash -c "(cd ${D4_HOME}/configs; ./update_conf.py -v 0)" +} + function launch_all { helptext; launch_redis; + update_config; launch_d4_server; launch_workers; launch_flask; diff --git a/server/workers/workers_1/worker.py b/server/workers/workers_1/worker.py index 1354414..6e79d98 100755 --- a/server/workers/workers_1/worker.py +++ b/server/workers/workers_1/worker.py @@ -8,6 +8,7 @@ import redis import shutil import datetime import subprocess +import configparser def data_incorrect_format(stream_name, session_uuid, uuid): redis_server_stream.sadd('Error:IncorrectType', session_uuid) @@ -66,6 +67,20 @@ redis_server_analyzer = redis.StrictRedis( port=port_redis_metadata, db=2) +# get file config +config_file_server = os.path.join(os.environ['D4_HOME'], 'configs/server.conf') +config_server = configparser.ConfigParser() +config_server.read(config_file_server) + +# get data directory +use_default_save_directory = config_server['Save_Directories'].getboolean('use_default_save_directory') +# check if field is None +if use_default_save_directory: + data_directory = os.path.join(os.environ['D4_HOME'], 'data') +else: + data_directory = config_server['Save_Directories'].get('save_directory') + + type = 1 tcp_dump_cycle = '300' stream_buffer = 100 @@ -88,8 +103,8 @@ if __name__ == "__main__": if res: uuid = res[0][1][0][1][b'uuid'].decode() date = datetime.datetime.now().strftime("%Y%m%d") - tcpdump_path = os.path.join('../../data', uuid, str(type)) - full_tcpdump_path = os.path.join(os.environ['D4_HOME'], 'data', uuid, str(type)) + tcpdump_path = os.path.join(data_directory, uuid, str(type)) + full_tcpdump_path = os.path.join(data_directory, uuid, str(type)) rel_path = os.path.join(tcpdump_path, date[0:4], date[4:6], date[6:8]) if not os.path.isdir(rel_path): os.makedirs(rel_path) diff --git a/server/workers/workers_2/meta_types_modules/MetaTypesDefault.py b/server/workers/workers_2/meta_types_modules/MetaTypesDefault.py index 6eb2afc..978ed30 100755 --- a/server/workers/workers_2/meta_types_modules/MetaTypesDefault.py +++ b/server/workers/workers_2/meta_types_modules/MetaTypesDefault.py @@ -8,6 +8,7 @@ import gzip import redis import shutil import datetime +import configparser DEFAULT_FILE_EXTENSION = 'txt' DEFAULT_FILE_SEPARATOR = b'\n' @@ -23,6 +24,20 @@ class MetaTypesDefault: self.save_path = None self.buffer = b'' self.file_rotation_mode = True + + # get file config + config_file_server = os.path.join(os.environ['D4_HOME'], 'configs/server.conf') + config_server = configparser.ConfigParser() + config_server.read(config_file_server) + # get data directory + use_default_save_directory = config_server['Save_Directories'].getboolean('use_default_save_directory') + # check if field is None + if use_default_save_directory: + data_directory = os.path.join(os.environ['D4_HOME'], 'data') + else: + data_directory = config_server['Save_Directories'].get('save_directory') + self.data_directory = data_directory + self.parse_json(json_file) def test(self): @@ -182,15 +197,18 @@ class MetaTypesDefault: if self.is_file_rotation_mode() or save_by_uuid: return '{}-{}-{}-{}-{}.{}'.format(self.uuid, self.get_last_saved_year(), self.get_last_saved_month(), self.get_last_saved_day(), self.get_last_saved_hour_minute(), file_extention) + def get_data_save_directory(): + return self.data_directory + def get_save_dir(self, save_by_uuid=False): # File Rotation, save data in directory: data//254//// if self.is_file_rotation_mode() or save_by_uuid: - data_directory_uuid_type = os.path.join('../../data', self.get_uuid(), str(TYPE)) + data_directory_uuid_type = os.path.join(self.get_data_save_directory(), self.get_uuid(), str(TYPE)) return os.path.join(data_directory_uuid_type, self.get_last_saved_year(), self.get_last_saved_month(), self.get_last_saved_day() , self.type_name) # data save in the same directory else: - save_dir = os.path.join('../../data/datas', self.get_type_name()) + save_dir = os.path.join(self.get_data_save_directory(), 'datas', self.get_type_name()) if not os.path.isdir(save_dir): os.makedirs(save_dir) return save_dir diff --git a/server/workers/workers_4/worker.py b/server/workers/workers_4/worker.py index 8a029ba..2e044c8 100755 --- a/server/workers/workers_4/worker.py +++ b/server/workers/workers_4/worker.py @@ -6,6 +6,7 @@ import time import redis import datetime +import configparser def data_incorrect_format(session_uuid): print('Incorrect format') @@ -19,6 +20,20 @@ redis_server_stream = redis.StrictRedis( port=port_redis_stream, db=0) +# get file config +config_file_server = os.path.join(os.environ['D4_HOME'], 'configs/server.conf') +config_server = configparser.ConfigParser() +config_server.read(config_file_server) + +# get data directory +use_default_save_directory = config_server['Save_Directories'].getboolean('use_default_save_directory') +# check if field is None +if use_default_save_directory: + data_directory = os.path.join(os.environ['D4_HOME'], 'data') +else: + data_directory = config_server['Save_Directories'].get('save_directory') + + type = 4 rotation_save_cycle = 300 #seconds @@ -38,7 +53,7 @@ if __name__ == "__main__": if res: date = datetime.datetime.now().strftime("%Y%m%d%H%M%S") uuid = res[0][1][0][1][b'uuid'].decode() - data_rel_path = os.path.join('../../data', uuid, str(type)) + data_rel_path = os.path.join(data_directory, uuid, str(type)) dir_path = os.path.join(data_rel_path, date[0:4], date[4:6], date[6:8]) if not os.path.isdir(dir_path): os.makedirs(dir_path) diff --git a/server/workers/workers_8/worker.py b/server/workers/workers_8/worker.py index c5d856b..d7b4f57 100755 --- a/server/workers/workers_8/worker.py +++ b/server/workers/workers_8/worker.py @@ -6,6 +6,7 @@ import time import redis import datetime +import configparser def data_incorrect_format(session_uuid): print('Incorrect format') @@ -32,6 +33,20 @@ redis_server_analyzer = redis.StrictRedis( port=port_redis_metadata, db=2) +# get file config +config_file_server = os.path.join(os.environ['D4_HOME'], 'configs/server.conf') +config_server = configparser.ConfigParser() +config_server.read(config_file_server) + +# get data directory +use_default_save_directory = config_server['Save_Directories'].getboolean('use_default_save_directory') +# check if field is None +if use_default_save_directory: + data_directory = os.path.join(os.environ['D4_HOME'], 'data') +else: + data_directory = config_server['Save_Directories'].get('save_directory') + + type = 8 rotation_save_cycle = 300 #seconds @@ -70,7 +85,7 @@ if __name__ == "__main__": rotate_file = False time_file = time.time() date_file = datetime.datetime.now().strftime("%Y%m%d%H%M%S") - dir_data_uuid = os.path.join('../../data', uuid, str(type)) + dir_data_uuid = os.path.join(data_directory, uuid, str(type)) dir_full_path = get_save_dir(dir_data_uuid, date_file[0:4], date_file[4:6], date_file[6:8]) filename = '{}-{}-{}-{}-{}.passivedns.txt'.format(uuid, date_file[0:4], date_file[4:6], date_file[6:8], date_file[8:14]) save_path = os.path.join(dir_full_path, filename)