mirror of https://github.com/D4-project/d4-core
chg: [server + workers] add config file + add option to specify save directory
parent
c7ea2d7863
commit
f67224e2e9
|
@ -205,9 +205,15 @@ function update_web {
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function update_config {
|
||||||
|
echo -e $GREEN"\t* Updating Config File"$DEFAULT
|
||||||
|
bash -c "(cd ${D4_HOME}/configs; ./update_conf.py -v 0)"
|
||||||
|
}
|
||||||
|
|
||||||
function launch_all {
|
function launch_all {
|
||||||
helptext;
|
helptext;
|
||||||
launch_redis;
|
launch_redis;
|
||||||
|
update_config;
|
||||||
launch_d4_server;
|
launch_d4_server;
|
||||||
launch_workers;
|
launch_workers;
|
||||||
launch_flask;
|
launch_flask;
|
||||||
|
|
|
@ -8,6 +8,7 @@ import redis
|
||||||
import shutil
|
import shutil
|
||||||
import datetime
|
import datetime
|
||||||
import subprocess
|
import subprocess
|
||||||
|
import configparser
|
||||||
|
|
||||||
def data_incorrect_format(stream_name, session_uuid, uuid):
|
def data_incorrect_format(stream_name, session_uuid, uuid):
|
||||||
redis_server_stream.sadd('Error:IncorrectType', session_uuid)
|
redis_server_stream.sadd('Error:IncorrectType', session_uuid)
|
||||||
|
@ -66,6 +67,20 @@ redis_server_analyzer = redis.StrictRedis(
|
||||||
port=port_redis_metadata,
|
port=port_redis_metadata,
|
||||||
db=2)
|
db=2)
|
||||||
|
|
||||||
|
# get file config
|
||||||
|
config_file_server = os.path.join(os.environ['D4_HOME'], 'configs/server.conf')
|
||||||
|
config_server = configparser.ConfigParser()
|
||||||
|
config_server.read(config_file_server)
|
||||||
|
|
||||||
|
# get data directory
|
||||||
|
use_default_save_directory = config_server['Save_Directories'].getboolean('use_default_save_directory')
|
||||||
|
# check if field is None
|
||||||
|
if use_default_save_directory:
|
||||||
|
data_directory = os.path.join(os.environ['D4_HOME'], 'data')
|
||||||
|
else:
|
||||||
|
data_directory = config_server['Save_Directories'].get('save_directory')
|
||||||
|
|
||||||
|
|
||||||
type = 1
|
type = 1
|
||||||
tcp_dump_cycle = '300'
|
tcp_dump_cycle = '300'
|
||||||
stream_buffer = 100
|
stream_buffer = 100
|
||||||
|
@ -88,8 +103,8 @@ if __name__ == "__main__":
|
||||||
if res:
|
if res:
|
||||||
uuid = res[0][1][0][1][b'uuid'].decode()
|
uuid = res[0][1][0][1][b'uuid'].decode()
|
||||||
date = datetime.datetime.now().strftime("%Y%m%d")
|
date = datetime.datetime.now().strftime("%Y%m%d")
|
||||||
tcpdump_path = os.path.join('../../data', uuid, str(type))
|
tcpdump_path = os.path.join(data_directory, uuid, str(type))
|
||||||
full_tcpdump_path = os.path.join(os.environ['D4_HOME'], 'data', uuid, str(type))
|
full_tcpdump_path = os.path.join(data_directory, uuid, str(type))
|
||||||
rel_path = os.path.join(tcpdump_path, date[0:4], date[4:6], date[6:8])
|
rel_path = os.path.join(tcpdump_path, date[0:4], date[4:6], date[6:8])
|
||||||
if not os.path.isdir(rel_path):
|
if not os.path.isdir(rel_path):
|
||||||
os.makedirs(rel_path)
|
os.makedirs(rel_path)
|
||||||
|
|
|
@ -8,6 +8,7 @@ import gzip
|
||||||
import redis
|
import redis
|
||||||
import shutil
|
import shutil
|
||||||
import datetime
|
import datetime
|
||||||
|
import configparser
|
||||||
|
|
||||||
DEFAULT_FILE_EXTENSION = 'txt'
|
DEFAULT_FILE_EXTENSION = 'txt'
|
||||||
DEFAULT_FILE_SEPARATOR = b'\n'
|
DEFAULT_FILE_SEPARATOR = b'\n'
|
||||||
|
@ -23,6 +24,20 @@ class MetaTypesDefault:
|
||||||
self.save_path = None
|
self.save_path = None
|
||||||
self.buffer = b''
|
self.buffer = b''
|
||||||
self.file_rotation_mode = True
|
self.file_rotation_mode = True
|
||||||
|
|
||||||
|
# get file config
|
||||||
|
config_file_server = os.path.join(os.environ['D4_HOME'], 'configs/server.conf')
|
||||||
|
config_server = configparser.ConfigParser()
|
||||||
|
config_server.read(config_file_server)
|
||||||
|
# get data directory
|
||||||
|
use_default_save_directory = config_server['Save_Directories'].getboolean('use_default_save_directory')
|
||||||
|
# check if field is None
|
||||||
|
if use_default_save_directory:
|
||||||
|
data_directory = os.path.join(os.environ['D4_HOME'], 'data')
|
||||||
|
else:
|
||||||
|
data_directory = config_server['Save_Directories'].get('save_directory')
|
||||||
|
self.data_directory = data_directory
|
||||||
|
|
||||||
self.parse_json(json_file)
|
self.parse_json(json_file)
|
||||||
|
|
||||||
def test(self):
|
def test(self):
|
||||||
|
@ -182,15 +197,18 @@ class MetaTypesDefault:
|
||||||
if self.is_file_rotation_mode() or save_by_uuid:
|
if self.is_file_rotation_mode() or save_by_uuid:
|
||||||
return '{}-{}-{}-{}-{}.{}'.format(self.uuid, self.get_last_saved_year(), self.get_last_saved_month(), self.get_last_saved_day(), self.get_last_saved_hour_minute(), file_extention)
|
return '{}-{}-{}-{}-{}.{}'.format(self.uuid, self.get_last_saved_year(), self.get_last_saved_month(), self.get_last_saved_day(), self.get_last_saved_hour_minute(), file_extention)
|
||||||
|
|
||||||
|
def get_data_save_directory():
|
||||||
|
return self.data_directory
|
||||||
|
|
||||||
def get_save_dir(self, save_by_uuid=False):
|
def get_save_dir(self, save_by_uuid=False):
|
||||||
# File Rotation, save data in directory: data/<uuid>/254/<year>/<month>/<day>/
|
# File Rotation, save data in directory: data/<uuid>/254/<year>/<month>/<day>/
|
||||||
if self.is_file_rotation_mode() or save_by_uuid:
|
if self.is_file_rotation_mode() or save_by_uuid:
|
||||||
data_directory_uuid_type = os.path.join('../../data', self.get_uuid(), str(TYPE))
|
data_directory_uuid_type = os.path.join(self.get_data_save_directory(), self.get_uuid(), str(TYPE))
|
||||||
return os.path.join(data_directory_uuid_type, self.get_last_saved_year(), self.get_last_saved_month(), self.get_last_saved_day() , self.type_name)
|
return os.path.join(data_directory_uuid_type, self.get_last_saved_year(), self.get_last_saved_month(), self.get_last_saved_day() , self.type_name)
|
||||||
|
|
||||||
# data save in the same directory
|
# data save in the same directory
|
||||||
else:
|
else:
|
||||||
save_dir = os.path.join('../../data/datas', self.get_type_name())
|
save_dir = os.path.join(self.get_data_save_directory(), 'datas', self.get_type_name())
|
||||||
if not os.path.isdir(save_dir):
|
if not os.path.isdir(save_dir):
|
||||||
os.makedirs(save_dir)
|
os.makedirs(save_dir)
|
||||||
return save_dir
|
return save_dir
|
||||||
|
|
|
@ -6,6 +6,7 @@ import time
|
||||||
import redis
|
import redis
|
||||||
|
|
||||||
import datetime
|
import datetime
|
||||||
|
import configparser
|
||||||
|
|
||||||
def data_incorrect_format(session_uuid):
|
def data_incorrect_format(session_uuid):
|
||||||
print('Incorrect format')
|
print('Incorrect format')
|
||||||
|
@ -19,6 +20,20 @@ redis_server_stream = redis.StrictRedis(
|
||||||
port=port_redis_stream,
|
port=port_redis_stream,
|
||||||
db=0)
|
db=0)
|
||||||
|
|
||||||
|
# get file config
|
||||||
|
config_file_server = os.path.join(os.environ['D4_HOME'], 'configs/server.conf')
|
||||||
|
config_server = configparser.ConfigParser()
|
||||||
|
config_server.read(config_file_server)
|
||||||
|
|
||||||
|
# get data directory
|
||||||
|
use_default_save_directory = config_server['Save_Directories'].getboolean('use_default_save_directory')
|
||||||
|
# check if field is None
|
||||||
|
if use_default_save_directory:
|
||||||
|
data_directory = os.path.join(os.environ['D4_HOME'], 'data')
|
||||||
|
else:
|
||||||
|
data_directory = config_server['Save_Directories'].get('save_directory')
|
||||||
|
|
||||||
|
|
||||||
type = 4
|
type = 4
|
||||||
rotation_save_cycle = 300 #seconds
|
rotation_save_cycle = 300 #seconds
|
||||||
|
|
||||||
|
@ -38,7 +53,7 @@ if __name__ == "__main__":
|
||||||
if res:
|
if res:
|
||||||
date = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
|
date = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
|
||||||
uuid = res[0][1][0][1][b'uuid'].decode()
|
uuid = res[0][1][0][1][b'uuid'].decode()
|
||||||
data_rel_path = os.path.join('../../data', uuid, str(type))
|
data_rel_path = os.path.join(data_directory, uuid, str(type))
|
||||||
dir_path = os.path.join(data_rel_path, date[0:4], date[4:6], date[6:8])
|
dir_path = os.path.join(data_rel_path, date[0:4], date[4:6], date[6:8])
|
||||||
if not os.path.isdir(dir_path):
|
if not os.path.isdir(dir_path):
|
||||||
os.makedirs(dir_path)
|
os.makedirs(dir_path)
|
||||||
|
|
|
@ -6,6 +6,7 @@ import time
|
||||||
import redis
|
import redis
|
||||||
|
|
||||||
import datetime
|
import datetime
|
||||||
|
import configparser
|
||||||
|
|
||||||
def data_incorrect_format(session_uuid):
|
def data_incorrect_format(session_uuid):
|
||||||
print('Incorrect format')
|
print('Incorrect format')
|
||||||
|
@ -32,6 +33,20 @@ redis_server_analyzer = redis.StrictRedis(
|
||||||
port=port_redis_metadata,
|
port=port_redis_metadata,
|
||||||
db=2)
|
db=2)
|
||||||
|
|
||||||
|
# get file config
|
||||||
|
config_file_server = os.path.join(os.environ['D4_HOME'], 'configs/server.conf')
|
||||||
|
config_server = configparser.ConfigParser()
|
||||||
|
config_server.read(config_file_server)
|
||||||
|
|
||||||
|
# get data directory
|
||||||
|
use_default_save_directory = config_server['Save_Directories'].getboolean('use_default_save_directory')
|
||||||
|
# check if field is None
|
||||||
|
if use_default_save_directory:
|
||||||
|
data_directory = os.path.join(os.environ['D4_HOME'], 'data')
|
||||||
|
else:
|
||||||
|
data_directory = config_server['Save_Directories'].get('save_directory')
|
||||||
|
|
||||||
|
|
||||||
type = 8
|
type = 8
|
||||||
rotation_save_cycle = 300 #seconds
|
rotation_save_cycle = 300 #seconds
|
||||||
|
|
||||||
|
@ -70,7 +85,7 @@ if __name__ == "__main__":
|
||||||
rotate_file = False
|
rotate_file = False
|
||||||
time_file = time.time()
|
time_file = time.time()
|
||||||
date_file = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
|
date_file = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
|
||||||
dir_data_uuid = os.path.join('../../data', uuid, str(type))
|
dir_data_uuid = os.path.join(data_directory, uuid, str(type))
|
||||||
dir_full_path = get_save_dir(dir_data_uuid, date_file[0:4], date_file[4:6], date_file[6:8])
|
dir_full_path = get_save_dir(dir_data_uuid, date_file[0:4], date_file[4:6], date_file[6:8])
|
||||||
filename = '{}-{}-{}-{}-{}.passivedns.txt'.format(uuid, date_file[0:4], date_file[4:6], date_file[6:8], date_file[8:14])
|
filename = '{}-{}-{}-{}-{}.passivedns.txt'.format(uuid, date_file[0:4], date_file[4:6], date_file[6:8], date_file[8:14])
|
||||||
save_path = os.path.join(dir_full_path, filename)
|
save_path = os.path.join(dir_full_path, filename)
|
||||||
|
|
Loading…
Reference in New Issue