D4
/
d4-core
mirror of https://github.com/D4-project/d4-core
2
0
Fork 0
Code Issues Releases Wiki Activity

chg: [server + workers] add config file + add option to specify save directory

pull/23/head
Terrtia 2019-03-26 15:20:06 +01:00
parent c7ea2d7863
commit f67224e2e9
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
5 changed files with 75 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
server/LAUNCH.sh
View File

@ -205,9 +205,15 @@ function update_web {
fi
}
function update_config {
echo -e $GREEN"\t* Updating Config File"$DEFAULT
bash -c "(cd ${D4_HOME}/configs; ./update_conf.py -v 0)"
}
function launch_all {
helptext;
launch_redis;
update_config;
launch_d4_server;
launch_workers;
launch_flask;

19
server/workers/workers_1/worker.py
View File

@ -8,6 +8,7 @@ import redis
import shutil
import datetime
import subprocess
import configparser
def data_incorrect_format(stream_name, session_uuid, uuid):
redis_server_stream.sadd('Error:IncorrectType', session_uuid)
@ -66,6 +67,20 @@ redis_server_analyzer = redis.StrictRedis(
port=port_redis_metadata,
db=2)
# get file config
config_file_server = os.path.join(os.environ['D4_HOME'], 'configs/server.conf')
config_server = configparser.ConfigParser()
config_server.read(config_file_server)
# get data directory
use_default_save_directory = config_server['Save_Directories'].getboolean('use_default_save_directory')
# check if field is None
if use_default_save_directory:
data_directory = os.path.join(os.environ['D4_HOME'], 'data')
else:
data_directory = config_server['Save_Directories'].get('save_directory')
type = 1
tcp_dump_cycle = '300'
stream_buffer = 100
@ -88,8 +103,8 @@ if __name__ == "__main__":
if res:
uuid = res[0][1][0][1][b'uuid'].decode()
date = datetime.datetime.now().strftime("%Y%m%d")
tcpdump_path = os.path.join('../../data', uuid, str(type))
full_tcpdump_path = os.path.join(os.environ['D4_HOME'], 'data', uuid, str(type))
tcpdump_path = os.path.join(data_directory, uuid, str(type))
full_tcpdump_path = os.path.join(data_directory, uuid, str(type))
rel_path = os.path.join(tcpdump_path, date[0:4], date[4:6], date[6:8])
if not os.path.isdir(rel_path):
os.makedirs(rel_path)

22
server/workers/workers_2/meta_types_modules/MetaTypesDefault.py
View File

@ -8,6 +8,7 @@ import gzip
import redis
import shutil
import datetime
import configparser
DEFAULT_FILE_EXTENSION = 'txt'
DEFAULT_FILE_SEPARATOR = b'\n'
@ -23,6 +24,20 @@ class MetaTypesDefault:
self.save_path = None
self.buffer = b''
self.file_rotation_mode = True
# get file config
config_file_server = os.path.join(os.environ['D4_HOME'], 'configs/server.conf')
config_server = configparser.ConfigParser()
config_server.read(config_file_server)
# get data directory
use_default_save_directory = config_server['Save_Directories'].getboolean('use_default_save_directory')
# check if field is None
if use_default_save_directory:
data_directory = os.path.join(os.environ['D4_HOME'], 'data')
else:
data_directory = config_server['Save_Directories'].get('save_directory')
self.data_directory = data_directory
self.parse_json(json_file)
def test(self):
@ -182,15 +197,18 @@ class MetaTypesDefault:
if self.is_file_rotation_mode() or save_by_uuid:
return '{}-{}-{}-{}-{}.{}'.format(self.uuid, self.get_last_saved_year(), self.get_last_saved_month(), self.get_last_saved_day(), self.get_last_saved_hour_minute(), file_extention)
def get_data_save_directory():
return self.data_directory
def get_save_dir(self, save_by_uuid=False):
# File Rotation, save data in directory: data/<uuid>/254/<year>/<month>/<day>/
if self.is_file_rotation_mode() or save_by_uuid:
data_directory_uuid_type = os.path.join('../../data', self.get_uuid(), str(TYPE))
data_directory_uuid_type = os.path.join(self.get_data_save_directory(), self.get_uuid(), str(TYPE))
return os.path.join(data_directory_uuid_type, self.get_last_saved_year(), self.get_last_saved_month(), self.get_last_saved_day() , self.type_name)
# data save in the same directory
else:
save_dir = os.path.join('../../data/datas', self.get_type_name())
save_dir = os.path.join(self.get_data_save_directory(), 'datas', self.get_type_name())
if not os.path.isdir(save_dir):
os.makedirs(save_dir)
return save_dir

17
server/workers/workers_4/worker.py
View File

@ -6,6 +6,7 @@ import time
import redis
import datetime
import configparser
def data_incorrect_format(session_uuid):
print('Incorrect format')
@ -19,6 +20,20 @@ redis_server_stream = redis.StrictRedis(
port=port_redis_stream,
db=0)
# get file config
config_file_server = os.path.join(os.environ['D4_HOME'], 'configs/server.conf')
config_server = configparser.ConfigParser()
config_server.read(config_file_server)
# get data directory
use_default_save_directory = config_server['Save_Directories'].getboolean('use_default_save_directory')
# check if field is None
if use_default_save_directory:
data_directory = os.path.join(os.environ['D4_HOME'], 'data')
else:
data_directory = config_server['Save_Directories'].get('save_directory')
type = 4
rotation_save_cycle = 300 #seconds
@ -38,7 +53,7 @@ if __name__ == "__main__":
if res:
date = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
uuid = res[0][1][0][1][b'uuid'].decode()
data_rel_path = os.path.join('../../data', uuid, str(type))
data_rel_path = os.path.join(data_directory, uuid, str(type))
dir_path = os.path.join(data_rel_path, date[0:4], date[4:6], date[6:8])
if not os.path.isdir(dir_path):
os.makedirs(dir_path)

17
server/workers/workers_8/worker.py
View File

@ -6,6 +6,7 @@ import time
import redis
import datetime
import configparser
def data_incorrect_format(session_uuid):
print('Incorrect format')
@ -32,6 +33,20 @@ redis_server_analyzer = redis.StrictRedis(
port=port_redis_metadata,
db=2)
# get file config
config_file_server = os.path.join(os.environ['D4_HOME'], 'configs/server.conf')
config_server = configparser.ConfigParser()
config_server.read(config_file_server)
# get data directory
use_default_save_directory = config_server['Save_Directories'].getboolean('use_default_save_directory')
# check if field is None
if use_default_save_directory:
data_directory = os.path.join(os.environ['D4_HOME'], 'data')
else:
data_directory = config_server['Save_Directories'].get('save_directory')
type = 8
rotation_save_cycle = 300 #seconds
@ -70,7 +85,7 @@ if __name__ == "__main__":
rotate_file = False
time_file = time.time()
date_file = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
dir_data_uuid = os.path.join('../../data', uuid, str(type))
dir_data_uuid = os.path.join(data_directory, uuid, str(type))
dir_full_path = get_save_dir(dir_data_uuid, date_file[0:4], date_file[4:6], date_file[6:8])
filename = '{}-{}-{}-{}-{}.passivedns.txt'.format(uuid, date_file[0:4], date_file[4:6], date_file[6:8], date_file[8:14])
save_path = os.path.join(dir_full_path, filename)