D4
/
d4-core
mirror of https://github.com/D4-project/d4-core
2
0
Fork 0
Code Issues Releases Wiki Activity

chg: [worker8] compress files

pull/23/head
Terrtia 2019-04-04 14:51:49 +02:00
parent 45546fd4a2
commit f726237c65
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
1 changed files with 22 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
server/workers/workers_8/worker.py
View File

@ -3,8 +3,10 @@
import os import os
import sys import sys
import time import time
import gzip
import redis import redis
import shutil
import datetime import datetime
import configparser import configparser
@ -56,6 +58,19 @@ max_buffer_length = 10000
save_to_file = True save_to_file = True
def compress_file(file_full_path, i=0):
if i==0:
compressed_filename = '{}.gz'.format(file_full_path)
else:
compressed_filename = '{}.{}.gz'.format(file_full_path, i)
if os.path.isfile(compressed_filename):
compress_file(file_full_path, i+1)
else:
with open(file_full_path, 'rb') as f_in:
with gzip.open(compressed_filename, 'wb') as f_out:
shutil.copyfileobj(f_in, f_out)
os.remove(file_full_path)
def get_save_dir(dir_data_uuid, year, month, day): def get_save_dir(dir_data_uuid, year, month, day):
dir_path = os.path.join(dir_data_uuid, year, month, day) dir_path = os.path.join(dir_data_uuid, year, month, day)
if not os.path.isdir(dir_path): if not os.path.isdir(dir_path):
@ -150,6 +165,7 @@ if __name__ == "__main__":
# save end of file # save end of file
with open(save_path, 'ab') as f: with open(save_path, 'ab') as f:
f.write(end_file) f.write(end_file)
compress_file(save_path)
# get new save_path # get new save_path
dir_full_path = get_save_dir(dir_data_uuid, date_file[0:4], date_file[4:6], date_file[6:8]) dir_full_path = get_save_dir(dir_data_uuid, date_file[0:4], date_file[4:6], date_file[6:8])
@ -178,6 +194,12 @@ if __name__ == "__main__":
redis_server_stream.srem('working_session_uuid:{}'.format(type), session_uuid) redis_server_stream.srem('working_session_uuid:{}'.format(type), session_uuid)
redis_server_stream.hdel('map-type:session_uuid-uuid:{}'.format(type), session_uuid) redis_server_stream.hdel('map-type:session_uuid-uuid:{}'.format(type), session_uuid)
redis_server_stream.delete(stream_name) redis_server_stream.delete(stream_name)
try:
if os.path.isfile(save_path):
print('save')
compress_file(save_path)
except NameError:
pass
print('---- passivedns DONE, uuid={} session_uuid={} epoch={}'.format(uuid, session_uuid, time.time())) print('---- passivedns DONE, uuid={} session_uuid={} epoch={}'.format(uuid, session_uuid, time.time()))
sys.exit(0) sys.exit(0)
else: else: