From 6b5ec52e28b1c48405f37edf6463f5b62d41d6b5 Mon Sep 17 00:00:00 2001 From: ljaqueme Date: Tue, 11 Jun 2019 12:27:39 -0600 Subject: [PATCH] make it docker compliance --- server/Dockerfile.d4-server | 29 +++++++++++++++++++ server/docker-compose.yml | 27 +++++++++++++++++ server/server.py | 10 +++---- server/web/Flask_server.py | 8 ++--- server/workers/workers_1/file_compressor.py | 8 ++--- server/workers/workers_1/worker.py | 8 ++--- server/workers/workers_1/workers_manager.py | 6 ++-- server/workers/workers_2/file_compressor.py | 8 ++--- .../meta_types_modules/MetaTypesDefault.py | 8 ++--- server/workers/workers_2/worker.py | 8 ++--- server/workers/workers_2/workers_manager.py | 6 ++-- server/workers/workers_4/worker.py | 4 +-- server/workers/workers_4/workers_manager.py | 6 ++-- server/workers/workers_8/worker.py | 8 ++--- server/workers/workers_8/workers_manager.py | 6 ++-- 15 files changed, 103 insertions(+), 47 deletions(-) create mode 100644 server/Dockerfile.d4-server create mode 100644 server/docker-compose.yml diff --git a/server/Dockerfile.d4-server b/server/Dockerfile.d4-server new file mode 100644 index 0000000..6f946b8 --- /dev/null +++ b/server/Dockerfile.d4-server @@ -0,0 +1,29 @@ +FROM python:3 + +WORKDIR /usr/src/d4-server + +#RUN git clone https://github.com/D4-project/d4-core.git +RUN git clone https://github.com/trolldbois/d4-core.git +RUN mv d4-core + +# that doesn't work on windows due to linefeeds +# COPY . . + +ENV D4_HOME=/usr/src/d4-server +RUN pip install --no-cache-dir -r requirement.txt + +# move to tls proxy ? +#WORKDIR /usr/src/d4-server/gen_cert +#RUN ./gen_root.sh +#RUN ./gen_cert.sh + +# setup a lots of files +#WORKDIR /usr/src/d4-server/web +#RUN ./update_web.sh + + +#WORKDIR /usr/src/d4-server + +#ENTRYPOINT ["python", "server.py", "-v", "10"] + +CMD bash -l \ No newline at end of file diff --git a/server/docker-compose.yml b/server/docker-compose.yml new file mode 100644 index 0000000..d71b8a8 --- /dev/null +++ b/server/docker-compose.yml @@ -0,0 +1,27 @@ +version: "3" +services: + redis-stream: + image: redis + expose: + - 6379 + redis-metadata: + image: redis + expose: + - 6380 + web: + build: + context: . + dockerfile: Dockerfile.d4-server + image: d4-server:latest + #build: ./Dockerfile.d4-server + depends_on: + - redis-stream + - redis-metadata + environment: + - D4_REDIS_STREAM_HOST=redis + - D4_REDIS_STREAM_PORT=6379 + - D4_REDIS_METADATA_HOST=redis + - D4_REDIS_METADATA_PORT=6380 + ports: + - "80:80" + diff --git a/server/server.py b/server/server.py index 178b5e1..4cd04fa 100755 --- a/server/server.py +++ b/server/server.py @@ -21,7 +21,7 @@ from twisted.internet.protocol import Protocol from twisted.protocols.policies import TimeoutMixin hmac_reset = bytearray(32) -hmac_key = b'private key to change' +hmac_key = os.getenv('D4_HMAC_KEY', b'private key to change') accepted_type = [1, 2, 4, 8, 254] accepted_extended_type = ['ja3-jl'] @@ -33,11 +33,11 @@ header_size = 62 data_default_size_limit = 1000000 default_max_entries_by_stream = 10000 -host_redis_stream = "localhost" -port_redis_stream = 6379 +host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost") +port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379)) -host_redis_metadata = "localhost" -port_redis_metadata= 6380 +host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost") +port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380)) redis_server_stream = redis.StrictRedis( host=host_redis_stream, diff --git a/server/web/Flask_server.py b/server/web/Flask_server.py index ae8a0a9..81a07b7 100755 --- a/server/web/Flask_server.py +++ b/server/web/Flask_server.py @@ -21,8 +21,8 @@ baseUrl = '' if baseUrl != '': baseUrl = '/'+baseUrl -host_redis_stream = "localhost" -port_redis_stream = 6379 +host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost") +port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379)) default_max_entries_by_stream = 10000 analyzer_list_max_default_size = 10000 @@ -50,8 +50,8 @@ redis_server_stream = redis.StrictRedis( db=0, decode_responses=True) -host_redis_metadata = "localhost" -port_redis_metadata= 6380 +host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost") +port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380)) redis_server_metadata = redis.StrictRedis( host=host_redis_metadata, diff --git a/server/workers/workers_1/file_compressor.py b/server/workers/workers_1/file_compressor.py index 3406566..cf812bc 100755 --- a/server/workers/workers_1/file_compressor.py +++ b/server/workers/workers_1/file_compressor.py @@ -46,11 +46,11 @@ def compress_file(file_full_path, session_uuid,i=0): redis_server_analyzer.ltrim('analyzer:{}:{}'.format(type, analyzer_uuid), 0, analyser_queue_max_size) -host_redis_stream = "localhost" -port_redis_stream = 6379 +host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost") +port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379)) -host_redis_metadata = "localhost" -port_redis_metadata = 6380 +host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost") +port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380)) redis_server_stream = redis.StrictRedis( host=host_redis_stream, diff --git a/server/workers/workers_1/worker.py b/server/workers/workers_1/worker.py index a643ed5..0994924 100755 --- a/server/workers/workers_1/worker.py +++ b/server/workers/workers_1/worker.py @@ -46,11 +46,11 @@ def compress_file(file_full_path, i=0): analyser_queue_max_size = analyzer_list_max_default_size redis_server_analyzer.ltrim('analyzer:{}:{}'.format(type, analyzer_uuid), 0, analyser_queue_max_size) -host_redis_stream = "localhost" -port_redis_stream = 6379 +host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost") +port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379)) -host_redis_metadata = "localhost" -port_redis_metadata = 6380 +host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost") +port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380)) redis_server_stream = redis.StrictRedis( host=host_redis_stream, diff --git a/server/workers/workers_1/workers_manager.py b/server/workers/workers_1/workers_manager.py index 15a87ad..0e90289 100755 --- a/server/workers/workers_1/workers_manager.py +++ b/server/workers/workers_1/workers_manager.py @@ -6,8 +6,8 @@ import time import redis import subprocess -host_redis_stream = "localhost" -port_redis_stream = 6379 +host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost") +port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379)) redis_server_stream = redis.StrictRedis( host=host_redis_stream, @@ -18,7 +18,7 @@ type = 1 try: redis_server_stream.ping() except redis.exceptions.ConnectionError: - print('Error: Redis server {}:{}, ConnectionError'.format(host_redis, port_redis)) + print('Error: Redis server {}:{}, ConnectionError'.format(host_redis_stream, port_redis_stream)) sys.exit(1) if __name__ == "__main__": diff --git a/server/workers/workers_2/file_compressor.py b/server/workers/workers_2/file_compressor.py index 3406566..cf812bc 100755 --- a/server/workers/workers_2/file_compressor.py +++ b/server/workers/workers_2/file_compressor.py @@ -46,11 +46,11 @@ def compress_file(file_full_path, session_uuid,i=0): redis_server_analyzer.ltrim('analyzer:{}:{}'.format(type, analyzer_uuid), 0, analyser_queue_max_size) -host_redis_stream = "localhost" -port_redis_stream = 6379 +host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost") +port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379)) -host_redis_metadata = "localhost" -port_redis_metadata = 6380 +host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost") +port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380)) redis_server_stream = redis.StrictRedis( host=host_redis_stream, diff --git a/server/workers/workers_2/meta_types_modules/MetaTypesDefault.py b/server/workers/workers_2/meta_types_modules/MetaTypesDefault.py index 9a540f6..d0a1451 100755 --- a/server/workers/workers_2/meta_types_modules/MetaTypesDefault.py +++ b/server/workers/workers_2/meta_types_modules/MetaTypesDefault.py @@ -16,16 +16,16 @@ ROTATION_SAVE_CYCLE = 300 # seconds MAX_BUFFER_LENGTH = 100000 TYPE = 254 -host_redis_stream = "localhost" -port_redis_stream = 6379 +host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost") +port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379)) redis_server_stream = redis.StrictRedis( host=host_redis_stream, port=port_redis_stream, db=0) -host_redis_metadata = "localhost" -port_redis_metadata = 6380 +host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost") +port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380)) redis_server_metadata = redis.StrictRedis( host=host_redis_metadata, diff --git a/server/workers/workers_2/worker.py b/server/workers/workers_2/worker.py index 14f2745..18b1b5f 100755 --- a/server/workers/workers_2/worker.py +++ b/server/workers/workers_2/worker.py @@ -10,16 +10,16 @@ import datetime from meta_types_modules import MetaTypesDefault -host_redis_stream = "localhost" -port_redis_stream = 6379 +host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost") +port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379)) redis_server_stream = redis.StrictRedis( host=host_redis_stream, port=port_redis_stream, db=0) -host_redis_metadata = "localhost" -port_redis_metadata = 6380 +host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost") +port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380)) redis_server_metadata = redis.StrictRedis( host=host_redis_metadata, diff --git a/server/workers/workers_2/workers_manager.py b/server/workers/workers_2/workers_manager.py index d66b873..92b9a08 100755 --- a/server/workers/workers_2/workers_manager.py +++ b/server/workers/workers_2/workers_manager.py @@ -6,8 +6,8 @@ import time import redis import subprocess -host_redis_stream = "localhost" -port_redis_stream = 6379 +host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost") +port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379)) redis_server_stream = redis.StrictRedis( host=host_redis_stream, @@ -18,7 +18,7 @@ type = 2 try: redis_server_stream.ping() except redis.exceptions.ConnectionError: - print('Error: Redis server {}:{}, ConnectionError'.format(host_redis, port_redis)) + print('Error: Redis server {}:{}, ConnectionError'.format(host_redis_stream, port_redis_stream)) sys.exit(1) if __name__ == "__main__": diff --git a/server/workers/workers_4/worker.py b/server/workers/workers_4/worker.py index 2e044c8..5384459 100755 --- a/server/workers/workers_4/worker.py +++ b/server/workers/workers_4/worker.py @@ -12,8 +12,8 @@ def data_incorrect_format(session_uuid): print('Incorrect format') sys.exit(1) -host_redis_stream = "localhost" -port_redis_stream = 6379 +host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost") +port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379)) redis_server_stream = redis.StrictRedis( host=host_redis_stream, diff --git a/server/workers/workers_4/workers_manager.py b/server/workers/workers_4/workers_manager.py index 9c1d7e4..ddbbd7d 100755 --- a/server/workers/workers_4/workers_manager.py +++ b/server/workers/workers_4/workers_manager.py @@ -6,8 +6,8 @@ import time import redis import subprocess -host_redis_stream = "localhost" -port_redis_stream = 6379 +host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost") +port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379)) redis_server_stream = redis.StrictRedis( host=host_redis_stream, @@ -18,7 +18,7 @@ type = 4 try: redis_server_stream.ping() except redis.exceptions.ConnectionError: - print('Error: Redis server {}:{}, ConnectionError'.format(host_redis, port_redis)) + print('Error: Redis server {}:{}, ConnectionError'.format(host_redis_stream, port_redis_stream)) sys.exit(1) if __name__ == "__main__": diff --git a/server/workers/workers_8/worker.py b/server/workers/workers_8/worker.py index 07a406e..8103b11 100755 --- a/server/workers/workers_8/worker.py +++ b/server/workers/workers_8/worker.py @@ -14,16 +14,16 @@ def data_incorrect_format(session_uuid): print('Incorrect format') sys.exit(1) -host_redis_stream = "localhost" -port_redis_stream = 6379 +host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost") +port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379)) redis_server_stream = redis.StrictRedis( host=host_redis_stream, port=port_redis_stream, db=0) -host_redis_metadata = "localhost" -port_redis_metadata = 6380 +host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost") +port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380)) redis_server_metadata = redis.StrictRedis( host=host_redis_metadata, diff --git a/server/workers/workers_8/workers_manager.py b/server/workers/workers_8/workers_manager.py index c1e64d9..8b99c50 100755 --- a/server/workers/workers_8/workers_manager.py +++ b/server/workers/workers_8/workers_manager.py @@ -6,8 +6,8 @@ import time import redis import subprocess -host_redis_stream = "localhost" -port_redis_stream = 6379 +host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost") +port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379)) redis_server_stream = redis.StrictRedis( host=host_redis_stream, @@ -18,7 +18,7 @@ type = 8 try: redis_server_stream.ping() except redis.exceptions.ConnectionError: - print('Error: Redis server {}:{}, ConnectionError'.format(host_redis, port_redis)) + print('Error: Redis server {}:{}, ConnectionError'.format(host_redis_stream, port_redis_stream)) sys.exit(1) if __name__ == "__main__":