make it docker compliance

server/Dockerfile.d4-server

@@ -0,0 +1,29 @@
+FROM python:3
+
+WORKDIR /usr/src/d4-server
+
+#RUN git clone https://github.com/D4-project/d4-core.git
+RUN git clone https://github.com/trolldbois/d4-core.git
+RUN mv d4-core
+
+# that doesn't work on windows due to linefeeds
+# COPY . .
+
+ENV D4_HOME=/usr/src/d4-server
+RUN pip install --no-cache-dir -r requirement.txt
+
+# move to tls proxy ?
+#WORKDIR /usr/src/d4-server/gen_cert
+#RUN ./gen_root.sh
+#RUN ./gen_cert.sh
+
+# setup a lots of files
+#WORKDIR /usr/src/d4-server/web
+#RUN ./update_web.sh
+
+
+#WORKDIR /usr/src/d4-server
+
+#ENTRYPOINT ["python", "server.py", "-v", "10"]
+
+CMD bash -l

server/docker-compose.yml

@@ -0,0 +1,27 @@
+version: "3"
+services:
+  redis-stream:
+    image: redis
+    expose:
+      - 6379
+  redis-metadata:
+    image: redis
+    expose:
+      - 6380
+  web:
+    build:
+      context: .
+      dockerfile: Dockerfile.d4-server
+    image: d4-server:latest
+    #build: ./Dockerfile.d4-server
+    depends_on:
+          - redis-stream
+          - redis-metadata
+    environment:
+      - D4_REDIS_STREAM_HOST=redis
+      - D4_REDIS_STREAM_PORT=6379
+      - D4_REDIS_METADATA_HOST=redis
+      - D4_REDIS_METADATA_PORT=6380
+    ports:
+      - "80:80"
+

server/server.py

@@ -21,7 +21,7 @@ from twisted.internet.protocol import Protocol
 from twisted.protocols.policies import TimeoutMixin
 
 hmac_reset = bytearray(32)
-hmac_key = b'private key to change'
+hmac_key = os.getenv('D4_HMAC_KEY', b'private key to change')
 
 accepted_type = [1, 2, 4, 8, 254]
 accepted_extended_type = ['ja3-jl']
@@ -33,11 +33,11 @@ header_size = 62
 data_default_size_limit = 1000000
 default_max_entries_by_stream = 10000
 
-host_redis_stream = "localhost"
+host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost")
-port_redis_stream = 6379
+port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379))
 
-host_redis_metadata = "localhost"
+host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost")
-port_redis_metadata= 6380
+port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380))
 
 redis_server_stream = redis.StrictRedis(
                     host=host_redis_stream,

server/web/Flask_server.py

@@ -21,8 +21,8 @@ baseUrl = ''
 if baseUrl != '':
     baseUrl = '/'+baseUrl
 
-host_redis_stream = "localhost"
+host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost")
-port_redis_stream = 6379
+port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379))
 
 default_max_entries_by_stream = 10000
 analyzer_list_max_default_size = 10000
@@ -50,8 +50,8 @@ redis_server_stream = redis.StrictRedis(
                     db=0,
                     decode_responses=True)
 
-host_redis_metadata = "localhost"
+host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost")
-port_redis_metadata= 6380
+port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380))
 
 redis_server_metadata = redis.StrictRedis(
                     host=host_redis_metadata,

server/workers/workers_1/file_compressor.py

@@ -46,11 +46,11 @@ def compress_file(file_full_path, session_uuid,i=0):
             redis_server_analyzer.ltrim('analyzer:{}:{}'.format(type, analyzer_uuid), 0, analyser_queue_max_size)
 
 
-host_redis_stream = "localhost"
+host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost")
-port_redis_stream = 6379
+port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379))
 
-host_redis_metadata = "localhost"
+host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost")
-port_redis_metadata = 6380
+port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380))
 
 redis_server_stream = redis.StrictRedis(
                     host=host_redis_stream,

server/workers/workers_1/worker.py

@@ -46,11 +46,11 @@ def compress_file(file_full_path, i=0):
                 analyser_queue_max_size = analyzer_list_max_default_size
             redis_server_analyzer.ltrim('analyzer:{}:{}'.format(type, analyzer_uuid), 0, analyser_queue_max_size)
 
-host_redis_stream = "localhost"
+host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost")
-port_redis_stream = 6379
+port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379))
 
-host_redis_metadata = "localhost"
+host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost")
-port_redis_metadata = 6380
+port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380))
 
 redis_server_stream = redis.StrictRedis(
                     host=host_redis_stream,

server/workers/workers_1/workers_manager.py

@@ -6,8 +6,8 @@ import time
 import redis
 import subprocess
 
-host_redis_stream = "localhost"
+host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost")
-port_redis_stream = 6379
+port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379))
 
 redis_server_stream = redis.StrictRedis(
                     host=host_redis_stream,
@@ -18,7 +18,7 @@ type = 1
 try:
     redis_server_stream.ping()
 except redis.exceptions.ConnectionError:
-    print('Error: Redis server {}:{}, ConnectionError'.format(host_redis, port_redis))
+    print('Error: Redis server {}:{}, ConnectionError'.format(host_redis_stream, port_redis_stream))
     sys.exit(1)
 
 if __name__ == "__main__":

server/workers/workers_2/file_compressor.py

@@ -46,11 +46,11 @@ def compress_file(file_full_path, session_uuid,i=0):
             redis_server_analyzer.ltrim('analyzer:{}:{}'.format(type, analyzer_uuid), 0, analyser_queue_max_size)
 
 
-host_redis_stream = "localhost"
+host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost")
-port_redis_stream = 6379
+port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379))
 
-host_redis_metadata = "localhost"
+host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost")
-port_redis_metadata = 6380
+port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380))
 
 redis_server_stream = redis.StrictRedis(
                     host=host_redis_stream,

server/workers/workers_2/meta_types_modules/MetaTypesDefault.py

@@ -16,16 +16,16 @@ ROTATION_SAVE_CYCLE = 300 # seconds
 MAX_BUFFER_LENGTH = 100000
 TYPE = 254
 
-host_redis_stream = "localhost"
+host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost")
-port_redis_stream = 6379
+port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379))
 
 redis_server_stream = redis.StrictRedis(
                     host=host_redis_stream,
                     port=port_redis_stream,
                     db=0)
 
-host_redis_metadata = "localhost"
+host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost")
-port_redis_metadata = 6380
+port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380))
 
 redis_server_metadata = redis.StrictRedis(
                     host=host_redis_metadata,

server/workers/workers_2/worker.py

@@ -10,16 +10,16 @@ import datetime
 
 from meta_types_modules import MetaTypesDefault
 
-host_redis_stream = "localhost"
+host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost")
-port_redis_stream = 6379
+port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379))
 
 redis_server_stream = redis.StrictRedis(
                     host=host_redis_stream,
                     port=port_redis_stream,
                     db=0)
 
-host_redis_metadata = "localhost"
+host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost")
-port_redis_metadata = 6380
+port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380))
 
 redis_server_metadata = redis.StrictRedis(
                     host=host_redis_metadata,

server/workers/workers_2/workers_manager.py

@@ -6,8 +6,8 @@ import time
 import redis
 import subprocess
 
-host_redis_stream = "localhost"
+host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost")
-port_redis_stream = 6379
+port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379))
 
 redis_server_stream = redis.StrictRedis(
                     host=host_redis_stream,
@@ -18,7 +18,7 @@ type = 2
 try:
     redis_server_stream.ping()
 except redis.exceptions.ConnectionError:
-    print('Error: Redis server {}:{}, ConnectionError'.format(host_redis, port_redis))
+    print('Error: Redis server {}:{}, ConnectionError'.format(host_redis_stream, port_redis_stream))
     sys.exit(1)
 
 if __name__ == "__main__":

server/workers/workers_4/worker.py

@@ -12,8 +12,8 @@ def data_incorrect_format(session_uuid):
     print('Incorrect format')
     sys.exit(1)
 
-host_redis_stream = "localhost"
+host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost")
-port_redis_stream = 6379
+port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379))
 
 redis_server_stream = redis.StrictRedis(
                     host=host_redis_stream,

server/workers/workers_4/workers_manager.py

@@ -6,8 +6,8 @@ import time
 import redis
 import subprocess
 
-host_redis_stream = "localhost"
+host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost")
-port_redis_stream = 6379
+port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379))
 
 redis_server_stream = redis.StrictRedis(
                     host=host_redis_stream,
@@ -18,7 +18,7 @@ type = 4
 try:
     redis_server_stream.ping()
 except redis.exceptions.ConnectionError:
-    print('Error: Redis server {}:{}, ConnectionError'.format(host_redis, port_redis))
+    print('Error: Redis server {}:{}, ConnectionError'.format(host_redis_stream, port_redis_stream))
     sys.exit(1)
 
 if __name__ == "__main__":

server/workers/workers_8/worker.py

@@ -14,16 +14,16 @@ def data_incorrect_format(session_uuid):
     print('Incorrect format')
     sys.exit(1)
 
-host_redis_stream = "localhost"
+host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost")
-port_redis_stream = 6379
+port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379))
 
 redis_server_stream = redis.StrictRedis(
                     host=host_redis_stream,
                     port=port_redis_stream,
                     db=0)
 
-host_redis_metadata = "localhost"
+host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost")
-port_redis_metadata = 6380
+port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380))
 
 redis_server_metadata = redis.StrictRedis(
                     host=host_redis_metadata,

server/workers/workers_8/workers_manager.py

@@ -6,8 +6,8 @@ import time
 import redis
 import subprocess
 
-host_redis_stream = "localhost"
+host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost")
-port_redis_stream = 6379
+port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379))
 
 redis_server_stream = redis.StrictRedis(
                     host=host_redis_stream,
@@ -18,7 +18,7 @@ type = 8
 try:
     redis_server_stream.ping()
 except redis.exceptions.ConnectionError:
-    print('Error: Redis server {}:{}, ConnectionError'.format(host_redis, port_redis))
+    print('Error: Redis server {}:{}, ConnectionError'.format(host_redis_stream, port_redis_stream))
     sys.exit(1)
 
 if __name__ == "__main__":