Merge pull request #23 from trolldbois/master

Use Environmental variables for redis & Docker container configuration files

server/Dockerfile.analyzer-d4-passivedns

@@ -0,0 +1,15 @@
+FROM python:3
+
+WORKDIR /usr/src/
+RUN git clone https://github.com/D4-project/analyzer-d4-passivedns.git
+# RUN git clone https://github.com/trolldbois/analyzer-d4-passivedns.git
+WORKDIR /usr/src/analyzer-d4-passivedns
+
+# FIXME typo in requirements.txt filename
+RUN pip install --no-cache-dir -r requirements
+WORKDIR /usr/src/analyzer-d4-passivedns/bin
+
+# should be a config
+# RUN cat /usr/src/analyzer-d4-passivedns/etc/analyzer.conf.sample | sed "s/127.0.0.1/redis-metadata/g" > /usr/src/analyzer-d4-passivedns/etc/analyzer.conf
+# ignore the config and use ENV variables.
+RUN cp ../etc/analyzer.conf.sample ../etc/analyzer.conf

server/Dockerfile.d4-server

@@ -0,0 +1,37 @@
+FROM python:3
+
+
+# that doesn't work on windows docker due to linefeeds
+# WORKDIR /usr/src/d4-server
+# COPY . .
+
+## alternate solution
+WORKDIR /usr/src/tmp
+# RUN git clone https://github.com/trolldbois/d4-core.git
+RUN git clone https://github.com/D4-project/d4-core.git
+RUN mv d4-core/server/ /usr/src/d4-server
+WORKDIR /usr/src/d4-server
+
+ENV D4_HOME=/usr/src/d4-server
+RUN pip install --no-cache-dir -r requirement.txt
+
+# move to tls proxy ?
+WORKDIR /usr/src/d4-server/gen_cert
+RUN ./gen_root.sh
+RUN ./gen_cert.sh
+
+# setup a lots of files
+WORKDIR /usr/src/d4-server/web
+RUN ./update_web.sh
+
+WORKDIR /usr/src/d4-server
+
+# Should be using configs instead. but not supported until docker 17.06+
+RUN cp configs/server.conf.sample configs/server.conf
+
+# workers need tcpdump
+RUN apt-get update && apt-get install -y tcpdump
+
+ENTRYPOINT ["python", "server.py", "-v", "10"]
+
+# CMD bash -l

server/docker-compose.yml

@@ -0,0 +1,156 @@
+# Should be using configs but not supported until docker 17.06+
+# https://www.d4-project.org/2019/05/28/passive-dns-tutorial.html
+
+version: "3"
+services:
+  redis-stream:
+    image: redis
+    command: redis-server --port 6379
+
+  redis-metadata:
+    image: redis
+    command: redis-server --port 6380
+
+  redis-analyzer:
+    image: redis
+    command: redis-server --port 6400
+
+  d4-server:
+    build:
+      context: .
+      dockerfile: Dockerfile.d4-server
+    image: d4-server:latest
+    depends_on:
+          - redis-stream
+          - redis-metadata
+    environment:
+      - D4_REDIS_STREAM_HOST=redis-stream
+      - D4_REDIS_STREAM_PORT=6379
+      - D4_REDIS_METADATA_HOST=redis-metadata
+      - D4_REDIS_METADATA_PORT=6380
+    ports:
+      - "4443:4443"
+
+  d4-worker_1:
+    build:
+      context: .
+      dockerfile: Dockerfile.d4-server
+    image: d4-server:latest
+    depends_on:
+          - redis-stream
+          - redis-metadata
+    environment:
+      - D4_REDIS_STREAM_HOST=redis-stream
+      - D4_REDIS_STREAM_PORT=6379
+      - D4_REDIS_METADATA_HOST=redis-metadata
+      - D4_REDIS_METADATA_PORT=6380
+    entrypoint: bash -c "cd workers/workers_1; ./workers_manager.py; read x"
+    volumes:
+       - d4-data:/usr/src/d4-server/data
+
+  d4-worker_2:
+    build:
+      context: .
+      dockerfile: Dockerfile.d4-server
+    image: d4-server:latest
+    depends_on:
+          - redis-stream
+          - redis-metadata
+    environment:
+      - D4_REDIS_STREAM_HOST=redis-stream
+      - D4_REDIS_STREAM_PORT=6379
+      - D4_REDIS_METADATA_HOST=redis-metadata
+      - D4_REDIS_METADATA_PORT=6380
+    entrypoint: bash -c "cd workers/workers_2; ./workers_manager.py; read x"
+    volumes:
+       - d4-data:/usr/src/d4-server/data
+
+  d4-worker_4:
+    build:
+      context: .
+      dockerfile: Dockerfile.d4-server
+    image: d4-server:latest
+    depends_on:
+          - redis-stream
+          - redis-metadata
+    environment:
+      - D4_REDIS_STREAM_HOST=redis-stream
+      - D4_REDIS_STREAM_PORT=6379
+      - D4_REDIS_METADATA_HOST=redis-metadata
+      - D4_REDIS_METADATA_PORT=6380
+    entrypoint: bash -c "cd workers/workers_4; ./workers_manager.py; read x"
+    volumes:
+       - d4-data:/usr/src/d4-server/data
+
+  d4-worker_8:
+    build:
+      context: .
+      dockerfile: Dockerfile.d4-server
+    image: d4-server:latest
+    depends_on:
+          - redis-stream
+          - redis-metadata
+    environment:
+      - D4_REDIS_STREAM_HOST=redis-stream
+      - D4_REDIS_STREAM_PORT=6379
+      - D4_REDIS_METADATA_HOST=redis-metadata
+      - D4_REDIS_METADATA_PORT=6380
+    entrypoint: bash -c "cd workers/workers_8; ./workers_manager.py; read x"
+    volumes:
+       - d4-data:/usr/src/d4-server/data
+
+  d4-web:
+    build:
+      context: .
+      dockerfile: Dockerfile.d4-server
+    image: d4-server:latest
+    depends_on:
+          - redis-stream
+          - redis-metadata
+    environment:
+      - D4_REDIS_STREAM_HOST=redis-stream
+      - D4_REDIS_STREAM_PORT=6379
+      - D4_REDIS_METADATA_HOST=redis-metadata
+      - D4_REDIS_METADATA_PORT=6380
+    entrypoint: bash -c "cd web; ./Flask_server.py; read x"
+    ports:
+      - "7000:7000"
+    volumes:
+       - d4-data:/usr/src/d4-server/data
+
+  d4-analyzer-passivedns-cof:
+    build:
+      context: .
+      dockerfile: Dockerfile.analyzer-d4-passivedns
+    image: analyzer-d4-passivedns:latest
+    depends_on:
+          - redis-metadata
+          - redis-analyzer
+    environment:
+      - D4_ANALYZER_REDIS_HOST=redis-analyzer
+      - D4_ANALYZER_REDIS_PORT=6400
+      - D4_REDIS_METADATA_HOST=redis-metadata
+      - D4_REDIS_METADATA_PORT=6380
+      - DEBUG=true
+    entrypoint: bash -c "python ./pdns-cof-server.py; read x"
+    ports:
+      - "8400:8400"
+
+  d4-analyzer-passivedns-ingestion:
+    build:
+      context: .
+      dockerfile: Dockerfile.analyzer-d4-passivedns
+    image: analyzer-d4-passivedns:latest
+    depends_on:
+          - redis-metadata
+          - redis-analyzer
+    environment:
+      - D4_ANALYZER_REDIS_HOST=redis-analyzer
+      - D4_ANALYZER_REDIS_PORT=6400
+      - D4_REDIS_METADATA_HOST=redis-metadata
+      - D4_REDIS_METADATA_PORT=6380
+      - DEBUG=true
+    entrypoint: bash -c "python ./pdns-ingestion.py; read x"
+
+volumes:
+  d4-data:

server/server.py

@@ -21,7 +21,7 @@ from twisted.internet.protocol import Protocol
 from twisted.protocols.policies import TimeoutMixin
 
 hmac_reset = bytearray(32)
-hmac_key = b'private key to change'
+hmac_key = os.getenv('D4_HMAC_KEY', b'private key to change')
 
 accepted_type = [1, 2, 4, 8, 254]
 accepted_extended_type = ['ja3-jl']
@@ -33,11 +33,11 @@ header_size = 62
 data_default_size_limit = 1000000
 default_max_entries_by_stream = 10000
 
-host_redis_stream = "localhost"
+host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost")
-port_redis_stream = 6379
+port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379))
 
-host_redis_metadata = "localhost"
+host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost")
-port_redis_metadata= 6380
+port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380))
 
 redis_server_stream = redis.StrictRedis(
                     host=host_redis_stream,

server/web/Flask_server.py

@@ -21,8 +21,8 @@ baseUrl = ''
 if baseUrl != '':
     baseUrl = '/'+baseUrl
 
-host_redis_stream = "localhost"
+host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost")
-port_redis_stream = 6379
+port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379))
 
 default_max_entries_by_stream = 10000
 analyzer_list_max_default_size = 10000
@@ -50,8 +50,8 @@ redis_server_stream = redis.StrictRedis(
                     db=0,
                     decode_responses=True)
 
-host_redis_metadata = "localhost"
+host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost")
-port_redis_metadata= 6380
+port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380))
 
 redis_server_metadata = redis.StrictRedis(
                     host=host_redis_metadata,

server/workers/workers_1/file_compressor.py

@@ -46,11 +46,11 @@ def compress_file(file_full_path, session_uuid,i=0):
             redis_server_analyzer.ltrim('analyzer:{}:{}'.format(type, analyzer_uuid), 0, analyser_queue_max_size)
 
 
-host_redis_stream = "localhost"
+host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost")
-port_redis_stream = 6379
+port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379))
 
-host_redis_metadata = "localhost"
+host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost")
-port_redis_metadata = 6380
+port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380))
 
 redis_server_stream = redis.StrictRedis(
                     host=host_redis_stream,

server/workers/workers_1/worker.py

@@ -46,11 +46,11 @@ def compress_file(file_full_path, i=0):
                 analyser_queue_max_size = analyzer_list_max_default_size
             redis_server_analyzer.ltrim('analyzer:{}:{}'.format(type, analyzer_uuid), 0, analyser_queue_max_size)
 
-host_redis_stream = "localhost"
+host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost")
-port_redis_stream = 6379
+port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379))
 
-host_redis_metadata = "localhost"
+host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost")
-port_redis_metadata = 6380
+port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380))
 
 redis_server_stream = redis.StrictRedis(
                     host=host_redis_stream,

server/workers/workers_1/workers_manager.py

@@ -6,8 +6,8 @@ import time
 import redis
 import subprocess
 
-host_redis_stream = "localhost"
+host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost")
-port_redis_stream = 6379
+port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379))
 
 redis_server_stream = redis.StrictRedis(
                     host=host_redis_stream,
@@ -18,7 +18,7 @@ type = 1
 try:
     redis_server_stream.ping()
 except redis.exceptions.ConnectionError:
-    print('Error: Redis server {}:{}, ConnectionError'.format(host_redis, port_redis))
+    print('Error: Redis server {}:{}, ConnectionError'.format(host_redis_stream, port_redis_stream))
     sys.exit(1)
 
 if __name__ == "__main__":

server/workers/workers_2/file_compressor.py

@@ -46,11 +46,11 @@ def compress_file(file_full_path, session_uuid,i=0):
             redis_server_analyzer.ltrim('analyzer:{}:{}'.format(type, analyzer_uuid), 0, analyser_queue_max_size)
 
 
-host_redis_stream = "localhost"
+host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost")
-port_redis_stream = 6379
+port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379))
 
-host_redis_metadata = "localhost"
+host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost")
-port_redis_metadata = 6380
+port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380))
 
 redis_server_stream = redis.StrictRedis(
                     host=host_redis_stream,

server/workers/workers_2/meta_types_modules/MetaTypesDefault.py

@@ -16,16 +16,16 @@ ROTATION_SAVE_CYCLE = 300 # seconds
 MAX_BUFFER_LENGTH = 100000
 TYPE = 254
 
-host_redis_stream = "localhost"
+host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost")
-port_redis_stream = 6379
+port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379))
 
 redis_server_stream = redis.StrictRedis(
                     host=host_redis_stream,
                     port=port_redis_stream,
                     db=0)
 
-host_redis_metadata = "localhost"
+host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost")
-port_redis_metadata = 6380
+port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380))
 
 redis_server_metadata = redis.StrictRedis(
                     host=host_redis_metadata,

server/workers/workers_2/worker.py

@@ -10,16 +10,16 @@ import datetime
 
 from meta_types_modules import MetaTypesDefault
 
-host_redis_stream = "localhost"
+host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost")
-port_redis_stream = 6379
+port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379))
 
 redis_server_stream = redis.StrictRedis(
                     host=host_redis_stream,
                     port=port_redis_stream,
                     db=0)
 
-host_redis_metadata = "localhost"
+host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost")
-port_redis_metadata = 6380
+port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380))
 
 redis_server_metadata = redis.StrictRedis(
                     host=host_redis_metadata,

server/workers/workers_2/workers_manager.py

@@ -6,8 +6,8 @@ import time
 import redis
 import subprocess
 
-host_redis_stream = "localhost"
+host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost")
-port_redis_stream = 6379
+port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379))
 
 redis_server_stream = redis.StrictRedis(
                     host=host_redis_stream,
@@ -18,7 +18,7 @@ type = 2
 try:
     redis_server_stream.ping()
 except redis.exceptions.ConnectionError:
-    print('Error: Redis server {}:{}, ConnectionError'.format(host_redis, port_redis))
+    print('Error: Redis server {}:{}, ConnectionError'.format(host_redis_stream, port_redis_stream))
     sys.exit(1)
 
 if __name__ == "__main__":

server/workers/workers_4/worker.py

@@ -12,8 +12,8 @@ def data_incorrect_format(session_uuid):
     print('Incorrect format')
     sys.exit(1)
 
-host_redis_stream = "localhost"
+host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost")
-port_redis_stream = 6379
+port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379))
 
 redis_server_stream = redis.StrictRedis(
                     host=host_redis_stream,

server/workers/workers_4/workers_manager.py

@@ -6,8 +6,8 @@ import time
 import redis
 import subprocess
 
-host_redis_stream = "localhost"
+host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost")
-port_redis_stream = 6379
+port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379))
 
 redis_server_stream = redis.StrictRedis(
                     host=host_redis_stream,
@@ -18,7 +18,7 @@ type = 4
 try:
     redis_server_stream.ping()
 except redis.exceptions.ConnectionError:
-    print('Error: Redis server {}:{}, ConnectionError'.format(host_redis, port_redis))
+    print('Error: Redis server {}:{}, ConnectionError'.format(host_redis_stream, port_redis_stream))
     sys.exit(1)
 
 if __name__ == "__main__":

server/workers/workers_8/worker.py

@@ -14,16 +14,16 @@ def data_incorrect_format(session_uuid):
     print('Incorrect format')
     sys.exit(1)
 
-host_redis_stream = "localhost"
+host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost")
-port_redis_stream = 6379
+port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379))
 
 redis_server_stream = redis.StrictRedis(
                     host=host_redis_stream,
                     port=port_redis_stream,
                     db=0)
 
-host_redis_metadata = "localhost"
+host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost")
-port_redis_metadata = 6380
+port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380))
 
 redis_server_metadata = redis.StrictRedis(
                     host=host_redis_metadata,

server/workers/workers_8/workers_manager.py

@@ -6,8 +6,8 @@ import time
 import redis
 import subprocess
 
-host_redis_stream = "localhost"
+host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost")
-port_redis_stream = 6379
+port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379))
 
 redis_server_stream = redis.StrictRedis(
                     host=host_redis_stream,
@@ -18,7 +18,7 @@ type = 8
 try:
     redis_server_stream.ping()
 except redis.exceptions.ConnectionError:
-    print('Error: Redis server {}:{}, ConnectionError'.format(host_redis, port_redis))
+    print('Error: Redis server {}:{}, ConnectionError'.format(host_redis_stream, port_redis_stream))
     sys.exit(1)
 
 if __name__ == "__main__":