diff --git a/server/Dockerfile.analyzer-d4-passivedns b/server/Dockerfile.analyzer-d4-passivedns new file mode 100644 index 0000000..4041c57 --- /dev/null +++ b/server/Dockerfile.analyzer-d4-passivedns @@ -0,0 +1,15 @@ +FROM python:3 + +WORKDIR /usr/src/ +RUN git clone https://github.com/D4-project/analyzer-d4-passivedns.git +# RUN git clone https://github.com/trolldbois/analyzer-d4-passivedns.git +WORKDIR /usr/src/analyzer-d4-passivedns + +# FIXME typo in requirements.txt filename +RUN pip install --no-cache-dir -r requirements +WORKDIR /usr/src/analyzer-d4-passivedns/bin + +# should be a config +# RUN cat /usr/src/analyzer-d4-passivedns/etc/analyzer.conf.sample | sed "s/127.0.0.1/redis-metadata/g" > /usr/src/analyzer-d4-passivedns/etc/analyzer.conf +# ignore the config and use ENV variables. +RUN cp ../etc/analyzer.conf.sample ../etc/analyzer.conf diff --git a/server/Dockerfile.d4-server b/server/Dockerfile.d4-server new file mode 100644 index 0000000..b5e2ae3 --- /dev/null +++ b/server/Dockerfile.d4-server @@ -0,0 +1,37 @@ +FROM python:3 + + +# that doesn't work on windows docker due to linefeeds +# WORKDIR /usr/src/d4-server +# COPY . . + +## alternate solution +WORKDIR /usr/src/tmp +# RUN git clone https://github.com/trolldbois/d4-core.git +RUN git clone https://github.com/D4-project/d4-core.git +RUN mv d4-core/server/ /usr/src/d4-server +WORKDIR /usr/src/d4-server + +ENV D4_HOME=/usr/src/d4-server +RUN pip install --no-cache-dir -r requirement.txt + +# move to tls proxy ? +WORKDIR /usr/src/d4-server/gen_cert +RUN ./gen_root.sh +RUN ./gen_cert.sh + +# setup a lots of files +WORKDIR /usr/src/d4-server/web +RUN ./update_web.sh + +WORKDIR /usr/src/d4-server + +# Should be using configs instead. but not supported until docker 17.06+ +RUN cp configs/server.conf.sample configs/server.conf + +# workers need tcpdump +RUN apt-get update && apt-get install -y tcpdump + +ENTRYPOINT ["python", "server.py", "-v", "10"] + +# CMD bash -l \ No newline at end of file diff --git a/server/docker-compose.yml b/server/docker-compose.yml new file mode 100644 index 0000000..c4d7b88 --- /dev/null +++ b/server/docker-compose.yml @@ -0,0 +1,156 @@ +# Should be using configs but not supported until docker 17.06+ +# https://www.d4-project.org/2019/05/28/passive-dns-tutorial.html + +version: "3" +services: + redis-stream: + image: redis + command: redis-server --port 6379 + + redis-metadata: + image: redis + command: redis-server --port 6380 + + redis-analyzer: + image: redis + command: redis-server --port 6400 + + d4-server: + build: + context: . + dockerfile: Dockerfile.d4-server + image: d4-server:latest + depends_on: + - redis-stream + - redis-metadata + environment: + - D4_REDIS_STREAM_HOST=redis-stream + - D4_REDIS_STREAM_PORT=6379 + - D4_REDIS_METADATA_HOST=redis-metadata + - D4_REDIS_METADATA_PORT=6380 + ports: + - "4443:4443" + + d4-worker_1: + build: + context: . + dockerfile: Dockerfile.d4-server + image: d4-server:latest + depends_on: + - redis-stream + - redis-metadata + environment: + - D4_REDIS_STREAM_HOST=redis-stream + - D4_REDIS_STREAM_PORT=6379 + - D4_REDIS_METADATA_HOST=redis-metadata + - D4_REDIS_METADATA_PORT=6380 + entrypoint: bash -c "cd workers/workers_1; ./workers_manager.py; read x" + volumes: + - d4-data:/usr/src/d4-server/data + + d4-worker_2: + build: + context: . + dockerfile: Dockerfile.d4-server + image: d4-server:latest + depends_on: + - redis-stream + - redis-metadata + environment: + - D4_REDIS_STREAM_HOST=redis-stream + - D4_REDIS_STREAM_PORT=6379 + - D4_REDIS_METADATA_HOST=redis-metadata + - D4_REDIS_METADATA_PORT=6380 + entrypoint: bash -c "cd workers/workers_2; ./workers_manager.py; read x" + volumes: + - d4-data:/usr/src/d4-server/data + + d4-worker_4: + build: + context: . + dockerfile: Dockerfile.d4-server + image: d4-server:latest + depends_on: + - redis-stream + - redis-metadata + environment: + - D4_REDIS_STREAM_HOST=redis-stream + - D4_REDIS_STREAM_PORT=6379 + - D4_REDIS_METADATA_HOST=redis-metadata + - D4_REDIS_METADATA_PORT=6380 + entrypoint: bash -c "cd workers/workers_4; ./workers_manager.py; read x" + volumes: + - d4-data:/usr/src/d4-server/data + + d4-worker_8: + build: + context: . + dockerfile: Dockerfile.d4-server + image: d4-server:latest + depends_on: + - redis-stream + - redis-metadata + environment: + - D4_REDIS_STREAM_HOST=redis-stream + - D4_REDIS_STREAM_PORT=6379 + - D4_REDIS_METADATA_HOST=redis-metadata + - D4_REDIS_METADATA_PORT=6380 + entrypoint: bash -c "cd workers/workers_8; ./workers_manager.py; read x" + volumes: + - d4-data:/usr/src/d4-server/data + + d4-web: + build: + context: . + dockerfile: Dockerfile.d4-server + image: d4-server:latest + depends_on: + - redis-stream + - redis-metadata + environment: + - D4_REDIS_STREAM_HOST=redis-stream + - D4_REDIS_STREAM_PORT=6379 + - D4_REDIS_METADATA_HOST=redis-metadata + - D4_REDIS_METADATA_PORT=6380 + entrypoint: bash -c "cd web; ./Flask_server.py; read x" + ports: + - "7000:7000" + volumes: + - d4-data:/usr/src/d4-server/data + + d4-analyzer-passivedns-cof: + build: + context: . + dockerfile: Dockerfile.analyzer-d4-passivedns + image: analyzer-d4-passivedns:latest + depends_on: + - redis-metadata + - redis-analyzer + environment: + - D4_ANALYZER_REDIS_HOST=redis-analyzer + - D4_ANALYZER_REDIS_PORT=6400 + - D4_REDIS_METADATA_HOST=redis-metadata + - D4_REDIS_METADATA_PORT=6380 + - DEBUG=true + entrypoint: bash -c "python ./pdns-cof-server.py; read x" + ports: + - "8400:8400" + + d4-analyzer-passivedns-ingestion: + build: + context: . + dockerfile: Dockerfile.analyzer-d4-passivedns + image: analyzer-d4-passivedns:latest + depends_on: + - redis-metadata + - redis-analyzer + environment: + - D4_ANALYZER_REDIS_HOST=redis-analyzer + - D4_ANALYZER_REDIS_PORT=6400 + - D4_REDIS_METADATA_HOST=redis-metadata + - D4_REDIS_METADATA_PORT=6380 + - DEBUG=true + entrypoint: bash -c "python ./pdns-ingestion.py; read x" + +volumes: + d4-data: diff --git a/server/server.py b/server/server.py index 178b5e1..4cd04fa 100755 --- a/server/server.py +++ b/server/server.py @@ -21,7 +21,7 @@ from twisted.internet.protocol import Protocol from twisted.protocols.policies import TimeoutMixin hmac_reset = bytearray(32) -hmac_key = b'private key to change' +hmac_key = os.getenv('D4_HMAC_KEY', b'private key to change') accepted_type = [1, 2, 4, 8, 254] accepted_extended_type = ['ja3-jl'] @@ -33,11 +33,11 @@ header_size = 62 data_default_size_limit = 1000000 default_max_entries_by_stream = 10000 -host_redis_stream = "localhost" -port_redis_stream = 6379 +host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost") +port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379)) -host_redis_metadata = "localhost" -port_redis_metadata= 6380 +host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost") +port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380)) redis_server_stream = redis.StrictRedis( host=host_redis_stream, diff --git a/server/web/Flask_server.py b/server/web/Flask_server.py index ae8a0a9..81a07b7 100755 --- a/server/web/Flask_server.py +++ b/server/web/Flask_server.py @@ -21,8 +21,8 @@ baseUrl = '' if baseUrl != '': baseUrl = '/'+baseUrl -host_redis_stream = "localhost" -port_redis_stream = 6379 +host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost") +port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379)) default_max_entries_by_stream = 10000 analyzer_list_max_default_size = 10000 @@ -50,8 +50,8 @@ redis_server_stream = redis.StrictRedis( db=0, decode_responses=True) -host_redis_metadata = "localhost" -port_redis_metadata= 6380 +host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost") +port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380)) redis_server_metadata = redis.StrictRedis( host=host_redis_metadata, diff --git a/server/workers/workers_1/file_compressor.py b/server/workers/workers_1/file_compressor.py index 3406566..cf812bc 100755 --- a/server/workers/workers_1/file_compressor.py +++ b/server/workers/workers_1/file_compressor.py @@ -46,11 +46,11 @@ def compress_file(file_full_path, session_uuid,i=0): redis_server_analyzer.ltrim('analyzer:{}:{}'.format(type, analyzer_uuid), 0, analyser_queue_max_size) -host_redis_stream = "localhost" -port_redis_stream = 6379 +host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost") +port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379)) -host_redis_metadata = "localhost" -port_redis_metadata = 6380 +host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost") +port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380)) redis_server_stream = redis.StrictRedis( host=host_redis_stream, diff --git a/server/workers/workers_1/worker.py b/server/workers/workers_1/worker.py index a643ed5..0994924 100755 --- a/server/workers/workers_1/worker.py +++ b/server/workers/workers_1/worker.py @@ -46,11 +46,11 @@ def compress_file(file_full_path, i=0): analyser_queue_max_size = analyzer_list_max_default_size redis_server_analyzer.ltrim('analyzer:{}:{}'.format(type, analyzer_uuid), 0, analyser_queue_max_size) -host_redis_stream = "localhost" -port_redis_stream = 6379 +host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost") +port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379)) -host_redis_metadata = "localhost" -port_redis_metadata = 6380 +host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost") +port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380)) redis_server_stream = redis.StrictRedis( host=host_redis_stream, diff --git a/server/workers/workers_1/workers_manager.py b/server/workers/workers_1/workers_manager.py index 15a87ad..0e90289 100755 --- a/server/workers/workers_1/workers_manager.py +++ b/server/workers/workers_1/workers_manager.py @@ -6,8 +6,8 @@ import time import redis import subprocess -host_redis_stream = "localhost" -port_redis_stream = 6379 +host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost") +port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379)) redis_server_stream = redis.StrictRedis( host=host_redis_stream, @@ -18,7 +18,7 @@ type = 1 try: redis_server_stream.ping() except redis.exceptions.ConnectionError: - print('Error: Redis server {}:{}, ConnectionError'.format(host_redis, port_redis)) + print('Error: Redis server {}:{}, ConnectionError'.format(host_redis_stream, port_redis_stream)) sys.exit(1) if __name__ == "__main__": diff --git a/server/workers/workers_2/file_compressor.py b/server/workers/workers_2/file_compressor.py index 3406566..cf812bc 100755 --- a/server/workers/workers_2/file_compressor.py +++ b/server/workers/workers_2/file_compressor.py @@ -46,11 +46,11 @@ def compress_file(file_full_path, session_uuid,i=0): redis_server_analyzer.ltrim('analyzer:{}:{}'.format(type, analyzer_uuid), 0, analyser_queue_max_size) -host_redis_stream = "localhost" -port_redis_stream = 6379 +host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost") +port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379)) -host_redis_metadata = "localhost" -port_redis_metadata = 6380 +host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost") +port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380)) redis_server_stream = redis.StrictRedis( host=host_redis_stream, diff --git a/server/workers/workers_2/meta_types_modules/MetaTypesDefault.py b/server/workers/workers_2/meta_types_modules/MetaTypesDefault.py index 9a540f6..d0a1451 100755 --- a/server/workers/workers_2/meta_types_modules/MetaTypesDefault.py +++ b/server/workers/workers_2/meta_types_modules/MetaTypesDefault.py @@ -16,16 +16,16 @@ ROTATION_SAVE_CYCLE = 300 # seconds MAX_BUFFER_LENGTH = 100000 TYPE = 254 -host_redis_stream = "localhost" -port_redis_stream = 6379 +host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost") +port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379)) redis_server_stream = redis.StrictRedis( host=host_redis_stream, port=port_redis_stream, db=0) -host_redis_metadata = "localhost" -port_redis_metadata = 6380 +host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost") +port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380)) redis_server_metadata = redis.StrictRedis( host=host_redis_metadata, diff --git a/server/workers/workers_2/worker.py b/server/workers/workers_2/worker.py index 14f2745..18b1b5f 100755 --- a/server/workers/workers_2/worker.py +++ b/server/workers/workers_2/worker.py @@ -10,16 +10,16 @@ import datetime from meta_types_modules import MetaTypesDefault -host_redis_stream = "localhost" -port_redis_stream = 6379 +host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost") +port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379)) redis_server_stream = redis.StrictRedis( host=host_redis_stream, port=port_redis_stream, db=0) -host_redis_metadata = "localhost" -port_redis_metadata = 6380 +host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost") +port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380)) redis_server_metadata = redis.StrictRedis( host=host_redis_metadata, diff --git a/server/workers/workers_2/workers_manager.py b/server/workers/workers_2/workers_manager.py index d66b873..92b9a08 100755 --- a/server/workers/workers_2/workers_manager.py +++ b/server/workers/workers_2/workers_manager.py @@ -6,8 +6,8 @@ import time import redis import subprocess -host_redis_stream = "localhost" -port_redis_stream = 6379 +host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost") +port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379)) redis_server_stream = redis.StrictRedis( host=host_redis_stream, @@ -18,7 +18,7 @@ type = 2 try: redis_server_stream.ping() except redis.exceptions.ConnectionError: - print('Error: Redis server {}:{}, ConnectionError'.format(host_redis, port_redis)) + print('Error: Redis server {}:{}, ConnectionError'.format(host_redis_stream, port_redis_stream)) sys.exit(1) if __name__ == "__main__": diff --git a/server/workers/workers_4/worker.py b/server/workers/workers_4/worker.py index 2e044c8..5384459 100755 --- a/server/workers/workers_4/worker.py +++ b/server/workers/workers_4/worker.py @@ -12,8 +12,8 @@ def data_incorrect_format(session_uuid): print('Incorrect format') sys.exit(1) -host_redis_stream = "localhost" -port_redis_stream = 6379 +host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost") +port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379)) redis_server_stream = redis.StrictRedis( host=host_redis_stream, diff --git a/server/workers/workers_4/workers_manager.py b/server/workers/workers_4/workers_manager.py index 9c1d7e4..ddbbd7d 100755 --- a/server/workers/workers_4/workers_manager.py +++ b/server/workers/workers_4/workers_manager.py @@ -6,8 +6,8 @@ import time import redis import subprocess -host_redis_stream = "localhost" -port_redis_stream = 6379 +host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost") +port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379)) redis_server_stream = redis.StrictRedis( host=host_redis_stream, @@ -18,7 +18,7 @@ type = 4 try: redis_server_stream.ping() except redis.exceptions.ConnectionError: - print('Error: Redis server {}:{}, ConnectionError'.format(host_redis, port_redis)) + print('Error: Redis server {}:{}, ConnectionError'.format(host_redis_stream, port_redis_stream)) sys.exit(1) if __name__ == "__main__": diff --git a/server/workers/workers_8/worker.py b/server/workers/workers_8/worker.py index 07a406e..8103b11 100755 --- a/server/workers/workers_8/worker.py +++ b/server/workers/workers_8/worker.py @@ -14,16 +14,16 @@ def data_incorrect_format(session_uuid): print('Incorrect format') sys.exit(1) -host_redis_stream = "localhost" -port_redis_stream = 6379 +host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost") +port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379)) redis_server_stream = redis.StrictRedis( host=host_redis_stream, port=port_redis_stream, db=0) -host_redis_metadata = "localhost" -port_redis_metadata = 6380 +host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost") +port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380)) redis_server_metadata = redis.StrictRedis( host=host_redis_metadata, diff --git a/server/workers/workers_8/workers_manager.py b/server/workers/workers_8/workers_manager.py index c1e64d9..8b99c50 100755 --- a/server/workers/workers_8/workers_manager.py +++ b/server/workers/workers_8/workers_manager.py @@ -6,8 +6,8 @@ import time import redis import subprocess -host_redis_stream = "localhost" -port_redis_stream = 6379 +host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost") +port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379)) redis_server_stream = redis.StrictRedis( host=host_redis_stream, @@ -18,7 +18,7 @@ type = 8 try: redis_server_stream.ping() except redis.exceptions.ConnectionError: - print('Error: Redis server {}:{}, ConnectionError'.format(host_redis, port_redis)) + print('Error: Redis server {}:{}, ConnectionError'.format(host_redis_stream, port_redis_stream)) sys.exit(1) if __name__ == "__main__":