diff --git a/server/Dockerfile.analyzer-d4-passivedns b/server/Dockerfile.analyzer-d4-passivedns new file mode 100644 index 0000000..5c37916 --- /dev/null +++ b/server/Dockerfile.analyzer-d4-passivedns @@ -0,0 +1,15 @@ +FROM python:3 + +WORKDIR /usr/src/ +# RUN git clone https://github.com/D4-project/analyzer-d4-passivedns.git +RUN git clone https://github.com/trolldbois/analyzer-d4-passivedns.git +WORKDIR /usr/src/analyzer-d4-passivedns + +# FIXME typo in requirements.txt filename +RUN pip install --no-cache-dir -r requirements +WORKDIR /usr/src/analyzer-d4-passivedns/bin + +# should be a config +# RUN cat /usr/src/analyzer-d4-passivedns/etc/analyzer.conf.sample | sed "s/127.0.0.1/redis-metadata/g" > /usr/src/analyzer-d4-passivedns/etc/analyzer.conf +# ignore the config and use ENV variables. +RUN cp ../etc/analyzer.conf.sample ../etc/analyzer.conf diff --git a/server/Dockerfile.d4-server b/server/Dockerfile.d4-server index ee2d389..b2824a6 100644 --- a/server/Dockerfile.d4-server +++ b/server/Dockerfile.d4-server @@ -26,9 +26,12 @@ RUN ./update_web.sh WORKDIR /usr/src/d4-server -# configure +# Should be using configs instead. but not supported until docker 17.06+ RUN cp configs/server.conf.sample configs/server.conf +# workers need tcpdump +RUN apt-get update && apt-get install -y tcpdump + ENTRYPOINT ["python", "server.py", "-v", "10"] # CMD bash -l \ No newline at end of file diff --git a/server/docker-compose.yml b/server/docker-compose.yml index 2d1e30b..c4d7b88 100644 --- a/server/docker-compose.yml +++ b/server/docker-compose.yml @@ -1,15 +1,20 @@ +# Should be using configs but not supported until docker 17.06+ +# https://www.d4-project.org/2019/05/28/passive-dns-tutorial.html + version: "3" services: redis-stream: image: redis - entrypoint: - - redis-server - - --port 6379 + command: redis-server --port 6379 + redis-metadata: image: redis - entrypoint: - - redis-server - - --port 6380 + command: redis-server --port 6380 + + redis-analyzer: + image: redis + command: redis-server --port 6400 + d4-server: build: context: . @@ -25,23 +30,75 @@ services: - D4_REDIS_METADATA_PORT=6380 ports: - "4443:4443" -# d4-worker_1: -# build: -# context: . -# dockerfile: Dockerfile.d4-server -# image: d4-server:latest -# depends_on: -# - redis-stream -# - redis-metadata -# environment: -# - D4_REDIS_STREAM_HOST=redis-stream -# - D4_REDIS_STREAM_PORT=6379 -# - D4_REDIS_METADATA_HOST=redis-metadata -# - D4_REDIS_METADATA_PORT=6380 -# entrypoint: -# - bash -# - -c -# - "cd workers/workers_1; ./workers_manager.py; read x" + + d4-worker_1: + build: + context: . + dockerfile: Dockerfile.d4-server + image: d4-server:latest + depends_on: + - redis-stream + - redis-metadata + environment: + - D4_REDIS_STREAM_HOST=redis-stream + - D4_REDIS_STREAM_PORT=6379 + - D4_REDIS_METADATA_HOST=redis-metadata + - D4_REDIS_METADATA_PORT=6380 + entrypoint: bash -c "cd workers/workers_1; ./workers_manager.py; read x" + volumes: + - d4-data:/usr/src/d4-server/data + + d4-worker_2: + build: + context: . + dockerfile: Dockerfile.d4-server + image: d4-server:latest + depends_on: + - redis-stream + - redis-metadata + environment: + - D4_REDIS_STREAM_HOST=redis-stream + - D4_REDIS_STREAM_PORT=6379 + - D4_REDIS_METADATA_HOST=redis-metadata + - D4_REDIS_METADATA_PORT=6380 + entrypoint: bash -c "cd workers/workers_2; ./workers_manager.py; read x" + volumes: + - d4-data:/usr/src/d4-server/data + + d4-worker_4: + build: + context: . + dockerfile: Dockerfile.d4-server + image: d4-server:latest + depends_on: + - redis-stream + - redis-metadata + environment: + - D4_REDIS_STREAM_HOST=redis-stream + - D4_REDIS_STREAM_PORT=6379 + - D4_REDIS_METADATA_HOST=redis-metadata + - D4_REDIS_METADATA_PORT=6380 + entrypoint: bash -c "cd workers/workers_4; ./workers_manager.py; read x" + volumes: + - d4-data:/usr/src/d4-server/data + + d4-worker_8: + build: + context: . + dockerfile: Dockerfile.d4-server + image: d4-server:latest + depends_on: + - redis-stream + - redis-metadata + environment: + - D4_REDIS_STREAM_HOST=redis-stream + - D4_REDIS_STREAM_PORT=6379 + - D4_REDIS_METADATA_HOST=redis-metadata + - D4_REDIS_METADATA_PORT=6380 + entrypoint: bash -c "cd workers/workers_8; ./workers_manager.py; read x" + volumes: + - d4-data:/usr/src/d4-server/data + d4-web: build: context: . @@ -55,10 +112,45 @@ services: - D4_REDIS_STREAM_PORT=6379 - D4_REDIS_METADATA_HOST=redis-metadata - D4_REDIS_METADATA_PORT=6380 - entrypoint: - - bash - - -c - - "cd web; ./Flask_server.py; read x" + entrypoint: bash -c "cd web; ./Flask_server.py; read x" ports: - - "8080:7000" + - "7000:7000" + volumes: + - d4-data:/usr/src/d4-server/data + d4-analyzer-passivedns-cof: + build: + context: . + dockerfile: Dockerfile.analyzer-d4-passivedns + image: analyzer-d4-passivedns:latest + depends_on: + - redis-metadata + - redis-analyzer + environment: + - D4_ANALYZER_REDIS_HOST=redis-analyzer + - D4_ANALYZER_REDIS_PORT=6400 + - D4_REDIS_METADATA_HOST=redis-metadata + - D4_REDIS_METADATA_PORT=6380 + - DEBUG=true + entrypoint: bash -c "python ./pdns-cof-server.py; read x" + ports: + - "8400:8400" + + d4-analyzer-passivedns-ingestion: + build: + context: . + dockerfile: Dockerfile.analyzer-d4-passivedns + image: analyzer-d4-passivedns:latest + depends_on: + - redis-metadata + - redis-analyzer + environment: + - D4_ANALYZER_REDIS_HOST=redis-analyzer + - D4_ANALYZER_REDIS_PORT=6400 + - D4_REDIS_METADATA_HOST=redis-metadata + - D4_REDIS_METADATA_PORT=6380 + - DEBUG=true + entrypoint: bash -c "python ./pdns-ingestion.py; read x" + +volumes: + d4-data: