From 6b5ec52e28b1c48405f37edf6463f5b62d41d6b5 Mon Sep 17 00:00:00 2001
From: ljaqueme <ljaqueme@opentext.com>
Date: Tue, 11 Jun 2019 12:27:39 -0600
Subject: [PATCH 1/4] make it docker compliance
---
server/Dockerfile.d4-server | 29 +++++++++++++++++++
server/docker-compose.yml | 27 +++++++++++++++++
server/server.py | 10 +++----
server/web/Flask_server.py | 8 ++---
server/workers/workers_1/file_compressor.py | 8 ++---
server/workers/workers_1/worker.py | 8 ++---
server/workers/workers_1/workers_manager.py | 6 ++--
server/workers/workers_2/file_compressor.py | 8 ++---
.../meta_types_modules/MetaTypesDefault.py | 8 ++---
server/workers/workers_2/worker.py | 8 ++---
server/workers/workers_2/workers_manager.py | 6 ++--
server/workers/workers_4/worker.py | 4 +--
server/workers/workers_4/workers_manager.py | 6 ++--
server/workers/workers_8/worker.py | 8 ++---
server/workers/workers_8/workers_manager.py | 6 ++--
15 files changed, 103 insertions(+), 47 deletions(-)
create mode 100644 server/Dockerfile.d4-server
create mode 100644 server/docker-compose.yml
diff --git a/server/Dockerfile.d4-server b/server/Dockerfile.d4-server
new file mode 100644
index 0000000..6f946b8
--- /dev/null
+++ b/server/Dockerfile.d4-server
@@ -0,0 +1,29 @@
+FROM python:3
+
+WORKDIR /usr/src/d4-server
+
+#RUN git clone https://github.com/D4-project/d4-core.git
+RUN git clone https://github.com/trolldbois/d4-core.git
+RUN mv d4-core
+
+# that doesn't work on windows due to linefeeds
+# COPY . .
+
+ENV D4_HOME=/usr/src/d4-server
+RUN pip install --no-cache-dir -r requirement.txt
+
+# move to tls proxy ?
+#WORKDIR /usr/src/d4-server/gen_cert
+#RUN ./gen_root.sh
+#RUN ./gen_cert.sh
+
+# setup a lots of files
+#WORKDIR /usr/src/d4-server/web
+#RUN ./update_web.sh
+
+
+#WORKDIR /usr/src/d4-server
+
+#ENTRYPOINT ["python", "server.py", "-v", "10"]
+
+CMD bash -l
\ No newline at end of file
diff --git a/server/docker-compose.yml b/server/docker-compose.yml
new file mode 100644
index 0000000..d71b8a8
--- /dev/null
+++ b/server/docker-compose.yml
@@ -0,0 +1,27 @@
+version: "3"
+services:
+ redis-stream:
+ image: redis
+ expose:
+ - 6379
+ redis-metadata:
+ image: redis
+ expose:
+ - 6380
+ web:
+ build:
+ context: .
+ dockerfile: Dockerfile.d4-server
+ image: d4-server:latest
+ #build: ./Dockerfile.d4-server
+ depends_on:
+ - redis-stream
+ - redis-metadata
+ environment:
+ - D4_REDIS_STREAM_HOST=redis
+ - D4_REDIS_STREAM_PORT=6379
+ - D4_REDIS_METADATA_HOST=redis
+ - D4_REDIS_METADATA_PORT=6380
+ ports:
+ - "80:80"
+
diff --git a/server/server.py b/server/server.py
index 178b5e1..4cd04fa 100755
--- a/server/server.py
+++ b/server/server.py
@@ -21,7 +21,7 @@ from twisted.internet.protocol import Protocol
from twisted.protocols.policies import TimeoutMixin
hmac_reset = bytearray(32)
-hmac_key = b'private key to change'
+hmac_key = os.getenv('D4_HMAC_KEY', b'private key to change')
accepted_type = [1, 2, 4, 8, 254]
accepted_extended_type = ['ja3-jl']
@@ -33,11 +33,11 @@ header_size = 62
data_default_size_limit = 1000000
default_max_entries_by_stream = 10000
-host_redis_stream = "localhost"
-port_redis_stream = 6379
+host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost")
+port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379))
-host_redis_metadata = "localhost"
-port_redis_metadata= 6380
+host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost")
+port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380))
redis_server_stream = redis.StrictRedis(
host=host_redis_stream,
diff --git a/server/web/Flask_server.py b/server/web/Flask_server.py
index ae8a0a9..81a07b7 100755
--- a/server/web/Flask_server.py
+++ b/server/web/Flask_server.py
@@ -21,8 +21,8 @@ baseUrl = ''
if baseUrl != '':
baseUrl = '/'+baseUrl
-host_redis_stream = "localhost"
-port_redis_stream = 6379
+host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost")
+port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379))
default_max_entries_by_stream = 10000
analyzer_list_max_default_size = 10000
@@ -50,8 +50,8 @@ redis_server_stream = redis.StrictRedis(
db=0,
decode_responses=True)
-host_redis_metadata = "localhost"
-port_redis_metadata= 6380
+host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost")
+port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380))
redis_server_metadata = redis.StrictRedis(
host=host_redis_metadata,
diff --git a/server/workers/workers_1/file_compressor.py b/server/workers/workers_1/file_compressor.py
index 3406566..cf812bc 100755
--- a/server/workers/workers_1/file_compressor.py
+++ b/server/workers/workers_1/file_compressor.py
@@ -46,11 +46,11 @@ def compress_file(file_full_path, session_uuid,i=0):
redis_server_analyzer.ltrim('analyzer:{}:{}'.format(type, analyzer_uuid), 0, analyser_queue_max_size)
-host_redis_stream = "localhost"
-port_redis_stream = 6379
+host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost")
+port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379))
-host_redis_metadata = "localhost"
-port_redis_metadata = 6380
+host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost")
+port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380))
redis_server_stream = redis.StrictRedis(
host=host_redis_stream,
diff --git a/server/workers/workers_1/worker.py b/server/workers/workers_1/worker.py
index a643ed5..0994924 100755
--- a/server/workers/workers_1/worker.py
+++ b/server/workers/workers_1/worker.py
@@ -46,11 +46,11 @@ def compress_file(file_full_path, i=0):
analyser_queue_max_size = analyzer_list_max_default_size
redis_server_analyzer.ltrim('analyzer:{}:{}'.format(type, analyzer_uuid), 0, analyser_queue_max_size)
-host_redis_stream = "localhost"
-port_redis_stream = 6379
+host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost")
+port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379))
-host_redis_metadata = "localhost"
-port_redis_metadata = 6380
+host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost")
+port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380))
redis_server_stream = redis.StrictRedis(
host=host_redis_stream,
diff --git a/server/workers/workers_1/workers_manager.py b/server/workers/workers_1/workers_manager.py
index 15a87ad..0e90289 100755
--- a/server/workers/workers_1/workers_manager.py
+++ b/server/workers/workers_1/workers_manager.py
@@ -6,8 +6,8 @@ import time
import redis
import subprocess
-host_redis_stream = "localhost"
-port_redis_stream = 6379
+host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost")
+port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379))
redis_server_stream = redis.StrictRedis(
host=host_redis_stream,
@@ -18,7 +18,7 @@ type = 1
try:
redis_server_stream.ping()
except redis.exceptions.ConnectionError:
- print('Error: Redis server {}:{}, ConnectionError'.format(host_redis, port_redis))
+ print('Error: Redis server {}:{}, ConnectionError'.format(host_redis_stream, port_redis_stream))
sys.exit(1)
if __name__ == "__main__":
diff --git a/server/workers/workers_2/file_compressor.py b/server/workers/workers_2/file_compressor.py
index 3406566..cf812bc 100755
--- a/server/workers/workers_2/file_compressor.py
+++ b/server/workers/workers_2/file_compressor.py
@@ -46,11 +46,11 @@ def compress_file(file_full_path, session_uuid,i=0):
redis_server_analyzer.ltrim('analyzer:{}:{}'.format(type, analyzer_uuid), 0, analyser_queue_max_size)
-host_redis_stream = "localhost"
-port_redis_stream = 6379
+host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost")
+port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379))
-host_redis_metadata = "localhost"
-port_redis_metadata = 6380
+host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost")
+port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380))
redis_server_stream = redis.StrictRedis(
host=host_redis_stream,
diff --git a/server/workers/workers_2/meta_types_modules/MetaTypesDefault.py b/server/workers/workers_2/meta_types_modules/MetaTypesDefault.py
index 9a540f6..d0a1451 100755
--- a/server/workers/workers_2/meta_types_modules/MetaTypesDefault.py
+++ b/server/workers/workers_2/meta_types_modules/MetaTypesDefault.py
@@ -16,16 +16,16 @@ ROTATION_SAVE_CYCLE = 300 # seconds
MAX_BUFFER_LENGTH = 100000
TYPE = 254
-host_redis_stream = "localhost"
-port_redis_stream = 6379
+host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost")
+port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379))
redis_server_stream = redis.StrictRedis(
host=host_redis_stream,
port=port_redis_stream,
db=0)
-host_redis_metadata = "localhost"
-port_redis_metadata = 6380
+host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost")
+port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380))
redis_server_metadata = redis.StrictRedis(
host=host_redis_metadata,
diff --git a/server/workers/workers_2/worker.py b/server/workers/workers_2/worker.py
index 14f2745..18b1b5f 100755
--- a/server/workers/workers_2/worker.py
+++ b/server/workers/workers_2/worker.py
@@ -10,16 +10,16 @@ import datetime
from meta_types_modules import MetaTypesDefault
-host_redis_stream = "localhost"
-port_redis_stream = 6379
+host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost")
+port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379))
redis_server_stream = redis.StrictRedis(
host=host_redis_stream,
port=port_redis_stream,
db=0)
-host_redis_metadata = "localhost"
-port_redis_metadata = 6380
+host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost")
+port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380))
redis_server_metadata = redis.StrictRedis(
host=host_redis_metadata,
diff --git a/server/workers/workers_2/workers_manager.py b/server/workers/workers_2/workers_manager.py
index d66b873..92b9a08 100755
--- a/server/workers/workers_2/workers_manager.py
+++ b/server/workers/workers_2/workers_manager.py
@@ -6,8 +6,8 @@ import time
import redis
import subprocess
-host_redis_stream = "localhost"
-port_redis_stream = 6379
+host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost")
+port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379))
redis_server_stream = redis.StrictRedis(
host=host_redis_stream,
@@ -18,7 +18,7 @@ type = 2
try:
redis_server_stream.ping()
except redis.exceptions.ConnectionError:
- print('Error: Redis server {}:{}, ConnectionError'.format(host_redis, port_redis))
+ print('Error: Redis server {}:{}, ConnectionError'.format(host_redis_stream, port_redis_stream))
sys.exit(1)
if __name__ == "__main__":
diff --git a/server/workers/workers_4/worker.py b/server/workers/workers_4/worker.py
index 2e044c8..5384459 100755
--- a/server/workers/workers_4/worker.py
+++ b/server/workers/workers_4/worker.py
@@ -12,8 +12,8 @@ def data_incorrect_format(session_uuid):
print('Incorrect format')
sys.exit(1)
-host_redis_stream = "localhost"
-port_redis_stream = 6379
+host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost")
+port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379))
redis_server_stream = redis.StrictRedis(
host=host_redis_stream,
diff --git a/server/workers/workers_4/workers_manager.py b/server/workers/workers_4/workers_manager.py
index 9c1d7e4..ddbbd7d 100755
--- a/server/workers/workers_4/workers_manager.py
+++ b/server/workers/workers_4/workers_manager.py
@@ -6,8 +6,8 @@ import time
import redis
import subprocess
-host_redis_stream = "localhost"
-port_redis_stream = 6379
+host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost")
+port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379))
redis_server_stream = redis.StrictRedis(
host=host_redis_stream,
@@ -18,7 +18,7 @@ type = 4
try:
redis_server_stream.ping()
except redis.exceptions.ConnectionError:
- print('Error: Redis server {}:{}, ConnectionError'.format(host_redis, port_redis))
+ print('Error: Redis server {}:{}, ConnectionError'.format(host_redis_stream, port_redis_stream))
sys.exit(1)
if __name__ == "__main__":
diff --git a/server/workers/workers_8/worker.py b/server/workers/workers_8/worker.py
index 07a406e..8103b11 100755
--- a/server/workers/workers_8/worker.py
+++ b/server/workers/workers_8/worker.py
@@ -14,16 +14,16 @@ def data_incorrect_format(session_uuid):
print('Incorrect format')
sys.exit(1)
-host_redis_stream = "localhost"
-port_redis_stream = 6379
+host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost")
+port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379))
redis_server_stream = redis.StrictRedis(
host=host_redis_stream,
port=port_redis_stream,
db=0)
-host_redis_metadata = "localhost"
-port_redis_metadata = 6380
+host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost")
+port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380))
redis_server_metadata = redis.StrictRedis(
host=host_redis_metadata,
diff --git a/server/workers/workers_8/workers_manager.py b/server/workers/workers_8/workers_manager.py
index c1e64d9..8b99c50 100755
--- a/server/workers/workers_8/workers_manager.py
+++ b/server/workers/workers_8/workers_manager.py
@@ -6,8 +6,8 @@ import time
import redis
import subprocess
-host_redis_stream = "localhost"
-port_redis_stream = 6379
+host_redis_stream = os.getenv('D4_REDIS_STREAM_HOST', "localhost")
+port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379))
redis_server_stream = redis.StrictRedis(
host=host_redis_stream,
@@ -18,7 +18,7 @@ type = 8
try:
redis_server_stream.ping()
except redis.exceptions.ConnectionError:
- print('Error: Redis server {}:{}, ConnectionError'.format(host_redis, port_redis))
+ print('Error: Redis server {}:{}, ConnectionError'.format(host_redis_stream, port_redis_stream))
sys.exit(1)
if __name__ == "__main__":
From c6f21f0b5fa23817e374ffa2096da457e77ddaa9 Mon Sep 17 00:00:00 2001
From: ljaqueme <ljaqueme@opentext.com>
Date: Tue, 11 Jun 2019 13:49:56 -0600
Subject: [PATCH 2/4] that work
---
server/Dockerfile.d4-server | 33 ++++++++++++----------
server/docker-compose.yml | 55 +++++++++++++++++++++++++++++++------
2 files changed, 65 insertions(+), 23 deletions(-)
diff --git a/server/Dockerfile.d4-server b/server/Dockerfile.d4-server
index 6f946b8..ee2d389 100644
--- a/server/Dockerfile.d4-server
+++ b/server/Dockerfile.d4-server
@@ -1,29 +1,34 @@
FROM python:3
-WORKDIR /usr/src/d4-server
-#RUN git clone https://github.com/D4-project/d4-core.git
-RUN git clone https://github.com/trolldbois/d4-core.git
-RUN mv d4-core
-
-# that doesn't work on windows due to linefeeds
+# that doesn't work on windows docker due to linefeeds
+# WORKDIR /usr/src/d4-server
# COPY . .
+## alternate solution
+# RUN git clone https://github.com/D4-project/d4-core.git
+WORKDIR /usr/src/tmp
+RUN git clone https://github.com/trolldbois/d4-core.git
+RUN mv d4-core/server/ /usr/src/d4-server
+WORKDIR /usr/src/d4-server
+
ENV D4_HOME=/usr/src/d4-server
RUN pip install --no-cache-dir -r requirement.txt
# move to tls proxy ?
-#WORKDIR /usr/src/d4-server/gen_cert
-#RUN ./gen_root.sh
-#RUN ./gen_cert.sh
+WORKDIR /usr/src/d4-server/gen_cert
+RUN ./gen_root.sh
+RUN ./gen_cert.sh
# setup a lots of files
-#WORKDIR /usr/src/d4-server/web
-#RUN ./update_web.sh
+WORKDIR /usr/src/d4-server/web
+RUN ./update_web.sh
+WORKDIR /usr/src/d4-server
-#WORKDIR /usr/src/d4-server
+# configure
+RUN cp configs/server.conf.sample configs/server.conf
-#ENTRYPOINT ["python", "server.py", "-v", "10"]
+ENTRYPOINT ["python", "server.py", "-v", "10"]
-CMD bash -l
\ No newline at end of file
+# CMD bash -l
\ No newline at end of file
diff --git a/server/docker-compose.yml b/server/docker-compose.yml
index d71b8a8..2d1e30b 100644
--- a/server/docker-compose.yml
+++ b/server/docker-compose.yml
@@ -2,26 +2,63 @@ version: "3"
services:
redis-stream:
image: redis
- expose:
- - 6379
+ entrypoint:
+ - redis-server
+ - --port 6379
redis-metadata:
image: redis
- expose:
- - 6380
- web:
+ entrypoint:
+ - redis-server
+ - --port 6380
+ d4-server:
build:
context: .
dockerfile: Dockerfile.d4-server
image: d4-server:latest
- #build: ./Dockerfile.d4-server
depends_on:
- redis-stream
- redis-metadata
environment:
- - D4_REDIS_STREAM_HOST=redis
+ - D4_REDIS_STREAM_HOST=redis-stream
- D4_REDIS_STREAM_PORT=6379
- - D4_REDIS_METADATA_HOST=redis
+ - D4_REDIS_METADATA_HOST=redis-metadata
- D4_REDIS_METADATA_PORT=6380
ports:
- - "80:80"
+ - "4443:4443"
+# d4-worker_1:
+# build:
+# context: .
+# dockerfile: Dockerfile.d4-server
+# image: d4-server:latest
+# depends_on:
+# - redis-stream
+# - redis-metadata
+# environment:
+# - D4_REDIS_STREAM_HOST=redis-stream
+# - D4_REDIS_STREAM_PORT=6379
+# - D4_REDIS_METADATA_HOST=redis-metadata
+# - D4_REDIS_METADATA_PORT=6380
+# entrypoint:
+# - bash
+# - -c
+# - "cd workers/workers_1; ./workers_manager.py; read x"
+ d4-web:
+ build:
+ context: .
+ dockerfile: Dockerfile.d4-server
+ image: d4-server:latest
+ depends_on:
+ - redis-stream
+ - redis-metadata
+ environment:
+ - D4_REDIS_STREAM_HOST=redis-stream
+ - D4_REDIS_STREAM_PORT=6379
+ - D4_REDIS_METADATA_HOST=redis-metadata
+ - D4_REDIS_METADATA_PORT=6380
+ entrypoint:
+ - bash
+ - -c
+ - "cd web; ./Flask_server.py; read x"
+ ports:
+ - "8080:7000"
From 91500ba460a69e4e2a78056bf1603b053c344fe0 Mon Sep 17 00:00:00 2001
From: ljaqueme <ljaqueme@opentext.com>
Date: Tue, 11 Jun 2019 16:37:11 -0600
Subject: [PATCH 3/4] that works
---
server/Dockerfile.analyzer-d4-passivedns | 15 +++
server/Dockerfile.d4-server | 5 +-
server/docker-compose.yml | 148 ++++++++++++++++++-----
3 files changed, 139 insertions(+), 29 deletions(-)
create mode 100644 server/Dockerfile.analyzer-d4-passivedns
diff --git a/server/Dockerfile.analyzer-d4-passivedns b/server/Dockerfile.analyzer-d4-passivedns
new file mode 100644
index 0000000..5c37916
--- /dev/null
+++ b/server/Dockerfile.analyzer-d4-passivedns
@@ -0,0 +1,15 @@
+FROM python:3
+
+WORKDIR /usr/src/
+# RUN git clone https://github.com/D4-project/analyzer-d4-passivedns.git
+RUN git clone https://github.com/trolldbois/analyzer-d4-passivedns.git
+WORKDIR /usr/src/analyzer-d4-passivedns
+
+# FIXME typo in requirements.txt filename
+RUN pip install --no-cache-dir -r requirements
+WORKDIR /usr/src/analyzer-d4-passivedns/bin
+
+# should be a config
+# RUN cat /usr/src/analyzer-d4-passivedns/etc/analyzer.conf.sample | sed "s/127.0.0.1/redis-metadata/g" > /usr/src/analyzer-d4-passivedns/etc/analyzer.conf
+# ignore the config and use ENV variables.
+RUN cp ../etc/analyzer.conf.sample ../etc/analyzer.conf
diff --git a/server/Dockerfile.d4-server b/server/Dockerfile.d4-server
index ee2d389..b2824a6 100644
--- a/server/Dockerfile.d4-server
+++ b/server/Dockerfile.d4-server
@@ -26,9 +26,12 @@ RUN ./update_web.sh
WORKDIR /usr/src/d4-server
-# configure
+# Should be using configs instead. but not supported until docker 17.06+
RUN cp configs/server.conf.sample configs/server.conf
+# workers need tcpdump
+RUN apt-get update && apt-get install -y tcpdump
+
ENTRYPOINT ["python", "server.py", "-v", "10"]
# CMD bash -l
\ No newline at end of file
diff --git a/server/docker-compose.yml b/server/docker-compose.yml
index 2d1e30b..c4d7b88 100644
--- a/server/docker-compose.yml
+++ b/server/docker-compose.yml
@@ -1,15 +1,20 @@
+# Should be using configs but not supported until docker 17.06+
+# https://www.d4-project.org/2019/05/28/passive-dns-tutorial.html
+
version: "3"
services:
redis-stream:
image: redis
- entrypoint:
- - redis-server
- - --port 6379
+ command: redis-server --port 6379
+
redis-metadata:
image: redis
- entrypoint:
- - redis-server
- - --port 6380
+ command: redis-server --port 6380
+
+ redis-analyzer:
+ image: redis
+ command: redis-server --port 6400
+
d4-server:
build:
context: .
@@ -25,23 +30,75 @@ services:
- D4_REDIS_METADATA_PORT=6380
ports:
- "4443:4443"
-# d4-worker_1:
-# build:
-# context: .
-# dockerfile: Dockerfile.d4-server
-# image: d4-server:latest
-# depends_on:
-# - redis-stream
-# - redis-metadata
-# environment:
-# - D4_REDIS_STREAM_HOST=redis-stream
-# - D4_REDIS_STREAM_PORT=6379
-# - D4_REDIS_METADATA_HOST=redis-metadata
-# - D4_REDIS_METADATA_PORT=6380
-# entrypoint:
-# - bash
-# - -c
-# - "cd workers/workers_1; ./workers_manager.py; read x"
+
+ d4-worker_1:
+ build:
+ context: .
+ dockerfile: Dockerfile.d4-server
+ image: d4-server:latest
+ depends_on:
+ - redis-stream
+ - redis-metadata
+ environment:
+ - D4_REDIS_STREAM_HOST=redis-stream
+ - D4_REDIS_STREAM_PORT=6379
+ - D4_REDIS_METADATA_HOST=redis-metadata
+ - D4_REDIS_METADATA_PORT=6380
+ entrypoint: bash -c "cd workers/workers_1; ./workers_manager.py; read x"
+ volumes:
+ - d4-data:/usr/src/d4-server/data
+
+ d4-worker_2:
+ build:
+ context: .
+ dockerfile: Dockerfile.d4-server
+ image: d4-server:latest
+ depends_on:
+ - redis-stream
+ - redis-metadata
+ environment:
+ - D4_REDIS_STREAM_HOST=redis-stream
+ - D4_REDIS_STREAM_PORT=6379
+ - D4_REDIS_METADATA_HOST=redis-metadata
+ - D4_REDIS_METADATA_PORT=6380
+ entrypoint: bash -c "cd workers/workers_2; ./workers_manager.py; read x"
+ volumes:
+ - d4-data:/usr/src/d4-server/data
+
+ d4-worker_4:
+ build:
+ context: .
+ dockerfile: Dockerfile.d4-server
+ image: d4-server:latest
+ depends_on:
+ - redis-stream
+ - redis-metadata
+ environment:
+ - D4_REDIS_STREAM_HOST=redis-stream
+ - D4_REDIS_STREAM_PORT=6379
+ - D4_REDIS_METADATA_HOST=redis-metadata
+ - D4_REDIS_METADATA_PORT=6380
+ entrypoint: bash -c "cd workers/workers_4; ./workers_manager.py; read x"
+ volumes:
+ - d4-data:/usr/src/d4-server/data
+
+ d4-worker_8:
+ build:
+ context: .
+ dockerfile: Dockerfile.d4-server
+ image: d4-server:latest
+ depends_on:
+ - redis-stream
+ - redis-metadata
+ environment:
+ - D4_REDIS_STREAM_HOST=redis-stream
+ - D4_REDIS_STREAM_PORT=6379
+ - D4_REDIS_METADATA_HOST=redis-metadata
+ - D4_REDIS_METADATA_PORT=6380
+ entrypoint: bash -c "cd workers/workers_8; ./workers_manager.py; read x"
+ volumes:
+ - d4-data:/usr/src/d4-server/data
+
d4-web:
build:
context: .
@@ -55,10 +112,45 @@ services:
- D4_REDIS_STREAM_PORT=6379
- D4_REDIS_METADATA_HOST=redis-metadata
- D4_REDIS_METADATA_PORT=6380
- entrypoint:
- - bash
- - -c
- - "cd web; ./Flask_server.py; read x"
+ entrypoint: bash -c "cd web; ./Flask_server.py; read x"
ports:
- - "8080:7000"
+ - "7000:7000"
+ volumes:
+ - d4-data:/usr/src/d4-server/data
+ d4-analyzer-passivedns-cof:
+ build:
+ context: .
+ dockerfile: Dockerfile.analyzer-d4-passivedns
+ image: analyzer-d4-passivedns:latest
+ depends_on:
+ - redis-metadata
+ - redis-analyzer
+ environment:
+ - D4_ANALYZER_REDIS_HOST=redis-analyzer
+ - D4_ANALYZER_REDIS_PORT=6400
+ - D4_REDIS_METADATA_HOST=redis-metadata
+ - D4_REDIS_METADATA_PORT=6380
+ - DEBUG=true
+ entrypoint: bash -c "python ./pdns-cof-server.py; read x"
+ ports:
+ - "8400:8400"
+
+ d4-analyzer-passivedns-ingestion:
+ build:
+ context: .
+ dockerfile: Dockerfile.analyzer-d4-passivedns
+ image: analyzer-d4-passivedns:latest
+ depends_on:
+ - redis-metadata
+ - redis-analyzer
+ environment:
+ - D4_ANALYZER_REDIS_HOST=redis-analyzer
+ - D4_ANALYZER_REDIS_PORT=6400
+ - D4_REDIS_METADATA_HOST=redis-metadata
+ - D4_REDIS_METADATA_PORT=6380
+ - DEBUG=true
+ entrypoint: bash -c "python ./pdns-ingestion.py; read x"
+
+volumes:
+ d4-data:
From acb20a769bebb5044dc6020d4ae33a5a1f5460b5 Mon Sep 17 00:00:00 2001
From: ljaqueme <ljaqueme@opentext.com>
Date: Tue, 11 Jun 2019 16:41:31 -0600
Subject: [PATCH 4/4] use d4 github. Will works if pull request is accepted and
Environmental variables allowed to control REDIS servers config
---
server/Dockerfile.analyzer-d4-passivedns | 4 ++--
server/Dockerfile.d4-server | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/server/Dockerfile.analyzer-d4-passivedns b/server/Dockerfile.analyzer-d4-passivedns
index 5c37916..4041c57 100644
--- a/server/Dockerfile.analyzer-d4-passivedns
+++ b/server/Dockerfile.analyzer-d4-passivedns
@@ -1,8 +1,8 @@
FROM python:3
WORKDIR /usr/src/
-# RUN git clone https://github.com/D4-project/analyzer-d4-passivedns.git
-RUN git clone https://github.com/trolldbois/analyzer-d4-passivedns.git
+RUN git clone https://github.com/D4-project/analyzer-d4-passivedns.git
+# RUN git clone https://github.com/trolldbois/analyzer-d4-passivedns.git
WORKDIR /usr/src/analyzer-d4-passivedns
# FIXME typo in requirements.txt filename
diff --git a/server/Dockerfile.d4-server b/server/Dockerfile.d4-server
index b2824a6..b5e2ae3 100644
--- a/server/Dockerfile.d4-server
+++ b/server/Dockerfile.d4-server
@@ -6,9 +6,9 @@ FROM python:3
# COPY . .
## alternate solution
-# RUN git clone https://github.com/D4-project/d4-core.git
WORKDIR /usr/src/tmp
-RUN git clone https://github.com/trolldbois/d4-core.git
+# RUN git clone https://github.com/trolldbois/d4-core.git
+RUN git clone https://github.com/D4-project/d4-core.git
RUN mv d4-core/server/ /usr/src/d4-server
WORKDIR /usr/src/d4-server