Add docker-compose support

This commit separates the AIL Framework services into individual Docker containers that can be managed with docker-compose. It's intended to ease the installation, development and troubleshooting procedures for all-in-one deployments. No changes to architecture or base code are required in this commit. Changes to existing files: .gitignore * Ignore pystemon archive directory Dockerfile: * Add AIL shell environment variables * Install pystemon and crawler pip requirements into AIL virtual environment Docker-compose notes: * All containers are participating in a service network `network_mode: service:flask`. This allows the containers to share the same IP namespace to accommodate hard-coded localhost entries. * By default persistent data is saved to the following directories in the local AIL framework git directory: PASTES, HASHS, CRAWLED_SCREENSHOTS, pystemon/archives This was tested with docker-ce on Ubuntu 16.04 and MacOS. A typical deployment would look like: ``` git clone https://github.com/CIRCL/AIL-framework.git cd AIL-framework cp bin/packages/config.cfg.docker-compose-sample bin/packages/config.cfg (optionally enable activate_crawler in config.cfg) docker-compose build docker-compose up -d ```
2019-03-11 17:04:02 -05:00 · 2019-03-11 17:04:02 -05:00 · 23fb91d326
parent c825ea6b2b
commit 23fb91d326
5 changed files with 788 additions and 0 deletions
--- a/.gitignore
+++ b/.gitignore
@ -36,6 +36,9 @@ bin/packages/config.cfg.backup
 configs/keys
 files

+# Pystemon archives
+pystemon/archives
+
 # installed files
 nltk_data/
 doc/all_modules.txt
--- a/10
+++ b/10
@ -27,8 +27,18 @@ WORKDIR /opt/AIL

 # Default to UTF-8 file.encoding
 ENV LANG C.UTF-8
+ENV AIL_HOME /opt/AIL
+ENV AIL_BIN ${AIL_HOME}/bin
+ENV AIL_FLASK ${AIL_HOME}/var/www
+ENV AIL_REDIS ${AIL_HOME}/redis/src
+ENV AIL_ARDB ${AIL_HOME}/ardb/src
+ENV AIL_VENV ${AIL_HOME}/AILENV
+
+ENV PATH ${AIL_VENV}/bin:${AIL_HOME}:${AIL_REDIS}:${AIL_ARDB}:${AIL_BIN}:${AIL_FLASK}:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin

 RUN ./pystemon/install.sh
+RUN pip install -r /opt/pystemon/requirements.txt
+RUN pip install -r /opt/AIL/crawler_requirements.txt

 COPY docker_start.sh /docker_start.sh
 ENTRYPOINT ["/bin/bash", "docker_start.sh"]
--- a/bin/packages/config.cfg.docker-compose-sample
+++ b/bin/packages/config.cfg.docker-compose-sample
@ -0,0 +1,253 @@
+[Directories]
+bloomfilters = Blooms
+dicofilters = Dicos
+pastes = PASTES
+hash = HASHS
+crawled = crawled
+crawled_screenshot = CRAWLED_SCREENSHOT
+
+wordtrending_csv = var/www/static/csv/wordstrendingdata
+wordsfile = files/wordfile
+
+protocolstrending_csv = var/www/static/csv/protocolstrendingdata
+protocolsfile = files/protocolsfile
+
+tldstrending_csv = var/www/static/csv/tldstrendingdata
+tldsfile = faup/src/data/mozilla.tlds
+
+domainstrending_csv = var/www/static/csv/domainstrendingdata
+
+pystemonpath = /opt/pystemon/
+
+sentiment_lexicon_file = sentiment/vader_lexicon.zip/vader_lexicon/vader_lexicon.txt
+
+##### Notifications ######
+[Notifications]
+ail_domain = http://localhost:7000
+sender = sender@example.com
+sender_host = smtp.example.com
+sender_port = 1337
+sender_pw = None
+
+# optional for using with authenticated SMTP over SSL
+# sender_pw = securepassword
+
+##### Flask #####
+[Flask]
+#Proxying requests to the app
+baseUrl = /
+#Number of logs to display in the dashboard
+max_dashboard_logs = 15
+#Maximum number of character to display in the toolip
+max_preview_char = 250
+#Maximum number of character to display in the modal
+max_preview_modal = 800
+#Default number of header to display in trending graphs
+default_display = 10
+#Number of minutes displayed for the number of processed pastes.
+minute_processed_paste = 10
+#Maximum line length authorized to make a diff between duplicates
+DiffMaxLineLength = 10000
+
+#### Modules ####
+[BankAccount]
+max_execution_time = 60
+
+[Categ]
+#Minimum number of match between the paste and the category file
+matchingThreshold=1
+
+[Credential]
+#Minimum length that a credential must have to be considered as such
+minimumLengthThreshold=3
+#Will be pushed as alert if the number of credentials is greater to that number
+criticalNumberToAlert=8
+#Will be considered as false positive if less that X matches from the top password list
+minTopPassList=5
+
+[Curve]
+max_execution_time = 90
+
+[Onion]
+max_execution_time = 180
+
+[Base64]
+path = Base64/
+max_execution_time = 60
+
+[Binary]
+path = Base64/
+max_execution_time = 60
+
+[Hex]
+path = Base64/
+max_execution_time = 60
+
+[Modules_Duplicates]
+#Number of month to look back
+maximum_month_range = 3
+#The value where two pastes are considerate duplicate for ssdeep.
+threshold_duplicate_ssdeep = 50
+#The value where two pastes are considerate duplicate for tlsh.
+threshold_duplicate_tlsh = 52
+#Minimum size of the paste considered
+min_paste_size = 0.3
+
+[Module_ModuleInformation]
+#Threshold to deduce if a module is stuck or not, in seconds.
+threshold_stucked_module=600
+
+[Module_Mixer]
+#Define the configuration of the mixer, possible value: 1, 2 or 3
+operation_mode = 3
+#Define the time that a paste will be considerate duplicate. in seconds (1day = 86400)
+ttl_duplicate = 86400
+default_unnamed_feed_name = unnamed_feeder
+
+[RegexForTermsFrequency]
+max_execution_time = 60
+
+##### Redis #####
+[Redis_Cache]
+host = localhost
+port = 6379
+db = 0
+
+[Redis_Log]
+host = localhost
+port = 6380
+db = 0
+
+[Redis_Log_submit]
+host = localhost
+port = 6380
+db = 1
+
+[Redis_Queues]
+host = localhost
+port = 6381
+db = 0
+
+[Redis_Data_Merging]
+host = localhost
+port = 6379
+db = 1
+
+[Redis_Paste_Name]
+host = localhost
+port = 6379
+db = 2
+
+[Redis_Mixer_Cache]
+host = localhost
+port = 6381
+db = 1
+
+##### ARDB #####
+[ARDB_Curve]
+host = localhost
+port = 6382
+db = 1
+
+[ARDB_Sentiment]
+host = localhost
+port = 6382
+db = 4
+
+[ARDB_TermFreq]
+host = localhost
+port = 6382
+db = 2
+
+[ARDB_TermCred]
+host = localhost
+port = 6382
+db = 5
+
+[ARDB_DB]
+host = localhost
+port = 6382
+db = 0
+
+[ARDB_Trending]
+host = localhost
+port = 6382
+db = 3
+
+[ARDB_Hashs]
+host = localhost
+db = 1
+
+[ARDB_Tags]
+host = localhost
+port = 6382
+db = 6
+
+[ARDB_Metadata]
+host = localhost
+port = 6382
+db = 7
+
+[ARDB_Statistics]
+host = localhost
+port = 6382
+db = 8
+
+[ARDB_Onion]
+host = localhost
+port = 6382
+db = 9
+
+[Url]
+cc_critical = DE
+
+[DomClassifier]
+cc = DE
+cc_tld = r'\.de$'
+dns = 8.8.8.8
+
+[Mail]
+dns = 8.8.8.8
+
+[Web]
+dns = 149.13.33.69
+
+# Indexer configuration
+[Indexer]
+type = whoosh
+path = indexdir
+register = indexdir/all_index.txt
+#size in Mb
+index_max_size = 2000
+
+[ailleakObject]
+maxDuplicateToPushToMISP=10
+
+###############################################################################
+
+# For multiple feed, add them with "," without space
+# e.g.: tcp://127.0.0.1:5556,tcp://127.0.0.1:5557
+[ZMQ_Global]
+#address = tcp://crf.circl.lu:5556
+address = tcp://127.0.0.1:5556,tcp://crf.circl.lu:5556
+channel = 102
+bind = tcp://127.0.0.1:5556
+
+[ZMQ_Url]
+address = tcp://127.0.0.1:5004
+channel = urls
+
+[ZMQ_FetchedOnion]
+address = tcp://127.0.0.1:5005
+channel = FetchedOnion
+
+[RedisPubSub]
+host = localhost
+port = 6381
+db = 0
+
+[Crawler]
+activate_crawler = False
+crawler_depth_limit = 1
+splash_url_onion = http://172.17.0.1
+splash_onion_port = 8050
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -0,0 +1,521 @@
+version: '3'
+services:
+  ardb:
+    entrypoint:
+      - ardb-server
+      - /opt/AIL/configs/6382.conf
+    healthcheck:
+      test: ["CMD", "redis-cli", "-p", "6382", "ping"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./configs:/opt/AIL/configs:ro
+  crawler:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/Crawler.py
+      - onion
+      - "8050"
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./CRAWLED_SCREENSHOT/:/opt/AIL/CRAWLED_SCREENSHOT
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  flask:
+    build: .
+    entrypoint:
+      - /opt/AIL/var/www/Flask_server.py
+    ports:
+      - "7000:7000"
+    image: ail-framework
+    volumes:
+      - ./CRAWLED_SCREENSHOT/:/opt/AIL/CRAWLED_SCREENSHOT
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/var/www
+  log-queue:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/AILENV/bin/log_subscriber
+      - -p
+      - "6380"
+      - -c
+      - Queing
+      - -l
+      - /opt/AIL/logs/
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./configs:/opt/AIL/configs:ro
+  log-script:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/AILENV/bin/log_subscriber
+      - -p
+      - "6380"
+      - -c
+      - Script
+      - -l
+      - /opt/AIL/logs/
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./configs:/opt/AIL/configs:ro
+  pystemon:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/pystemon/pystemon.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./pystemon/archives:/opt/pystemon/archive
+      - ./pystemon/proxies.txt:/opt/pystemon/proxies.txt:ro
+      - ./pystemon/pystemon.yaml:/opt/pystemon/pystemon.yaml:ro
+    working_dir: /opt/pystemon
+  pystemon-feeder:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/feeder/pystemon-feeder.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./pystemon/archives:/opt/pystemon/archive
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  queues:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/launch_queues.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  redis-cache:
+    entrypoint:
+      - redis-server
+      - /opt/AIL/configs/6379.conf
+    healthcheck:
+      test: ["CMD", "redis-cli", "-p", "6379", "ping"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+    image: ail-framework
+    network_mode: service:flask
+    volumes:
+      - ./configs:/opt/AIL/configs:ro
+  redis-log:
+    entrypoint:
+      - redis-server
+      - /opt/AIL/configs/6380.conf
+    healthcheck:
+      test: ["CMD", "redis-cli", "-p", "6380", "ping"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./configs:/opt/AIL/configs:ro
+  redis-mixer-cache:
+    entrypoint:
+      - redis-server
+      - /opt/AIL/configs/6381.conf
+    healthcheck:
+      test: ["CMD", "redis-cli", "-p", "6381", "ping"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+    image: ail-framework
+    network_mode: service:flask
+    volumes:
+      - ./configs:/opt/AIL/configs:ro
+  script-alerthandler:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/alertHandler.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-apikey:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/ApiKey.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-bankaccount:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/BankAccount.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-bitcoin:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/Bitcoin.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-categ:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/Categ.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-credential:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/Credential.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-creditcards:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/CreditCards.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-curve:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/Curve.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-curvemanagetopsets:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/CurveManageTopSets.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-cve:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/Cve.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-decoder:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/Decoder.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./HASHS:/opt/AIL/HASHS
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-domclassifier:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/DomClassifier.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-duplicates:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/Duplicates.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-global:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/Global.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-indexer:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/Indexer.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-keys:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/Keys.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-liblinjection:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/LibInjection.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-lines:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/Lines.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-mail:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/Mail.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-mispthehivefeeder:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/MISP_The_Hive_feeder.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-mixer:
+    depends_on:
+      - redis-mixer-cache
+    entrypoint:
+      - /opt/AIL/bin/Mixer.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-modulestats:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/ModuleStats.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-onion:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/Onion.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-phone:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/Phone.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-regexfortermsfrequency:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/RegexForTermsFrequency.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-release:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/Release.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-sentimentanalysis:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/SentimentAnalysis.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-setfortermsfrequency:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/SetForTermsFrequency.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-sqlinjectiondetection:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/SQLInjectionDetection.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-submitpaste:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/submit_paste.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-tags:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/Tags.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-tokenize:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/Tokenize.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-web:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/Web.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-webstats:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/WebStats.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
--- a/pystemon/proxies.txt
+++ b/pystemon/proxies.txt
@ -0,0 +1 @@
+http://127.0.0.1:8080