From 9eae25bdf595a3fc950118523721d3ed008f7a77 Mon Sep 17 00:00:00 2001 From: Jason Hedden Date: Fri, 22 Feb 2019 12:22:20 -0600 Subject: [PATCH 1/5] Fix current_selected amount and action vars This commit fixes issue #268 ``` File "./bin/ModulesInformationV2.py", line 319, in _setValue action = current_selected_action + " " + str(current_selected_amount) + "x" ``` `current_selected_action` was defined twice. The second occurance should be `current_selected_amount`. This also fixes the generic exception from failing when `current_selected_value` is not defined. --- bin/ModulesInformationV2.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bin/ModulesInformationV2.py b/bin/ModulesInformationV2.py index 30a24f15..cef6301c 100755 --- a/bin/ModulesInformationV2.py +++ b/bin/ModulesInformationV2.py @@ -31,7 +31,7 @@ lastTimeKillCommand = {} current_selected_value = 0 current_selected_queue = "" current_selected_action = "" -current_selected_action = 0 +current_selected_amount = 0 # Map PID to Queue name (For restart and killing) PID_NAME_DICO = {} @@ -480,7 +480,10 @@ class Show_paste(Frame): self.label_list[i]._text = "" except Exception as e: - self.label_list[0]._text = "Error while displaying the paste: " + COMPLETE_PASTE_PATH_PER_PID[current_selected_value] + if current_selected_value in COMPLETE_PASTE_PATH_PER_PID: + self.label_list[0]._text = "Error while displaying the paste: " + COMPLETE_PASTE_PATH_PER_PID[current_selected_value] + else: + self.label_list[0]._text = "Error Generic exception caught" self.label_list[1]._text = str(e) for i in range(2,self.num_label): self.label_list[i]._text = "" From c825ea6b2b6f10af74bd616338872dffc45bbee2 Mon Sep 17 00:00:00 2001 From: Jason Hedden Date: Sun, 10 Mar 2019 22:34:02 -0500 Subject: [PATCH 2/5] Add pybgpranking to install deps The python module DomainClassifier requires pybgpranking which is not available in the community PyPI repository. This commit installs pybgpranking from git using the same version required by DomainClassifier https://github.com/adulau/DomainClassifier Fixes the following installation error when using installing_deps.sh: ``` Could not find a version that satisfies the requirement pybgpranking (from DomainClassifier->-r pip3_packages_requirement.txt (line 50)) (from versions: ) No matching distribution found for pybgpranking (from DomainClassifier->-r pip3_packages_requirement.txt (line 50)) ``` --- installing_deps.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/installing_deps.sh b/installing_deps.sh index 484ca770..579c2a9d 100755 --- a/installing_deps.sh +++ b/installing_deps.sh @@ -95,6 +95,7 @@ popd mkdir -p $AIL_HOME/PASTES pip3 install -U pip +pip3 install 'git+https://github.com/D4-project/BGP-Ranking.git/@7e698f87366e6f99b4d0d11852737db28e3ddc62#egg=pybgpranking&subdirectory=client' pip3 install -U -r pip3_packages_requirement.txt # Pyfaup From 23fb91d3267cacbf3d103b5780f9df9b4b44dc6d Mon Sep 17 00:00:00 2001 From: Jason Hedden Date: Mon, 11 Mar 2019 17:04:02 -0500 Subject: [PATCH 3/5] Add docker-compose support This commit separates the AIL Framework services into individual Docker containers that can be managed with docker-compose. It's intended to ease the installation, development and troubleshooting procedures for all-in-one deployments. No changes to architecture or base code are required in this commit. Changes to existing files: .gitignore * Ignore pystemon archive directory Dockerfile: * Add AIL shell environment variables * Install pystemon and crawler pip requirements into AIL virtual environment Docker-compose notes: * All containers are participating in a service network `network_mode: service:flask`. This allows the containers to share the same IP namespace to accommodate hard-coded localhost entries. * By default persistent data is saved to the following directories in the local AIL framework git directory: PASTES, HASHS, CRAWLED_SCREENSHOTS, pystemon/archives This was tested with docker-ce on Ubuntu 16.04 and MacOS. A typical deployment would look like: ``` git clone https://github.com/CIRCL/AIL-framework.git cd AIL-framework cp bin/packages/config.cfg.docker-compose-sample bin/packages/config.cfg (optionally enable activate_crawler in config.cfg) docker-compose build docker-compose up -d ``` --- .gitignore | 3 + Dockerfile | 10 + bin/packages/config.cfg.docker-compose-sample | 253 +++++++++ docker-compose.yml | 521 ++++++++++++++++++ pystemon/proxies.txt | 1 + 5 files changed, 788 insertions(+) create mode 100644 bin/packages/config.cfg.docker-compose-sample create mode 100644 docker-compose.yml create mode 100644 pystemon/proxies.txt diff --git a/.gitignore b/.gitignore index 2d276111..83c83b29 100644 --- a/.gitignore +++ b/.gitignore @@ -36,6 +36,9 @@ bin/packages/config.cfg.backup configs/keys files +# Pystemon archives +pystemon/archives + # installed files nltk_data/ doc/all_modules.txt diff --git a/Dockerfile b/Dockerfile index 340e5014..533c44c4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,8 +27,18 @@ WORKDIR /opt/AIL # Default to UTF-8 file.encoding ENV LANG C.UTF-8 +ENV AIL_HOME /opt/AIL +ENV AIL_BIN ${AIL_HOME}/bin +ENV AIL_FLASK ${AIL_HOME}/var/www +ENV AIL_REDIS ${AIL_HOME}/redis/src +ENV AIL_ARDB ${AIL_HOME}/ardb/src +ENV AIL_VENV ${AIL_HOME}/AILENV + +ENV PATH ${AIL_VENV}/bin:${AIL_HOME}:${AIL_REDIS}:${AIL_ARDB}:${AIL_BIN}:${AIL_FLASK}:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin RUN ./pystemon/install.sh +RUN pip install -r /opt/pystemon/requirements.txt +RUN pip install -r /opt/AIL/crawler_requirements.txt COPY docker_start.sh /docker_start.sh ENTRYPOINT ["/bin/bash", "docker_start.sh"] diff --git a/bin/packages/config.cfg.docker-compose-sample b/bin/packages/config.cfg.docker-compose-sample new file mode 100644 index 00000000..2f563493 --- /dev/null +++ b/bin/packages/config.cfg.docker-compose-sample @@ -0,0 +1,253 @@ +[Directories] +bloomfilters = Blooms +dicofilters = Dicos +pastes = PASTES +hash = HASHS +crawled = crawled +crawled_screenshot = CRAWLED_SCREENSHOT + +wordtrending_csv = var/www/static/csv/wordstrendingdata +wordsfile = files/wordfile + +protocolstrending_csv = var/www/static/csv/protocolstrendingdata +protocolsfile = files/protocolsfile + +tldstrending_csv = var/www/static/csv/tldstrendingdata +tldsfile = faup/src/data/mozilla.tlds + +domainstrending_csv = var/www/static/csv/domainstrendingdata + +pystemonpath = /opt/pystemon/ + +sentiment_lexicon_file = sentiment/vader_lexicon.zip/vader_lexicon/vader_lexicon.txt + +##### Notifications ###### +[Notifications] +ail_domain = http://localhost:7000 +sender = sender@example.com +sender_host = smtp.example.com +sender_port = 1337 +sender_pw = None + +# optional for using with authenticated SMTP over SSL +# sender_pw = securepassword + +##### Flask ##### +[Flask] +#Proxying requests to the app +baseUrl = / +#Number of logs to display in the dashboard +max_dashboard_logs = 15 +#Maximum number of character to display in the toolip +max_preview_char = 250 +#Maximum number of character to display in the modal +max_preview_modal = 800 +#Default number of header to display in trending graphs +default_display = 10 +#Number of minutes displayed for the number of processed pastes. +minute_processed_paste = 10 +#Maximum line length authorized to make a diff between duplicates +DiffMaxLineLength = 10000 + +#### Modules #### +[BankAccount] +max_execution_time = 60 + +[Categ] +#Minimum number of match between the paste and the category file +matchingThreshold=1 + +[Credential] +#Minimum length that a credential must have to be considered as such +minimumLengthThreshold=3 +#Will be pushed as alert if the number of credentials is greater to that number +criticalNumberToAlert=8 +#Will be considered as false positive if less that X matches from the top password list +minTopPassList=5 + +[Curve] +max_execution_time = 90 + +[Onion] +max_execution_time = 180 + +[Base64] +path = Base64/ +max_execution_time = 60 + +[Binary] +path = Base64/ +max_execution_time = 60 + +[Hex] +path = Base64/ +max_execution_time = 60 + +[Modules_Duplicates] +#Number of month to look back +maximum_month_range = 3 +#The value where two pastes are considerate duplicate for ssdeep. +threshold_duplicate_ssdeep = 50 +#The value where two pastes are considerate duplicate for tlsh. +threshold_duplicate_tlsh = 52 +#Minimum size of the paste considered +min_paste_size = 0.3 + +[Module_ModuleInformation] +#Threshold to deduce if a module is stuck or not, in seconds. +threshold_stucked_module=600 + +[Module_Mixer] +#Define the configuration of the mixer, possible value: 1, 2 or 3 +operation_mode = 3 +#Define the time that a paste will be considerate duplicate. in seconds (1day = 86400) +ttl_duplicate = 86400 +default_unnamed_feed_name = unnamed_feeder + +[RegexForTermsFrequency] +max_execution_time = 60 + +##### Redis ##### +[Redis_Cache] +host = localhost +port = 6379 +db = 0 + +[Redis_Log] +host = localhost +port = 6380 +db = 0 + +[Redis_Log_submit] +host = localhost +port = 6380 +db = 1 + +[Redis_Queues] +host = localhost +port = 6381 +db = 0 + +[Redis_Data_Merging] +host = localhost +port = 6379 +db = 1 + +[Redis_Paste_Name] +host = localhost +port = 6379 +db = 2 + +[Redis_Mixer_Cache] +host = localhost +port = 6381 +db = 1 + +##### ARDB ##### +[ARDB_Curve] +host = localhost +port = 6382 +db = 1 + +[ARDB_Sentiment] +host = localhost +port = 6382 +db = 4 + +[ARDB_TermFreq] +host = localhost +port = 6382 +db = 2 + +[ARDB_TermCred] +host = localhost +port = 6382 +db = 5 + +[ARDB_DB] +host = localhost +port = 6382 +db = 0 + +[ARDB_Trending] +host = localhost +port = 6382 +db = 3 + +[ARDB_Hashs] +host = localhost +db = 1 + +[ARDB_Tags] +host = localhost +port = 6382 +db = 6 + +[ARDB_Metadata] +host = localhost +port = 6382 +db = 7 + +[ARDB_Statistics] +host = localhost +port = 6382 +db = 8 + +[ARDB_Onion] +host = localhost +port = 6382 +db = 9 + +[Url] +cc_critical = DE + +[DomClassifier] +cc = DE +cc_tld = r'\.de$' +dns = 8.8.8.8 + +[Mail] +dns = 8.8.8.8 + +[Web] +dns = 149.13.33.69 + +# Indexer configuration +[Indexer] +type = whoosh +path = indexdir +register = indexdir/all_index.txt +#size in Mb +index_max_size = 2000 + +[ailleakObject] +maxDuplicateToPushToMISP=10 + +############################################################################### + +# For multiple feed, add them with "," without space +# e.g.: tcp://127.0.0.1:5556,tcp://127.0.0.1:5557 +[ZMQ_Global] +#address = tcp://crf.circl.lu:5556 +address = tcp://127.0.0.1:5556,tcp://crf.circl.lu:5556 +channel = 102 +bind = tcp://127.0.0.1:5556 + +[ZMQ_Url] +address = tcp://127.0.0.1:5004 +channel = urls + +[ZMQ_FetchedOnion] +address = tcp://127.0.0.1:5005 +channel = FetchedOnion + +[RedisPubSub] +host = localhost +port = 6381 +db = 0 + +[Crawler] +activate_crawler = False +crawler_depth_limit = 1 +splash_url_onion = http://172.17.0.1 +splash_onion_port = 8050 diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..98526a76 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,521 @@ +version: '3' +services: + ardb: + entrypoint: + - ardb-server + - /opt/AIL/configs/6382.conf + healthcheck: + test: ["CMD", "redis-cli", "-p", "6382", "ping"] + interval: 30s + timeout: 10s + retries: 5 + network_mode: service:flask + image: ail-framework + volumes: + - ./configs:/opt/AIL/configs:ro + crawler: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/Crawler.py + - onion + - "8050" + network_mode: service:flask + image: ail-framework + volumes: + - ./CRAWLED_SCREENSHOT/:/opt/AIL/CRAWLED_SCREENSHOT + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + flask: + build: . + entrypoint: + - /opt/AIL/var/www/Flask_server.py + ports: + - "7000:7000" + image: ail-framework + volumes: + - ./CRAWLED_SCREENSHOT/:/opt/AIL/CRAWLED_SCREENSHOT + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/var/www + log-queue: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/AILENV/bin/log_subscriber + - -p + - "6380" + - -c + - Queing + - -l + - /opt/AIL/logs/ + network_mode: service:flask + image: ail-framework + volumes: + - ./configs:/opt/AIL/configs:ro + log-script: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/AILENV/bin/log_subscriber + - -p + - "6380" + - -c + - Script + - -l + - /opt/AIL/logs/ + network_mode: service:flask + image: ail-framework + volumes: + - ./configs:/opt/AIL/configs:ro + pystemon: + depends_on: + - redis-log + entrypoint: + - /opt/pystemon/pystemon.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./pystemon/archives:/opt/pystemon/archive + - ./pystemon/proxies.txt:/opt/pystemon/proxies.txt:ro + - ./pystemon/pystemon.yaml:/opt/pystemon/pystemon.yaml:ro + working_dir: /opt/pystemon + pystemon-feeder: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/feeder/pystemon-feeder.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./pystemon/archives:/opt/pystemon/archive + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + queues: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/launch_queues.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + redis-cache: + entrypoint: + - redis-server + - /opt/AIL/configs/6379.conf + healthcheck: + test: ["CMD", "redis-cli", "-p", "6379", "ping"] + interval: 30s + timeout: 10s + retries: 5 + image: ail-framework + network_mode: service:flask + volumes: + - ./configs:/opt/AIL/configs:ro + redis-log: + entrypoint: + - redis-server + - /opt/AIL/configs/6380.conf + healthcheck: + test: ["CMD", "redis-cli", "-p", "6380", "ping"] + interval: 30s + timeout: 10s + retries: 5 + network_mode: service:flask + image: ail-framework + volumes: + - ./configs:/opt/AIL/configs:ro + redis-mixer-cache: + entrypoint: + - redis-server + - /opt/AIL/configs/6381.conf + healthcheck: + test: ["CMD", "redis-cli", "-p", "6381", "ping"] + interval: 30s + timeout: 10s + retries: 5 + image: ail-framework + network_mode: service:flask + volumes: + - ./configs:/opt/AIL/configs:ro + script-alerthandler: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/alertHandler.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-apikey: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/ApiKey.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-bankaccount: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/BankAccount.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-bitcoin: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/Bitcoin.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-categ: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/Categ.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-credential: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/Credential.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-creditcards: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/CreditCards.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-curve: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/Curve.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-curvemanagetopsets: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/CurveManageTopSets.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-cve: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/Cve.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-decoder: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/Decoder.py + network_mode: service:flask + image: ail-framework + volumes: + - ./HASHS:/opt/AIL/HASHS + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-domclassifier: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/DomClassifier.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-duplicates: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/Duplicates.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-global: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/Global.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-indexer: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/Indexer.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-keys: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/Keys.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-liblinjection: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/LibInjection.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-lines: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/Lines.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-mail: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/Mail.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-mispthehivefeeder: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/MISP_The_Hive_feeder.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-mixer: + depends_on: + - redis-mixer-cache + entrypoint: + - /opt/AIL/bin/Mixer.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-modulestats: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/ModuleStats.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-onion: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/Onion.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-phone: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/Phone.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-regexfortermsfrequency: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/RegexForTermsFrequency.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-release: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/Release.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-sentimentanalysis: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/SentimentAnalysis.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-setfortermsfrequency: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/SetForTermsFrequency.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-sqlinjectiondetection: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/SQLInjectionDetection.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-submitpaste: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/submit_paste.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-tags: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/Tags.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-tokenize: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/Tokenize.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-web: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/Web.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-webstats: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/WebStats.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin diff --git a/pystemon/proxies.txt b/pystemon/proxies.txt new file mode 100644 index 00000000..a4ae3812 --- /dev/null +++ b/pystemon/proxies.txt @@ -0,0 +1 @@ +http://127.0.0.1:8080 From 9279f560483ab376d89c4d47647f0c7a913f4440 Mon Sep 17 00:00:00 2001 From: Jason Hedden Date: Wed, 13 Mar 2019 17:05:54 -0500 Subject: [PATCH 4/5] Add indexdir volume to docker-compose Make the indexdir data persistent and shared with Flask for searching pastes. --- docker-compose.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 98526a76..fc7f2a91 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -37,6 +37,7 @@ services: volumes: - ./CRAWLED_SCREENSHOT/:/opt/AIL/CRAWLED_SCREENSHOT - ./PASTES/:/opt/AIL/PASTES + - ./indexdir:/opt/AIL/indexdir - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro working_dir: /opt/AIL/var/www log-queue: @@ -308,6 +309,7 @@ services: image: ail-framework volumes: - ./PASTES/:/opt/AIL/PASTES + - ./indexdir:/opt/AIL/indexdir - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro working_dir: /opt/AIL/bin script-keys: @@ -321,7 +323,7 @@ services: - ./PASTES/:/opt/AIL/PASTES - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro working_dir: /opt/AIL/bin - script-liblinjection: + script-libinjection: depends_on: - redis-log entrypoint: From 870fa1981caf6df3441980b1fa3091dd8b70c232 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 2 Apr 2019 09:50:52 +0200 Subject: [PATCH 5/5] fix: [update_thirdparty] detect and use virtualenv --- var/www/update_thirdparty.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/var/www/update_thirdparty.sh b/var/www/update_thirdparty.sh index 258fa7ca..317de4e4 100755 --- a/var/www/update_thirdparty.sh +++ b/var/www/update_thirdparty.sh @@ -83,8 +83,12 @@ pushd static/image wget https://www.circl.lu/assets/images/logos/AIL.png -O AIL.png popd -#active virtualenv -source ./../../AILENV/bin/activate +if ! [[ -n "$AIL_HOME" ]] +then + #active virtualenv + source ./../../AILENV/bin/activate +fi + #Update MISP Taxonomies and Galaxies python3 -m pip install git+https://github.com/MISP/PyTaxonomies --upgrade python3 -m pip install git+https://github.com/MISP/PyMISPGalaxies --upgrade