Add initial Travis file

.travis.yml

@@ -0,0 +1,83 @@
+language: python
+
+python:
+    - "2.7"
+
+sudo: required
+
+dist: trusty
+
+group: edge
+
+addons:
+    apt:
+        packages:
+            # General dependencies
+            - python-pip
+            - python-virtualenv
+            - python-dev
+            - g++
+            - python-tk
+            - unzip
+            - libsnappy-dev
+            # Needed for bloomfilters
+            - libssl-dev
+            - python-numpy
+            - libfreetype6-dev
+            # Leveldb
+            - libgmp-dev
+            - libev-dev
+
+env:
+    - AIL_HOME=$TRAVIS_BUILD_DIR AIL_BIN=$TRAVIS_BUILD_DIR/bin/ \
+      AIL_FLASK=$TRAVIS_BUILD_DIR/var/www/ AIL_REDIS=$TRAVIS_BUILD_DIR/redis/src/ \
+      AIL_LEVELDB=$TRAVIS_BUILD_DIR/redis-leveldb/ PATH=$AIL_HOME:$AIL_REDIS:$AIL_LEVELDB:$PATH
+
+
+
+install:
+    - pip install -U pip
+    # DNS
+    - sudo apt-get install libadns1 libadns1-dev screen
+    # required for mathplotlib
+    - test ! -L /usr/include/ft2build.h && sudo ln -s freetype2/ft2build.h /usr/include/
+    - pip install distribute
+    # Redis
+    - test ! -d redis/ && git clone https://github.com/antirez/redis.git
+    - pushd redis
+    - git checkout 3.0
+    - make
+    - popd
+    # Redis leveldb
+    - test ! -d redis-leveldb/ && git clone https://github.com/KDr2/redis-leveldb.git
+    - pushd redis-leveldb/
+    - git submodule init
+    - git submodule update
+    - make
+    - popd
+    # Set config
+    - cp bin/packages/config.cfg.sample bin/packages/config.cfg
+    - mkdir -p $AIL_HOME/{PASTES,Blooms,dumps}
+    - mkdir -p $AIL_HOME/LEVEL_DB_DATA/{2016,2015,2014,2013}
+    - pip install -r pip_packages_requirement.txt
+    - python -m textblob.download_corpora
+    - pushd var/www/
+    - ./update_thirdparty.sh
+    - popd
+
+
+script:
+    - pushd bin
+    - ./launch_redis.sh
+    - ./launch_lvldb.sh
+    - ./launch_logs.sh
+    - ./launch_queues.sh
+    - ./launch_scripts.sh
+    - sleep 120
+    - ./Shutdown.py
+    - popd
+
+notifications:
+    email:
+        on_success: change
+        on_failure: change

bin/launch_logs.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -e
+set -x
+
+screen -dmS "Logging"
+sleep 0.1
+echo -e $GREEN"\t* Launching logging process"$DEFAULT
+screen -S "Logging" -X screen -t "LogQueue" bash -c 'log_subscriber -p 6380 -c Queuing -l ../logs/; read x'
+sleep 0.1
+screen -S "Logging" -X screen -t "LogScript" bash -c 'log_subscriber -p 6380 -c Script -l ../logs/; read x'
+

bin/launch_lvldb.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+set -e
+set -x
+
+lvdbhost='127.0.0.1'
+lvdbdir="${AIL_HOME}/LEVEL_DB_DATA/"
+db1_y='2013'
+db2_y='2014'
+db2_y='2015'
+db2_y='2016'
+nb_db=13
+
+screen -dmS "LevelDB"
+sleep 0.1
+echo -e $GREEN"\t* Launching Levels DB servers"$DEFAULT
+#Add lines here with appropriates options.
+screen -S "LevelDB" -X screen -t "2013" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2013/ -P '$db1_y' -M '$nb_db'; read x'
+sleep 0.1
+screen -S "LevelDB" -X screen -t "2014" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2014/ -P '$db2_y' -M '$nb_db'; read x'
+sleep 0.1
+screen -S "LevelDB" -X screen -t "2015" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2015/ -P '$db2_y' -M '$nb_db'; read x'
+sleep 0.1
+screen -S "LevelDB" -X screen -t "2016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2016/ -P '$db2_y' -M '$nb_db'; read x'
+

bin/launch_queues.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -e
+set -x
+
+screen -dmS "Queue"
+sleep 0.1
+
+echo -e $GREEN"\t* Launching all the queues"$DEFAULT
+screen -S "Queue" -X screen -t "Queues" bash -c './launch_queues.py; read x'
+

bin/launch_redis.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+set -e
+set -x
+
+conf_dir="${AIL_HOME}/configs/"
+
+screen -dmS "Redis"
+sleep 0.1
+echo -e $GREEN"\t* Launching Redis servers"$DEFAULT
+screen -S "Redis" -X screen -t "6379" bash -c 'redis-server '$conf_dir'6379.conf ; read x'
+sleep 0.1
+screen -S "Redis" -X screen -t "6380" bash -c 'redis-server '$conf_dir'6380.conf ; read x'
+sleep 0.1
+screen -S "Redis" -X screen -t "6381" bash -c 'redis-server '$conf_dir'6381.conf ; read x'
+

bin/launch_scripts.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+set -e
+set -x
+
+screen -dmS "Script"
+sleep 0.1
+
+echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT
+
+screen -S "Script" -X screen -t "Global" bash -c './Global.py; read x'
+sleep 0.1
+screen -S "Script" -X screen -t "Duplicate" bash -c './Duplicate.py; read x'
+sleep 0.1
+screen -S "Script" -X screen -t "Attribute" bash -c './Attribute.py; read x'
+sleep 0.1
+screen -S "Script" -X screen -t "Line" bash -c './Line.py; read x'
+sleep 0.1
+screen -S "Script" -X screen -t "DomainClassifier" bash -c './DomClassifier.py; read x'
+sleep 0.1
+screen -S "Script" -X screen -t "Categ" bash -c './Categ.py; read x'
+sleep 0.1
+screen -S "Script" -X screen -t "Tokenize" bash -c './Tokenize.py; read x'
+sleep 0.1
+screen -S "Script" -X screen -t "CreditCard" bash -c './CreditCard.py; read x'
+sleep 0.1
+screen -S "Script" -X screen -t "Onion" bash -c './Onion.py; read x'
+sleep 0.1
+screen -S "Script" -X screen -t "Mail" bash -c './Mail.py; read x'
+sleep 0.1
+screen -S "Script" -X screen -t "Url" bash -c './Url.py; read x'
+sleep 0.1
+screen -S "Script" -X screen -t "Curve" bash -c './Curve.py; read x'
+sleep 0.1
+screen -S "Script" -X screen -t "Indexer" bash -c './Indexer.py; read x'
+