chg: [python] Added necessary checks in LAUNCH.sh to be able to launch the script without doing bin/activate

chg: [doc] Added correct pystemon path
fix: [zmq] Small ZMQ test fix.
fix: [doc] Moved the print statement in feeder so it is less verbose but still useful

HOWTO.md

@@ -27,7 +27,7 @@ Feed data to AIL:
 
 4. Edit your configuration file ```bin/packages/config.cfg``` and modify the pystemonpath path accordingly
 
-5. Launch pystemon-feeder ``` ./pystemon-feeder.py ```
+5. Launch pystemon-feeder ``` ./bin/feeder/pystemon-feeder.py ```
 
 
 How to create a new module

README.md

@@ -70,8 +70,6 @@ Type these command lines for a fully automated installation and start AIL framew
 git clone https://github.com/CIRCL/AIL-framework.git
 cd AIL-framework
 ./installing_deps.sh
-cd var/www/
-./update_thirdparty.sh
 cd ~/AIL-framework/
 . ./AILENV/bin/activate
 cd bin/

bin/LAUNCH.sh

@@ -9,11 +9,26 @@ WHITE="\\033[0;02m"
 YELLOW="\\033[1;33m"
 CYAN="\\033[1;36m"
 
-[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
-[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
-[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
-[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
-[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
+# Getting CWD where bash script resides
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+AIL_HOME="${DIR}"
+
+cd ${AIL_HOME}
+
+if [ -e "${DIR}/AILENV/bin/python" ]; then
+    echo "AIL-framework virtualenv seems to exist, good"
+    ENV_PY="${DIR}/AILENV/bin/python"
+else
+    echo "Please make sure you have a AIL-framework environment, au revoir"
+    exit 1
+fi
+
+[ ! -f "`which redis-server`" ] && echo "'redis-server' is not installed/not on PATH. Please fix and run again." && exit 1
+
+export AIL_BIN=${AIL_HOME}/bin/
+export AIL_FLASK=${AIL_HOME}/var/www/
+export AIL_REDIS=${AIL_HOME}/redis/src/
+export AIL_ARDB=${AIL_HOME}/ardb/src/
 
 export PATH=$AIL_HOME:$PATH
 export PATH=$AIL_REDIS:$PATH
@@ -101,16 +116,16 @@ function launching_queues {
     sleep 0.1
 
     echo -e $GREEN"\t* Launching all the queues"$DEFAULT
-    screen -S "Queue_AIL" -X screen -t "Queues" bash -c 'cd '${AIL_BIN}'; python3 launch_queues.py; read x'
+    screen -S "Queue_AIL" -X screen -t "Queues" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} launch_queues.py; read x'
 }
 
 function checking_configuration {
     bin_dir=${AIL_HOME}/bin
     echo -e "\t* Checking configuration"
     if [ "$1" == "automatic" ]; then
-        bash -c "python3 $bin_dir/Update-conf.py True"
+        bash -c "${ENV_PY} $bin_dir/Update-conf.py True"
     else
-        bash -c "python3 $bin_dir/Update-conf.py False"
+        bash -c "${ENV_PY} $bin_dir/Update-conf.py False"
     fi
 
     exitStatus=$?
@@ -128,75 +143,75 @@ function launching_scripts {
     sleep 0.1
     echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT
 
-    screen -S "Script_AIL" -X screen -t "ModuleInformation" bash -c 'cd '${AIL_BIN}'; ./ModulesInformationV2.py -k 0 -c 1; read x'
+    screen -S "Script_AIL" -X screen -t "ModuleInformation" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./ModulesInformationV2.py -k 0 -c 1; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "Mixer" bash -c 'cd '${AIL_BIN}'; ./Mixer.py; read x'
+    screen -S "Script_AIL" -X screen -t "Mixer" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Mixer.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "Global" bash -c 'cd '${AIL_BIN}'; ./Global.py; read x'
+    screen -S "Script_AIL" -X screen -t "Global" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Global.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "Duplicates" bash -c 'cd '${AIL_BIN}'; ./Duplicates.py; read x'
+    screen -S "Script_AIL" -X screen -t "Duplicates" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Duplicates.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "Lines" bash -c 'cd '${AIL_BIN}'; ./Lines.py; read x'
+    screen -S "Script_AIL" -X screen -t "Lines" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Lines.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "DomClassifier" bash -c 'cd '${AIL_BIN}'; ./DomClassifier.py; read x'
+    screen -S "Script_AIL" -X screen -t "DomClassifier" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./DomClassifier.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "Categ" bash -c 'cd '${AIL_BIN}'; ./Categ.py; read x'
+    screen -S "Script_AIL" -X screen -t "Categ" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Categ.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "Tokenize" bash -c 'cd '${AIL_BIN}'; ./Tokenize.py; read x'
+    screen -S "Script_AIL" -X screen -t "Tokenize" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Tokenize.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "CreditCards" bash -c 'cd '${AIL_BIN}'; ./CreditCards.py; read x'
+    screen -S "Script_AIL" -X screen -t "CreditCards" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./CreditCards.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "BankAccount" bash -c 'cd '${AIL_BIN}'; ./BankAccount.py; read x'
+    screen -S "Script_AIL" -X screen -t "BankAccount" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./BankAccount.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "Onion" bash -c 'cd '${AIL_BIN}'; ./Onion.py; read x'
+    screen -S "Script_AIL" -X screen -t "Onion" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Onion.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "Mail" bash -c 'cd '${AIL_BIN}'; ./Mail.py; read x'
+    screen -S "Script_AIL" -X screen -t "Mail" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Mail.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "ApiKey" bash -c 'cd '${AIL_BIN}'; ./ApiKey.py; read x'
+    screen -S "Script_AIL" -X screen -t "ApiKey" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./ApiKey.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "Web" bash -c 'cd '${AIL_BIN}'; ./Web.py; read x'
+    screen -S "Script_AIL" -X screen -t "Web" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Web.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "Credential" bash -c 'cd '${AIL_BIN}'; ./Credential.py; read x'
+    screen -S "Script_AIL" -X screen -t "Credential" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Credential.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "Curve" bash -c 'cd '${AIL_BIN}'; ./Curve.py; read x'
+    screen -S "Script_AIL" -X screen -t "Curve" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Curve.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "CurveManageTopSets" bash -c 'cd '${AIL_BIN}'; ./CurveManageTopSets.py; read x'
+    screen -S "Script_AIL" -X screen -t "CurveManageTopSets" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./CurveManageTopSets.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "RegexForTermsFrequency" bash -c 'cd '${AIL_BIN}'; ./RegexForTermsFrequency.py; read x'
+    screen -S "Script_AIL" -X screen -t "RegexForTermsFrequency" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./RegexForTermsFrequency.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "SetForTermsFrequency" bash -c 'cd '${AIL_BIN}'; ./SetForTermsFrequency.py; read x'
+    screen -S "Script_AIL" -X screen -t "SetForTermsFrequency" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./SetForTermsFrequency.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "Indexer" bash -c 'cd '${AIL_BIN}'; ./Indexer.py; read x'
+    screen -S "Script_AIL" -X screen -t "Indexer" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Indexer.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "Keys" bash -c 'cd '${AIL_BIN}'; ./Keys.py; read x'
+    screen -S "Script_AIL" -X screen -t "Keys" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Keys.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "Decoder" bash -c 'cd '${AIL_BIN}'; ./Decoder.py; read x'
+    screen -S "Script_AIL" -X screen -t "Decoder" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Decoder.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "Bitcoin" bash -c 'cd '${AIL_BIN}'; ./Bitcoin.py; read x'
+    screen -S "Script_AIL" -X screen -t "Bitcoin" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Bitcoin.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "Phone" bash -c 'cd '${AIL_BIN}'; ./Phone.py; read x'
+    screen -S "Script_AIL" -X screen -t "Phone" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Phone.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "Release" bash -c 'cd '${AIL_BIN}'; ./Release.py; read x'
+    screen -S "Script_AIL" -X screen -t "Release" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Release.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "Cve" bash -c 'cd '${AIL_BIN}'; ./Cve.py; read x'
+    screen -S "Script_AIL" -X screen -t "Cve" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Cve.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "WebStats" bash -c 'cd '${AIL_BIN}'; ./WebStats.py; read x'
+    screen -S "Script_AIL" -X screen -t "WebStats" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./WebStats.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "ModuleStats" bash -c 'cd '${AIL_BIN}'; ./ModuleStats.py; read x'
+    screen -S "Script_AIL" -X screen -t "ModuleStats" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./ModuleStats.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "SQLInjectionDetection" bash -c 'cd '${AIL_BIN}'; ./SQLInjectionDetection.py; read x'
+    screen -S "Script_AIL" -X screen -t "SQLInjectionDetection" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./SQLInjectionDetection.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "LibInjection" bash -c 'cd '${AIL_BIN}'; ./LibInjection.py; read x'
+    screen -S "Script_AIL" -X screen -t "LibInjection" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./LibInjection.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "alertHandler" bash -c 'cd '${AIL_BIN}'; ./alertHandler.py; read x'
+    screen -S "Script_AIL" -X screen -t "alertHandler" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./alertHandler.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "MISPtheHIVEfeeder" bash -c 'cd '${AIL_BIN}'; ./MISP_The_Hive_feeder.py; read x'
+    screen -S "Script_AIL" -X screen -t "MISPtheHIVEfeeder" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./MISP_The_Hive_feeder.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "Tags" bash -c 'cd '${AIL_BIN}'; ./Tags.py; read x'
+    screen -S "Script_AIL" -X screen -t "Tags" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Tags.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "SentimentAnalysis" bash -c 'cd '${AIL_BIN}'; ./SentimentAnalysis.py; read x'
+    screen -S "Script_AIL" -X screen -t "SentimentAnalysis" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./SentimentAnalysis.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "SubmitPaste" bash -c 'cd '${AIL_BIN}'; ./submit_paste.py; read x'
+    screen -S "Script_AIL" -X screen -t "SubmitPaste" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./submit_paste.py; read x'
 
 }
 
@@ -219,7 +234,7 @@ function launching_crawler {
         sleep 0.1
 
         for ((i=first_port;i<=last_port;i++)); do
-            screen -S "Crawler_AIL" -X screen -t "onion_crawler:$i" bash -c 'cd '${AIL_BIN}'; ./Crawler.py onion '$i'; read x'
+            screen -S "Crawler_AIL" -X screen -t "onion_crawler:$i" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Crawler.py onion '$i'; read x'
             sleep 0.1
         done
 
@@ -342,7 +357,7 @@ function launch_flask {
         screen -dmS "Flask_AIL"
         sleep 0.1
         echo -e $GREEN"\t* Launching Flask server"$DEFAULT
-        screen -S "Flask_AIL" -X screen -t "Flask_server" bash -c "cd $flask_dir; ls; ./Flask_server.py; read x"
+        screen -S "Flask_AIL" -X screen -t "Flask_server" bash -c "cd $flask_dir; ls; ${ENV_PY} ./Flask_server.py; read x"
     else
         echo -e $RED"\t* A Flask screen is already launched"$DEFAULT
     fi
@@ -353,9 +368,9 @@ function launch_feeder {
         screen -dmS "Feeder_Pystemon"
         sleep 0.1
         echo -e $GREEN"\t* Launching Pystemon feeder"$DEFAULT
-        screen -S "Feeder_Pystemon" -X screen -t "Pystemon_feeder" bash -c 'cd '${AIL_BIN}'; ./feeder/pystemon-feeder.py; read x'
+        screen -S "Feeder_Pystemon" -X screen -t "Pystemon_feeder" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./feeder/pystemon-feeder.py; read x'
         sleep 0.1
-        screen -S "Feeder_Pystemon" -X screen -t "Pystemon" bash -c 'cd '${AIL_HOME}/../pystemon'; python2 pystemon.py; read x'
+        screen -S "Feeder_Pystemon" -X screen -t "Pystemon" bash -c 'cd '${AIL_HOME}/../pystemon'; ${ENV_PY} ./pystemon.py; read x'
     else
         echo -e $RED"\t* A Feeder screen is already launched"$DEFAULT
     fi

bin/feeder/pystemon-feeder.py

@@ -61,10 +61,10 @@ topic = '102'
 while True:
     time.sleep(base_sleeptime + sleep_inc)
     paste = r.lpop("pastes")
-    print(paste)
     if paste is None:
         continue
     try:
+        print(paste)
         with open(pystemonpath+paste, 'rb') as f: #.read()
             messagedata = f.read()
         path_to_send = pastes_directory+paste

bin/feeder/test-zmq.py

@@ -20,7 +20,7 @@ socket.connect ("tcp://crf.circl.lu:%s" % port)
 # 102 Full pastes in raw base64(gz)
 
 topicfilter = "102"
-socket.setsockopt(zmq.SUBSCRIBE, topicfilter)
+socket.setsockopt_string(zmq.SUBSCRIBE, topicfilter)
 
 while True:
     message = socket.recv()