From e7c66d1aac93564c3573544d00a10cb6dfd03968 Mon Sep 17 00:00:00 2001 From: = Date: Wed, 23 Aug 2017 12:28:12 +0200 Subject: [PATCH] Updated individual launch scripts --- bin/launch_logs.sh | 4 ++ bin/launch_lvldb.sh | 26 ++++++---- bin/launch_queues.sh | 4 ++ bin/launch_redis.sh | 4 ++ bin/launch_scripts.sh | 118 ++++++++++++++++++++++++------------------ 5 files changed, 96 insertions(+), 60 deletions(-) diff --git a/bin/launch_logs.sh b/bin/launch_logs.sh index 107a2450..ed08ea48 100755 --- a/bin/launch_logs.sh +++ b/bin/launch_logs.sh @@ -3,6 +3,10 @@ set -e set -x +[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_LEVELDB" ] && echo "Needs the env var AIL_LEVELDB. Run the script from the virtual environment." && exit 1; + screen -dmS "Logging" sleep 0.1 echo -e $GREEN"\t* Launching logging process"$DEFAULT diff --git a/bin/launch_lvldb.sh b/bin/launch_lvldb.sh index ef4dcb2c..a534e7ae 100755 --- a/bin/launch_lvldb.sh +++ b/bin/launch_lvldb.sh @@ -3,21 +3,27 @@ set -e set -x +[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_LEVELDB" ] && echo "Needs the env var AIL_LEVELDB. Run the script from the virtual environment." && exit 1; + lvdbhost='127.0.0.1' lvdbdir="${AIL_HOME}/LEVEL_DB_DATA/" -db1_y='2013' -db2_y='2014' -db3_y='2016' -db4_y='3016' nb_db=13 +db_y=`date +%Y` +#Verify that a dir with the correct year exists, create it otherwise +if [ ! -d "$lvdbdir$db_y" ]; then + mkdir -p "$db_y" +fi + screen -dmS "LevelDB" sleep 0.1 echo -e $GREEN"\t* Launching Levels DB servers"$DEFAULT -sleep 0.1 -screen -S "LevelDB" -X screen -t "2016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2016/ -P '$db3_y' -M '$nb_db'; read x' - -# For Curve -sleep 0.1 -screen -S "LevelDB" -X screen -t "3016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'3016/ -P '$db4_y' -M '$nb_db'; read x' +#Launch a DB for each dir +for pathDir in $lvdbdir*/ ; do + yDir=$(basename "$pathDir") + sleep 0.1 + screen -S "LevelDB" -X screen -t "$yDir" bash -c 'redis-leveldb -H '$lvdbhost' -D '$pathDir'/ -P '$yDir' -M '$nb_db'; read x' +done diff --git a/bin/launch_queues.sh b/bin/launch_queues.sh index 8c2699ea..2ddd2b7d 100755 --- a/bin/launch_queues.sh +++ b/bin/launch_queues.sh @@ -3,6 +3,10 @@ set -e set -x +[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_LEVELDB" ] && echo "Needs the env var AIL_LEVELDB. Run the script from the virtual environment." && exit 1; + screen -dmS "Queue" sleep 0.1 diff --git a/bin/launch_redis.sh b/bin/launch_redis.sh index c7af27c8..69910927 100755 --- a/bin/launch_redis.sh +++ b/bin/launch_redis.sh @@ -3,6 +3,10 @@ set -e set -x +[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_LEVELDB" ] && echo "Needs the env var AIL_LEVELDB. Run the script from the virtual environment." && exit 1; + conf_dir="${AIL_HOME}/configs/" screen -dmS "Redis" diff --git a/bin/launch_scripts.sh b/bin/launch_scripts.sh index 694f0138..e593b11e 100755 --- a/bin/launch_scripts.sh +++ b/bin/launch_scripts.sh @@ -3,57 +3,75 @@ set -e set -x +[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_LEVELDB" ] && echo "Needs the env var AIL_LEVELDB. Run the script from the virtual environment." && exit 1; + +echo -e "\t* Checking configuration" +bash -c "./Update-conf.py" +exitStatus=$? +if [ $exitStatus -ge 1 ]; then + echo -e $RED"\t* Configuration not up-to-date"$DEFAULT + exit +fi +echo -e $GREEN"\t* Configuration up-to-date"$DEFAULT + screen -dmS "Script" sleep 0.1 - echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT - screen -S "Script" -X screen -t "ModuleInformation" bash -c './ModuleInformation.py -k 0 -c 1; read x' - sleep 0.1 - screen -S "Script" -X screen -t "Global" bash -c './Global.py; read x' - sleep 0.1 - screen -S "Script" -X screen -t "Duplicates" bash -c './Duplicates.py; read x' - sleep 0.1 - screen -S "Script" -X screen -t "Attributes" bash -c './Attributes.py; read x' - sleep 0.1 - screen -S "Script" -X screen -t "Lines" bash -c './Lines.py; read x' - sleep 0.1 - screen -S "Script" -X screen -t "DomClassifier" bash -c './DomClassifier.py; read x' - sleep 0.1 - screen -S "Script" -X screen -t "Categ" bash -c './Categ.py; read x' - sleep 0.1 - screen -S "Script" -X screen -t "Tokenize" bash -c './Tokenize.py; read x' - sleep 0.1 - screen -S "Script" -X screen -t "CreditCards" bash -c './CreditCards.py; read x' - sleep 0.1 - screen -S "Script" -X screen -t "Onion" bash -c './Onion.py; read x' - sleep 0.1 - screen -S "Script" -X screen -t "Mail" bash -c './Mail.py; read x' - sleep 0.1 - screen -S "Script" -X screen -t "Web" bash -c './Web.py; read x' - sleep 0.1 - screen -S "Script" -X screen -t "Credential" bash -c './Credential.py; read x' - sleep 0.1 - screen -S "Script" -X screen -t "Curve" bash -c './Curve.py; read x' - sleep 0.1 - screen -S "Script" -X screen -t "CurveManageTopSets" bash -c './CurveManageTopSets.py; read x' - sleep 0.1 - screen -S "Script" -X screen -t "Indexer" bash -c './Indexer.py; read x' - sleep 0.1 - screen -S "Script" -X screen -t "Keys" bash -c './Keys.py; read x' - sleep 0.1 - screen -S "Script" -X screen -t "Phone" bash -c './Phone.py; read x' - sleep 0.1 - screen -S "Script" -X screen -t "Release" bash -c './Release.py; read x' - sleep 0.1 - screen -S "Script" -X screen -t "Cve" bash -c './Cve.py; read x' - sleep 0.1 - screen -S "Script" -X screen -t "WebStats" bash -c './WebStats.py; read x' - sleep 0.1 - screen -S "Script" -X screen -t "ModuleStats" bash -c './ModuleStats.py; read x' - sleep 0.1 - screen -S "Script" -X screen -t "SQLInjectionDetection" bash -c './SQLInjectionDetection.py; read x' - sleep 0.1 - screen -S "Script" -X screen -t "Browse_warning_paste" bash -c './Browse_warning_paste.py; read x' - sleep 0.1 - screen -S "Script" -X screen -t "SentimentAnalysis" bash -c './SentimentAnalysis.py; read x' +screen -S "Script" -X screen -t "ModuleInformation" bash -c './ModulesInformationV2.py -k 0 -c 1; read x' +sleep 0.1 +screen -S "Script" -X screen -t "Mixer" bash -c './Mixer.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "Global" bash -c './Global.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "Duplicates" bash -c './Duplicates.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "Attributes" bash -c './Attributes.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "Lines" bash -c './Lines.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "DomClassifier" bash -c './DomClassifier.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "Categ" bash -c './Categ.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "Tokenize" bash -c './Tokenize.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "CreditCards" bash -c './CreditCards.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "Onion" bash -c './Onion.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "Mail" bash -c './Mail.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "Web" bash -c './Web.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "Credential" bash -c './Credential.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "Curve" bash -c './Curve.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "CurveManageTopSets" bash -c './CurveManageTopSets.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "RegexForTermsFrequency" bash -c './RegexForTermsFrequency.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "SetForTermsFrequency" bash -c './SetForTermsFrequency.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "Indexer" bash -c './Indexer.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "Keys" bash -c './Keys.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "Phone" bash -c './Phone.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "Release" bash -c './Release.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "Cve" bash -c './Cve.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "WebStats" bash -c './WebStats.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "ModuleStats" bash -c './ModuleStats.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "SQLInjectionDetection" bash -c './SQLInjectionDetection.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "BrowseWarningPaste" bash -c './BrowseWarningPaste.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "SentimentAnalysis" bash -c './SentimentAnalysis.py; read x'