2014-08-06 11:43:40 +02:00
|
|
|
#!/bin/bash
|
|
|
|
|
|
|
|
GREEN="\\033[1;32m"
|
|
|
|
DEFAULT="\\033[0;39m"
|
|
|
|
RED="\\033[1;31m"
|
|
|
|
ROSE="\\033[1;35m"
|
|
|
|
BLUE="\\033[1;34m"
|
|
|
|
WHITE="\\033[0;02m"
|
|
|
|
YELLOW="\\033[1;33m"
|
|
|
|
CYAN="\\033[1;36m"
|
|
|
|
|
2018-11-22 06:57:24 +01:00
|
|
|
# Getting CWD where bash script resides
|
2018-11-22 07:08:59 +01:00
|
|
|
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd |sed 's/bin//' )"
|
2018-11-22 08:16:31 +01:00
|
|
|
export AIL_HOME="${DIR}"
|
2018-11-22 06:57:24 +01:00
|
|
|
|
|
|
|
cd ${AIL_HOME}
|
|
|
|
|
|
|
|
if [ -e "${DIR}/AILENV/bin/python" ]; then
|
|
|
|
ENV_PY="${DIR}/AILENV/bin/python"
|
2019-05-01 14:31:47 +02:00
|
|
|
export AIL_VENV=${AIL_HOME}/AILENV/
|
|
|
|
elif [ ! -z "$TRAVIS" ]; then
|
|
|
|
echo "Travis detected"
|
|
|
|
ENV_PY="~/virtualenv/python3.6/bin/python"
|
|
|
|
export AIL_VENV="~/virtualenv/python3.6/"
|
2018-11-22 06:57:24 +01:00
|
|
|
else
|
|
|
|
echo "Please make sure you have a AIL-framework environment, au revoir"
|
|
|
|
exit 1
|
|
|
|
fi
|
|
|
|
|
2018-11-22 07:10:57 +01:00
|
|
|
# redis-server is bundled during install
|
|
|
|
## [ ! -f "`which redis-server`" ] && echo "'redis-server' is not installed/not on PATH. Please fix and run again." && exit 1
|
2018-11-22 06:57:24 +01:00
|
|
|
|
|
|
|
export AIL_BIN=${AIL_HOME}/bin/
|
|
|
|
export AIL_FLASK=${AIL_HOME}/var/www/
|
|
|
|
export AIL_REDIS=${AIL_HOME}/redis/src/
|
|
|
|
export AIL_ARDB=${AIL_HOME}/ardb/src/
|
2014-08-22 14:52:02 +02:00
|
|
|
|
2018-11-22 08:16:31 +01:00
|
|
|
export PATH=$AIL_VENV/bin:$PATH
|
2014-08-22 14:52:02 +02:00
|
|
|
export PATH=$AIL_HOME:$PATH
|
|
|
|
export PATH=$AIL_REDIS:$PATH
|
2018-05-07 14:50:40 +02:00
|
|
|
export PATH=$AIL_ARDB:$PATH
|
2018-07-24 10:41:10 +02:00
|
|
|
export PATH=$AIL_BIN:$PATH
|
|
|
|
export PATH=$AIL_FLASK:$PATH
|
2014-08-06 11:43:40 +02:00
|
|
|
|
2018-07-24 12:18:41 +02:00
|
|
|
isredis=`screen -ls | egrep '[0-9]+.Redis_AIL' | cut -d. -f1`
|
|
|
|
isardb=`screen -ls | egrep '[0-9]+.ARDB_AIL' | cut -d. -f1`
|
|
|
|
islogged=`screen -ls | egrep '[0-9]+.Logging_AIL' | cut -d. -f1`
|
|
|
|
isqueued=`screen -ls | egrep '[0-9]+.Queue_AIL' | cut -d. -f1`
|
|
|
|
isscripted=`screen -ls | egrep '[0-9]+.Script_AIL' | cut -d. -f1`
|
|
|
|
isflasked=`screen -ls | egrep '[0-9]+.Flask_AIL' | cut -d. -f1`
|
2018-09-24 16:23:14 +02:00
|
|
|
iscrawler=`screen -ls | egrep '[0-9]+.Crawler_AIL' | cut -d. -f1`
|
2018-09-19 09:52:27 +02:00
|
|
|
isfeeded=`screen -ls | egrep '[0-9]+.Feeder_Pystemon' | cut -d. -f1`
|
2014-08-06 11:43:40 +02:00
|
|
|
|
|
|
|
function helptext {
|
|
|
|
echo -e $YELLOW"
|
|
|
|
|
|
|
|
.o. ooooo ooooo
|
|
|
|
.888. \`888' \`888'
|
|
|
|
.8\"888. 888 888
|
|
|
|
.8' \`888. 888 888
|
|
|
|
.88ooo8888. 888 888
|
|
|
|
.8' \`888. 888 888 o
|
|
|
|
o88o o8888o o o888o o o888ooooood8
|
|
|
|
|
|
|
|
Analysis Information Leak framework
|
|
|
|
"$DEFAULT"
|
|
|
|
This script launch:
|
|
|
|
"$CYAN"
|
2019-07-26 14:28:02 +02:00
|
|
|
- All the queuing modules.
|
|
|
|
- All the processing modules.
|
2014-08-06 11:43:40 +02:00
|
|
|
- All Redis in memory servers.
|
2018-07-24 10:41:10 +02:00
|
|
|
- All ARDB on disk servers.
|
2014-08-06 11:43:40 +02:00
|
|
|
"$DEFAULT"
|
|
|
|
(Inside screen Daemons)
|
|
|
|
"$DEFAULT"
|
|
|
|
Usage:
|
|
|
|
-----
|
2018-07-24 10:41:10 +02:00
|
|
|
LAUNCH.sh
|
2019-09-12 13:25:29 +02:00
|
|
|
[-l | --launchAuto] LAUNCH DB + Scripts
|
|
|
|
[-k | --killAll] Kill DB + Scripts
|
|
|
|
[-ks | --killscript] Scripts
|
|
|
|
[-u | --update] Update AIL
|
|
|
|
[-c | --crawler] LAUNCH Crawlers
|
|
|
|
[-f | --launchFeeder] LAUNCH Pystemon feeder
|
|
|
|
[-t | --thirdpartyUpdate] Update Web
|
2020-05-19 10:07:27 +02:00
|
|
|
[-rp | --resetPassword] Reset Password
|
2019-09-12 13:25:29 +02:00
|
|
|
[-m | --menu] Display Advanced Menu
|
|
|
|
[-h | --help] Help
|
2014-08-06 11:43:40 +02:00
|
|
|
"
|
|
|
|
}
|
|
|
|
|
|
|
|
function launching_redis {
|
2014-08-22 14:52:02 +02:00
|
|
|
conf_dir="${AIL_HOME}/configs/"
|
2014-08-06 11:43:40 +02:00
|
|
|
|
2017-09-26 16:17:06 +02:00
|
|
|
screen -dmS "Redis_AIL"
|
2014-08-06 11:43:40 +02:00
|
|
|
sleep 0.1
|
|
|
|
echo -e $GREEN"\t* Launching Redis servers"$DEFAULT
|
2017-09-26 16:17:06 +02:00
|
|
|
screen -S "Redis_AIL" -X screen -t "6379" bash -c 'redis-server '$conf_dir'6379.conf ; read x'
|
2014-08-06 11:43:40 +02:00
|
|
|
sleep 0.1
|
2017-09-26 16:17:06 +02:00
|
|
|
screen -S "Redis_AIL" -X screen -t "6380" bash -c 'redis-server '$conf_dir'6380.conf ; read x'
|
2014-08-06 11:43:40 +02:00
|
|
|
sleep 0.1
|
2017-09-26 16:17:06 +02:00
|
|
|
screen -S "Redis_AIL" -X screen -t "6381" bash -c 'redis-server '$conf_dir'6381.conf ; read x'
|
2014-08-06 11:43:40 +02:00
|
|
|
}
|
|
|
|
|
2018-05-07 14:50:40 +02:00
|
|
|
function launching_ardb {
|
|
|
|
conf_dir="${AIL_HOME}/configs/"
|
2017-08-23 10:16:56 +02:00
|
|
|
|
2018-05-07 14:50:40 +02:00
|
|
|
screen -dmS "ARDB_AIL"
|
2014-08-06 11:43:40 +02:00
|
|
|
sleep 0.1
|
2018-05-07 14:50:40 +02:00
|
|
|
echo -e $GREEN"\t* Launching ARDB servers"$DEFAULT
|
2016-08-09 11:59:36 +02:00
|
|
|
|
2018-05-07 14:50:40 +02:00
|
|
|
sleep 0.1
|
2018-07-24 10:41:10 +02:00
|
|
|
screen -S "ARDB_AIL" -X screen -t "6382" bash -c 'cd '${AIL_HOME}'; ardb-server '$conf_dir'6382.conf ; read x'
|
2014-08-06 11:43:40 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
function launching_logs {
|
2017-09-26 16:17:06 +02:00
|
|
|
screen -dmS "Logging_AIL"
|
2014-08-06 11:43:40 +02:00
|
|
|
sleep 0.1
|
|
|
|
echo -e $GREEN"\t* Launching logging process"$DEFAULT
|
2018-11-22 07:48:12 +01:00
|
|
|
screen -S "Logging_AIL" -X screen -t "LogQueue" bash -c "cd ${AIL_BIN}; ${AIL_VENV}/bin/log_subscriber -p 6380 -c Queuing -l ../logs/; read x"
|
2014-08-06 11:43:40 +02:00
|
|
|
sleep 0.1
|
2018-11-22 07:48:12 +01:00
|
|
|
screen -S "Logging_AIL" -X screen -t "LogScript" bash -c "cd ${AIL_BIN}; ${AIL_VENV}/bin/log_subscriber -p 6380 -c Script -l ../logs/; read x"
|
2014-08-06 11:43:40 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
function launching_queues {
|
2017-09-26 16:17:06 +02:00
|
|
|
screen -dmS "Queue_AIL"
|
2014-08-06 11:43:40 +02:00
|
|
|
sleep 0.1
|
|
|
|
|
2014-09-02 15:21:36 +02:00
|
|
|
echo -e $GREEN"\t* Launching all the queues"$DEFAULT
|
2018-11-22 07:41:45 +01:00
|
|
|
screen -S "Queue_AIL" -X screen -t "Queues" bash -c "cd ${AIL_BIN}; ${ENV_PY} launch_queues.py; read x"
|
2014-08-06 11:43:40 +02:00
|
|
|
}
|
|
|
|
|
2018-07-24 10:41:10 +02:00
|
|
|
function checking_configuration {
|
|
|
|
bin_dir=${AIL_HOME}/bin
|
2017-04-25 09:32:05 +02:00
|
|
|
echo -e "\t* Checking configuration"
|
2019-04-23 15:04:45 +02:00
|
|
|
bash -c "${ENV_PY} $bin_dir/Update-conf.py"
|
2017-04-25 09:32:05 +02:00
|
|
|
exitStatus=$?
|
|
|
|
if [ $exitStatus -ge 1 ]; then
|
|
|
|
echo -e $RED"\t* Configuration not up-to-date"$DEFAULT
|
|
|
|
exit
|
|
|
|
fi
|
|
|
|
echo -e $GREEN"\t* Configuration up-to-date"$DEFAULT
|
2018-07-24 10:41:10 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
function launching_scripts {
|
2019-04-23 15:04:45 +02:00
|
|
|
checking_configuration;
|
2017-04-25 09:32:05 +02:00
|
|
|
|
2017-09-26 16:17:06 +02:00
|
|
|
screen -dmS "Script_AIL"
|
2014-08-06 11:43:40 +02:00
|
|
|
sleep 0.1
|
2019-07-26 14:28:02 +02:00
|
|
|
echo -e $GREEN"\t* Launching scripts"$DEFAULT
|
2014-08-06 11:43:40 +02:00
|
|
|
|
2020-04-28 14:14:45 +02:00
|
|
|
# LAUNCH CORE MODULE
|
|
|
|
screen -S "Script_AIL" -X screen -t "JSON_importer" bash -c "cd ${AIL_BIN}/import; ${ENV_PY} ./JSON_importer.py; read x"
|
|
|
|
sleep 0.1
|
|
|
|
|
|
|
|
|
2018-11-22 07:41:45 +01:00
|
|
|
screen -S "Script_AIL" -X screen -t "ModuleInformation" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./ModulesInformationV2.py -k 0 -c 1; read x"
|
2018-04-11 10:48:22 +02:00
|
|
|
sleep 0.1
|
2018-11-22 07:41:45 +01:00
|
|
|
screen -S "Script_AIL" -X screen -t "Mixer" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Mixer.py; read x"
|
2018-04-11 10:48:22 +02:00
|
|
|
sleep 0.1
|
2018-11-22 07:41:45 +01:00
|
|
|
screen -S "Script_AIL" -X screen -t "Global" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Global.py; read x"
|
2018-04-11 10:48:22 +02:00
|
|
|
sleep 0.1
|
2018-11-22 07:41:45 +01:00
|
|
|
screen -S "Script_AIL" -X screen -t "Duplicates" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Duplicates.py; read x"
|
2018-04-12 17:06:57 +02:00
|
|
|
sleep 0.1
|
2018-11-22 07:41:45 +01:00
|
|
|
screen -S "Script_AIL" -X screen -t "DomClassifier" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./DomClassifier.py; read x"
|
2018-04-12 17:06:57 +02:00
|
|
|
sleep 0.1
|
2018-11-22 07:41:45 +01:00
|
|
|
screen -S "Script_AIL" -X screen -t "Categ" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Categ.py; read x"
|
2018-04-16 14:50:04 +02:00
|
|
|
sleep 0.1
|
2018-11-22 07:41:45 +01:00
|
|
|
screen -S "Script_AIL" -X screen -t "CreditCards" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./CreditCards.py; read x"
|
2018-04-12 17:06:57 +02:00
|
|
|
sleep 0.1
|
2018-11-22 07:41:45 +01:00
|
|
|
screen -S "Script_AIL" -X screen -t "BankAccount" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./BankAccount.py; read x"
|
2018-07-26 15:31:58 +02:00
|
|
|
sleep 0.1
|
2018-11-22 07:41:45 +01:00
|
|
|
screen -S "Script_AIL" -X screen -t "Onion" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Onion.py; read x"
|
2018-04-12 17:06:57 +02:00
|
|
|
sleep 0.1
|
2018-11-22 07:41:45 +01:00
|
|
|
screen -S "Script_AIL" -X screen -t "Mail" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Mail.py; read x"
|
2018-04-12 17:06:57 +02:00
|
|
|
sleep 0.1
|
2018-11-22 07:41:45 +01:00
|
|
|
screen -S "Script_AIL" -X screen -t "ApiKey" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./ApiKey.py; read x"
|
2018-04-12 17:06:57 +02:00
|
|
|
sleep 0.1
|
2018-11-22 07:41:45 +01:00
|
|
|
screen -S "Script_AIL" -X screen -t "Web" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Web.py; read x"
|
2018-04-20 10:48:44 +02:00
|
|
|
sleep 0.1
|
2018-11-22 07:41:45 +01:00
|
|
|
screen -S "Script_AIL" -X screen -t "Credential" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Credential.py; read x"
|
2018-04-16 14:50:04 +02:00
|
|
|
sleep 0.1
|
2019-08-09 14:20:13 +02:00
|
|
|
screen -S "Script_AIL" -X screen -t "TermTrackerMod" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./TermTrackerMod.py; read x"
|
2018-04-12 17:06:57 +02:00
|
|
|
sleep 0.1
|
2019-08-09 14:20:13 +02:00
|
|
|
screen -S "Script_AIL" -X screen -t "RegexTracker" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./RegexTracker.py; read x"
|
2018-04-12 17:06:57 +02:00
|
|
|
sleep 0.1
|
2020-08-13 15:15:08 +02:00
|
|
|
screen -S "Script_AIL" -X screen -t "Tracker_Yara" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Yara.py; read x"
|
|
|
|
sleep 0.1
|
2018-11-22 07:41:45 +01:00
|
|
|
screen -S "Script_AIL" -X screen -t "Indexer" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Indexer.py; read x"
|
2018-04-12 17:06:57 +02:00
|
|
|
sleep 0.1
|
2018-11-22 07:41:45 +01:00
|
|
|
screen -S "Script_AIL" -X screen -t "Keys" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Keys.py; read x"
|
2018-04-12 17:06:57 +02:00
|
|
|
sleep 0.1
|
2019-05-20 14:32:07 +02:00
|
|
|
screen -S "Script_AIL" -X screen -t "PgpDump" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./PgpDump.py; read x"
|
|
|
|
sleep 0.1
|
2018-11-22 07:41:45 +01:00
|
|
|
screen -S "Script_AIL" -X screen -t "Decoder" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Decoder.py; read x"
|
2018-04-12 17:06:57 +02:00
|
|
|
sleep 0.1
|
2019-10-29 09:19:26 +01:00
|
|
|
screen -S "Script_AIL" -X screen -t "Cryptocurrency" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Cryptocurrencies.py; read x"
|
|
|
|
sleep 0.1
|
2020-05-12 16:46:11 +02:00
|
|
|
screen -S "Script_AIL" -X screen -t "Telegram" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Telegram.py; read x"
|
|
|
|
sleep 0.1
|
2019-10-29 09:19:26 +01:00
|
|
|
screen -S "Script_AIL" -X screen -t "Tools" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Tools.py; read x"
|
2018-07-19 16:50:42 +02:00
|
|
|
sleep 0.1
|
2018-11-22 07:41:45 +01:00
|
|
|
screen -S "Script_AIL" -X screen -t "Phone" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Phone.py; read x"
|
2018-04-23 16:22:23 +02:00
|
|
|
sleep 0.1
|
2020-06-24 15:13:56 +02:00
|
|
|
#screen -S "Script_AIL" -X screen -t "Release" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Release.py; read x"
|
|
|
|
#sleep 0.1
|
2018-11-22 07:41:45 +01:00
|
|
|
screen -S "Script_AIL" -X screen -t "Cve" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Cve.py; read x"
|
2018-04-12 17:06:57 +02:00
|
|
|
sleep 0.1
|
2018-11-22 07:41:45 +01:00
|
|
|
screen -S "Script_AIL" -X screen -t "WebStats" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./WebStats.py; read x"
|
2018-04-12 17:06:57 +02:00
|
|
|
sleep 0.1
|
2018-11-22 07:41:45 +01:00
|
|
|
screen -S "Script_AIL" -X screen -t "ModuleStats" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./ModuleStats.py; read x"
|
2018-04-12 17:06:57 +02:00
|
|
|
sleep 0.1
|
2018-11-22 07:41:45 +01:00
|
|
|
screen -S "Script_AIL" -X screen -t "SQLInjectionDetection" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./SQLInjectionDetection.py; read x"
|
2018-04-12 17:06:57 +02:00
|
|
|
sleep 0.1
|
2018-11-22 07:41:45 +01:00
|
|
|
screen -S "Script_AIL" -X screen -t "LibInjection" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./LibInjection.py; read x"
|
2018-04-16 14:50:04 +02:00
|
|
|
sleep 0.1
|
2018-11-22 07:41:45 +01:00
|
|
|
screen -S "Script_AIL" -X screen -t "MISPtheHIVEfeeder" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./MISP_The_Hive_feeder.py; read x"
|
2018-06-14 16:51:06 +02:00
|
|
|
sleep 0.1
|
2018-11-22 07:41:45 +01:00
|
|
|
screen -S "Script_AIL" -X screen -t "Tags" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Tags.py; read x"
|
2018-05-16 14:39:01 +02:00
|
|
|
sleep 0.1
|
2018-11-22 07:41:45 +01:00
|
|
|
screen -S "Script_AIL" -X screen -t "SentimentAnalysis" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./SentimentAnalysis.py; read x"
|
2018-06-08 16:49:20 +02:00
|
|
|
sleep 0.1
|
2019-08-09 14:20:13 +02:00
|
|
|
screen -S "Script_AIL" -X screen -t "DbCleaner" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./DbCleaner.py; read x"
|
|
|
|
sleep 0.1
|
2019-04-24 16:37:44 +02:00
|
|
|
screen -S "Script_AIL" -X screen -t "UpdateBackground" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./update-background.py; read x"
|
2019-04-12 16:07:40 +02:00
|
|
|
sleep 0.1
|
2018-11-22 07:41:45 +01:00
|
|
|
screen -S "Script_AIL" -X screen -t "SubmitPaste" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./submit_paste.py; read x"
|
2019-10-02 21:35:30 +02:00
|
|
|
sleep 0.1
|
|
|
|
screen -S "Script_AIL" -X screen -t "IPAddress" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./IPAddress.py; read x"
|
2016-08-19 13:34:02 +02:00
|
|
|
|
2014-08-06 11:43:40 +02:00
|
|
|
}
|
|
|
|
|
2018-09-24 16:23:14 +02:00
|
|
|
function launching_crawler {
|
2018-09-27 16:47:48 +02:00
|
|
|
if [[ ! $iscrawler ]]; then
|
2019-11-05 15:18:03 +01:00
|
|
|
CONFIG=$AIL_HOME/configs/core.cfg
|
2019-02-05 17:16:44 +01:00
|
|
|
lport=$(awk '/^\[Crawler\]/{f=1} f==1&&/^splash_port/{print $3;exit}' "${CONFIG}")
|
2018-09-27 16:47:48 +02:00
|
|
|
|
|
|
|
IFS='-' read -ra PORTS <<< "$lport"
|
|
|
|
if [ ${#PORTS[@]} -eq 1 ]
|
|
|
|
then
|
|
|
|
first_port=${PORTS[0]}
|
|
|
|
last_port=${PORTS[0]}
|
|
|
|
else
|
|
|
|
first_port=${PORTS[0]}
|
|
|
|
last_port=${PORTS[1]}
|
|
|
|
fi
|
|
|
|
|
|
|
|
screen -dmS "Crawler_AIL"
|
2018-09-27 11:14:29 +02:00
|
|
|
sleep 0.1
|
2018-09-24 16:23:14 +02:00
|
|
|
|
2018-09-27 16:47:48 +02:00
|
|
|
for ((i=first_port;i<=last_port;i++)); do
|
2019-02-26 14:50:48 +01:00
|
|
|
screen -S "Crawler_AIL" -X screen -t "onion_crawler:$i" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Crawler.py $i; read x"
|
2018-09-27 16:47:48 +02:00
|
|
|
sleep 0.1
|
|
|
|
done
|
|
|
|
|
|
|
|
echo -e $GREEN"\t* Launching Crawler_AIL scripts"$DEFAULT
|
|
|
|
else
|
|
|
|
echo -e $RED"\t* A screen is already launched"$DEFAULT
|
|
|
|
fi
|
2018-09-24 16:23:14 +02:00
|
|
|
}
|
|
|
|
|
2017-09-26 16:17:06 +02:00
|
|
|
function shutting_down_redis {
|
|
|
|
redis_dir=${AIL_HOME}/redis/src/
|
|
|
|
bash -c $redis_dir'redis-cli -p 6379 SHUTDOWN'
|
|
|
|
sleep 0.1
|
|
|
|
bash -c $redis_dir'redis-cli -p 6380 SHUTDOWN'
|
|
|
|
sleep 0.1
|
|
|
|
bash -c $redis_dir'redis-cli -p 6381 SHUTDOWN'
|
2018-05-07 14:50:40 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
function shutting_down_ardb {
|
|
|
|
redis_dir=${AIL_HOME}/redis/src/
|
2017-09-26 16:17:06 +02:00
|
|
|
bash -c $redis_dir'redis-cli -p 6382 SHUTDOWN'
|
|
|
|
}
|
|
|
|
|
|
|
|
function checking_redis {
|
|
|
|
flag_redis=0
|
|
|
|
redis_dir=${AIL_HOME}/redis/src/
|
|
|
|
bash -c $redis_dir'redis-cli -p 6379 PING | grep "PONG" &> /dev/null'
|
|
|
|
if [ ! $? == 0 ]; then
|
2019-03-26 16:45:01 +01:00
|
|
|
echo -e $RED"\t6379 not ready"$DEFAULT
|
|
|
|
flag_redis=1
|
2017-09-26 16:17:06 +02:00
|
|
|
fi
|
|
|
|
sleep 0.1
|
|
|
|
bash -c $redis_dir'redis-cli -p 6380 PING | grep "PONG" &> /dev/null'
|
|
|
|
if [ ! $? == 0 ]; then
|
2019-03-26 16:45:01 +01:00
|
|
|
echo -e $RED"\t6380 not ready"$DEFAULT
|
|
|
|
flag_redis=1
|
2017-09-26 16:17:06 +02:00
|
|
|
fi
|
|
|
|
sleep 0.1
|
|
|
|
bash -c $redis_dir'redis-cli -p 6381 PING | grep "PONG" &> /dev/null'
|
|
|
|
if [ ! $? == 0 ]; then
|
2019-03-26 16:45:01 +01:00
|
|
|
echo -e $RED"\t6381 not ready"$DEFAULT
|
|
|
|
flag_redis=1
|
2017-09-26 16:17:06 +02:00
|
|
|
fi
|
|
|
|
sleep 0.1
|
2018-05-07 14:50:40 +02:00
|
|
|
|
|
|
|
return $flag_redis;
|
|
|
|
}
|
|
|
|
|
|
|
|
function checking_ardb {
|
|
|
|
flag_ardb=0
|
|
|
|
redis_dir=${AIL_HOME}/redis/src/
|
|
|
|
sleep 0.2
|
2017-09-26 16:17:06 +02:00
|
|
|
bash -c $redis_dir'redis-cli -p 6382 PING | grep "PONG" &> /dev/null'
|
|
|
|
if [ ! $? == 0 ]; then
|
2019-03-26 16:45:01 +01:00
|
|
|
echo -e $RED"\t6382 ARDB not ready"$DEFAULT
|
|
|
|
flag_ardb=1
|
2017-09-26 16:17:06 +02:00
|
|
|
fi
|
2018-05-07 14:50:40 +02:00
|
|
|
|
|
|
|
return $flag_ardb;
|
2017-09-26 16:17:06 +02:00
|
|
|
}
|
|
|
|
|
2019-04-12 15:06:24 +02:00
|
|
|
function wait_until_redis_is_ready {
|
|
|
|
redis_not_ready=true
|
|
|
|
while $redis_not_ready; do
|
|
|
|
if checking_redis; then
|
|
|
|
redis_not_ready=false;
|
|
|
|
else
|
|
|
|
sleep 1
|
|
|
|
fi
|
|
|
|
done
|
|
|
|
echo -e $YELLOW"\t* Redis Launched"$DEFAULT
|
|
|
|
}
|
|
|
|
|
|
|
|
function wait_until_ardb_is_ready {
|
|
|
|
ardb_not_ready=true;
|
|
|
|
while $ardb_not_ready; do
|
|
|
|
if checking_ardb; then
|
|
|
|
ardb_not_ready=false
|
|
|
|
else
|
|
|
|
sleep 3
|
|
|
|
fi
|
|
|
|
done
|
|
|
|
echo -e $YELLOW"\t* ARDB Launched"$DEFAULT
|
|
|
|
}
|
|
|
|
|
2018-07-24 10:41:10 +02:00
|
|
|
function launch_redis {
|
|
|
|
if [[ ! $isredis ]]; then
|
|
|
|
launching_redis;
|
|
|
|
else
|
|
|
|
echo -e $RED"\t* A screen is already launched"$DEFAULT
|
|
|
|
fi
|
|
|
|
}
|
2014-08-06 11:43:40 +02:00
|
|
|
|
2018-07-24 10:41:10 +02:00
|
|
|
function launch_ardb {
|
|
|
|
if [[ ! $isardb ]]; then
|
|
|
|
launching_ardb;
|
|
|
|
else
|
|
|
|
echo -e $RED"\t* A screen is already launched"$DEFAULT
|
|
|
|
fi
|
|
|
|
}
|
2014-08-06 11:43:40 +02:00
|
|
|
|
2018-07-24 10:41:10 +02:00
|
|
|
function launch_logs {
|
|
|
|
if [[ ! $islogged ]]; then
|
|
|
|
launching_logs;
|
|
|
|
else
|
|
|
|
echo -e $RED"\t* A screen is already launched"$DEFAULT
|
|
|
|
fi
|
|
|
|
}
|
2014-08-06 11:43:40 +02:00
|
|
|
|
2018-07-24 10:41:10 +02:00
|
|
|
function launch_queues {
|
|
|
|
if [[ ! $isqueued ]]; then
|
|
|
|
launching_queues;
|
|
|
|
else
|
|
|
|
echo -e $RED"\t* A screen is already launched"$DEFAULT
|
|
|
|
fi
|
|
|
|
}
|
2014-08-06 11:43:40 +02:00
|
|
|
|
2018-07-24 10:41:10 +02:00
|
|
|
function launch_scripts {
|
|
|
|
if [[ ! $isscripted ]]; then
|
|
|
|
sleep 1
|
2018-07-24 12:18:41 +02:00
|
|
|
if checking_ardb && checking_redis; then
|
2019-04-23 15:04:45 +02:00
|
|
|
launching_scripts;
|
2018-07-24 10:41:10 +02:00
|
|
|
else
|
2018-07-24 12:18:41 +02:00
|
|
|
no_script_launched=true
|
|
|
|
while $no_script_launched; do
|
|
|
|
echo -e $YELLOW"\tScript not started, waiting 5 more secondes"$DEFAULT
|
2018-07-24 10:41:10 +02:00
|
|
|
sleep 5
|
|
|
|
if checking_redis && checking_ardb; then
|
2019-04-23 15:04:45 +02:00
|
|
|
launching_scripts;
|
2018-07-24 12:18:41 +02:00
|
|
|
no_script_launched=false
|
2018-07-24 10:41:10 +02:00
|
|
|
else
|
|
|
|
echo -e $RED"\tScript not started"$DEFAULT
|
|
|
|
fi;
|
|
|
|
done
|
|
|
|
fi;
|
|
|
|
else
|
|
|
|
echo -e $RED"\t* A screen is already launched"$DEFAULT
|
|
|
|
fi
|
2014-08-06 11:43:40 +02:00
|
|
|
}
|
|
|
|
|
2018-07-24 10:41:10 +02:00
|
|
|
function launch_flask {
|
|
|
|
if [[ ! $isflasked ]]; then
|
|
|
|
flask_dir=${AIL_FLASK}
|
|
|
|
screen -dmS "Flask_AIL"
|
|
|
|
sleep 0.1
|
|
|
|
echo -e $GREEN"\t* Launching Flask server"$DEFAULT
|
2018-11-22 06:57:24 +01:00
|
|
|
screen -S "Flask_AIL" -X screen -t "Flask_server" bash -c "cd $flask_dir; ls; ${ENV_PY} ./Flask_server.py; read x"
|
2018-07-24 10:41:10 +02:00
|
|
|
else
|
|
|
|
echo -e $RED"\t* A Flask screen is already launched"$DEFAULT
|
|
|
|
fi
|
|
|
|
}
|
2014-08-06 11:43:40 +02:00
|
|
|
|
2018-09-03 16:01:39 +02:00
|
|
|
function launch_feeder {
|
|
|
|
if [[ ! $isfeeded ]]; then
|
2018-09-19 09:52:27 +02:00
|
|
|
screen -dmS "Feeder_Pystemon"
|
2018-09-03 16:01:39 +02:00
|
|
|
sleep 0.1
|
|
|
|
echo -e $GREEN"\t* Launching Pystemon feeder"$DEFAULT
|
2018-11-22 07:41:45 +01:00
|
|
|
screen -S "Feeder_Pystemon" -X screen -t "Pystemon_feeder" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./feeder/pystemon-feeder.py; read x"
|
2018-09-03 16:01:39 +02:00
|
|
|
sleep 0.1
|
2018-11-22 07:41:45 +01:00
|
|
|
screen -S "Feeder_Pystemon" -X screen -t "Pystemon" bash -c "cd ${AIL_HOME}/../pystemon; ${ENV_PY} ./pystemon.py; read x"
|
2018-09-03 16:01:39 +02:00
|
|
|
else
|
|
|
|
echo -e $RED"\t* A Feeder screen is already launched"$DEFAULT
|
|
|
|
fi
|
|
|
|
}
|
|
|
|
|
2019-09-12 13:25:29 +02:00
|
|
|
function killscript {
|
|
|
|
if [[ $islogged || $isqueued || $isscripted || $isflasked || $isfeeded || $iscrawler ]]; then
|
|
|
|
echo -e $GREEN"Killing Script"$DEFAULT
|
|
|
|
kill $islogged $isqueued $isscripted $isflasked $isfeeded $iscrawler
|
|
|
|
sleep 0.2
|
|
|
|
echo -e $ROSE`screen -ls`$DEFAULT
|
|
|
|
echo -e $GREEN"\t* $islogged $isqueued $isscripted $isflasked $isfeeded $iscrawler killed."$DEFAULT
|
|
|
|
else
|
|
|
|
echo -e $RED"\t* No script to kill"$DEFAULT
|
|
|
|
fi
|
|
|
|
}
|
|
|
|
|
2018-07-24 10:41:10 +02:00
|
|
|
function killall {
|
2019-04-12 15:06:24 +02:00
|
|
|
if [[ $isredis || $isardb || $islogged || $isqueued || $isscripted || $isflasked || $isfeeded || $iscrawler ]]; then
|
2018-12-19 11:41:01 +01:00
|
|
|
if [[ $isredis ]]; then
|
|
|
|
echo -e $GREEN"Gracefully closing redis servers"$DEFAULT
|
|
|
|
shutting_down_redis;
|
|
|
|
sleep 0.2
|
|
|
|
fi
|
|
|
|
if [[ $isardb ]]; then
|
|
|
|
echo -e $GREEN"Gracefully closing ardb servers"$DEFAULT
|
|
|
|
shutting_down_ardb;
|
|
|
|
fi
|
2018-07-24 10:41:10 +02:00
|
|
|
echo -e $GREEN"Killing all"$DEFAULT
|
2019-04-12 15:06:24 +02:00
|
|
|
kill $isredis $isardb $islogged $isqueued $isscripted $isflasked $isfeeded $iscrawler
|
2018-07-24 10:41:10 +02:00
|
|
|
sleep 0.2
|
|
|
|
echo -e $ROSE`screen -ls`$DEFAULT
|
2019-04-12 15:06:24 +02:00
|
|
|
echo -e $GREEN"\t* $isredis $isardb $islogged $isqueued $isscripted $isflasked $isfeeded $iscrawler killed."$DEFAULT
|
2018-07-24 10:41:10 +02:00
|
|
|
else
|
|
|
|
echo -e $RED"\t* No screen to kill"$DEFAULT
|
|
|
|
fi
|
|
|
|
}
|
|
|
|
|
|
|
|
function shutdown {
|
|
|
|
bash -c "./Shutdown.py"
|
|
|
|
}
|
|
|
|
|
2018-11-13 15:45:22 +01:00
|
|
|
function update() {
|
2018-11-13 16:54:39 +01:00
|
|
|
bin_dir=${AIL_HOME}/bin
|
2018-11-13 15:45:22 +01:00
|
|
|
|
2019-12-10 16:12:05 +01:00
|
|
|
bash -c "python3 $bin_dir/Update.py $1"
|
2018-11-13 15:45:22 +01:00
|
|
|
exitStatus=$?
|
2020-02-26 10:19:36 +01:00
|
|
|
if [ $exitStatus -ge 3 ]; then
|
|
|
|
echo -e "\t* Update..."
|
|
|
|
bash -c "python3 $bin_dir/Update.py $1"
|
|
|
|
exitStatus=$?
|
|
|
|
if [ $exitStatus -ge 1 ]; then
|
|
|
|
echo -e $RED"\t* Update Error"$DEFAULT
|
|
|
|
exit
|
|
|
|
fi
|
|
|
|
fi
|
|
|
|
|
2018-11-13 15:45:22 +01:00
|
|
|
if [ $exitStatus -ge 1 ]; then
|
|
|
|
echo -e $RED"\t* Update Error"$DEFAULT
|
|
|
|
exit
|
|
|
|
fi
|
2020-05-12 16:46:11 +02:00
|
|
|
|
2018-11-13 15:45:22 +01:00
|
|
|
}
|
|
|
|
|
2018-07-24 10:41:10 +02:00
|
|
|
function update_thirdparty {
|
|
|
|
echo -e "\t* Updating thirdparty..."
|
|
|
|
bash -c "(cd ${AIL_FLASK}; ./update_thirdparty.sh)"
|
|
|
|
exitStatus=$?
|
|
|
|
if [ $exitStatus -ge 1 ]; then
|
|
|
|
echo -e $RED"\t* Thirdparty not up-to-date"$DEFAULT
|
|
|
|
exit
|
|
|
|
else
|
|
|
|
echo -e $GREEN"\t* Thirdparty updated"$DEFAULT
|
2014-08-06 11:43:40 +02:00
|
|
|
fi
|
2018-07-24 10:41:10 +02:00
|
|
|
}
|
|
|
|
|
2020-05-19 10:07:27 +02:00
|
|
|
function reset_password() {
|
|
|
|
echo -e "\t* Reseting UI admin password..."
|
|
|
|
if checking_ardb && checking_redis; then
|
|
|
|
python ${AIL_HOME}/var/www/create_default_user.py &
|
|
|
|
wait
|
|
|
|
else
|
|
|
|
echo -e $RED"\t* Error: Please launch all Redis and ARDB servers"$DEFAULT
|
|
|
|
exit
|
|
|
|
fi
|
|
|
|
}
|
|
|
|
|
2018-07-24 10:41:10 +02:00
|
|
|
function launch_all {
|
2019-04-25 17:17:12 +02:00
|
|
|
checking_configuration;
|
2018-11-13 15:45:22 +01:00
|
|
|
update;
|
2018-07-24 10:41:10 +02:00
|
|
|
launch_redis;
|
|
|
|
launch_ardb;
|
|
|
|
launch_logs;
|
|
|
|
launch_queues;
|
2019-04-23 15:04:45 +02:00
|
|
|
launch_scripts;
|
2018-07-24 10:41:10 +02:00
|
|
|
launch_flask;
|
|
|
|
}
|
2014-08-06 11:43:40 +02:00
|
|
|
|
2019-09-12 13:25:29 +02:00
|
|
|
function menu_display {
|
|
|
|
|
|
|
|
options=("Redis" "Ardb" "Logs" "Queues" "Scripts" "Flask" "Killall" "Shutdown" "Update" "Update-config" "Update-thirdparty")
|
|
|
|
|
|
|
|
menu() {
|
|
|
|
echo "What do you want to Launch?:"
|
|
|
|
for i in ${!options[@]}; do
|
|
|
|
printf "%3d%s) %s\n" $((i+1)) "${choices[i]:- }" "${options[i]}"
|
|
|
|
done
|
|
|
|
[[ "$msg" ]] && echo "$msg"; :
|
|
|
|
}
|
|
|
|
|
|
|
|
prompt="Check an option (again to uncheck, ENTER when done): "
|
|
|
|
|
|
|
|
while menu && read -rp "$prompt" numinput && [[ "$numinput" ]]; do
|
|
|
|
for num in $numinput; do
|
|
|
|
[[ "$num" != *[![:digit:]]* ]] && (( num > 0 && num <= ${#options[@]} )) || {
|
|
|
|
msg="Invalid option: $num"; break
|
|
|
|
}
|
|
|
|
((num--)); msg="${options[num]} was ${choices[num]:+un}checked"
|
|
|
|
[[ "${choices[num]}" ]] && choices[num]="" || choices[num]="+"
|
|
|
|
done
|
|
|
|
done
|
|
|
|
|
|
|
|
for i in ${!options[@]}; do
|
|
|
|
if [[ "${choices[i]}" ]]; then
|
|
|
|
case ${options[i]} in
|
|
|
|
Redis)
|
|
|
|
launch_redis;
|
|
|
|
;;
|
|
|
|
Ardb)
|
|
|
|
launch_ardb;
|
|
|
|
;;
|
|
|
|
Logs)
|
|
|
|
launch_logs;
|
|
|
|
;;
|
|
|
|
Queues)
|
|
|
|
launch_queues;
|
|
|
|
;;
|
|
|
|
Scripts)
|
|
|
|
launch_scripts;
|
|
|
|
;;
|
|
|
|
Flask)
|
|
|
|
launch_flask;
|
|
|
|
;;
|
|
|
|
Crawler)
|
|
|
|
launching_crawler;
|
|
|
|
;;
|
|
|
|
Killall)
|
|
|
|
killall;
|
|
|
|
;;
|
|
|
|
Shutdown)
|
|
|
|
shutdown;
|
|
|
|
;;
|
|
|
|
Update)
|
|
|
|
update;
|
|
|
|
;;
|
|
|
|
Update-config)
|
|
|
|
checking_configuration;
|
|
|
|
;;
|
|
|
|
Update-thirdparty)
|
|
|
|
update_thirdparty;
|
|
|
|
;;
|
|
|
|
esac
|
|
|
|
fi
|
|
|
|
done
|
|
|
|
|
|
|
|
exit
|
2014-08-06 11:43:40 +02:00
|
|
|
|
2019-09-12 13:25:29 +02:00
|
|
|
}
|
2018-07-24 10:41:10 +02:00
|
|
|
|
2014-08-06 11:43:40 +02:00
|
|
|
|
2019-09-12 13:25:29 +02:00
|
|
|
#If no params, display the help
|
|
|
|
[[ $@ ]] || {
|
2014-08-06 11:43:40 +02:00
|
|
|
|
2019-09-12 13:25:29 +02:00
|
|
|
helptext;
|
2018-07-24 10:41:10 +02:00
|
|
|
}
|
|
|
|
|
2019-04-11 17:49:20 +02:00
|
|
|
#echo "$@"
|
2019-03-26 16:45:01 +01:00
|
|
|
|
2018-07-24 10:41:10 +02:00
|
|
|
while [ "$1" != "" ]; do
|
|
|
|
case $1 in
|
2019-04-12 15:06:24 +02:00
|
|
|
-l | --launchAuto ) launch_all "automatic";
|
|
|
|
;;
|
|
|
|
-lr | --launchRedis ) launch_redis;
|
|
|
|
;;
|
|
|
|
-la | --launchARDB ) launch_ardb;
|
|
|
|
;;
|
|
|
|
-lrv | --launchRedisVerify ) launch_redis;
|
|
|
|
wait_until_redis_is_ready;
|
|
|
|
;;
|
|
|
|
-lav | --launchARDBVerify ) launch_ardb;
|
|
|
|
wait_until_ardb_is_ready;
|
|
|
|
;;
|
|
|
|
-k | --killAll ) killall;
|
|
|
|
;;
|
2019-09-12 13:25:29 +02:00
|
|
|
-ks | --killscript ) killscript;
|
|
|
|
;;
|
|
|
|
-m | --menu ) menu_display;
|
|
|
|
;;
|
2019-12-10 16:12:05 +01:00
|
|
|
-u | --update ) update "--manual";
|
2019-04-12 15:06:24 +02:00
|
|
|
;;
|
|
|
|
-t | --thirdpartyUpdate ) update_thirdparty;
|
|
|
|
;;
|
2020-05-19 10:07:27 +02:00
|
|
|
-rp | --resetPassword ) reset_password;
|
|
|
|
;;
|
2019-04-12 15:06:24 +02:00
|
|
|
-c | --crawler ) launching_crawler;
|
|
|
|
;;
|
|
|
|
-f | --launchFeeder ) launch_feeder;
|
|
|
|
;;
|
|
|
|
-h | --help ) helptext;
|
|
|
|
exit
|
|
|
|
;;
|
|
|
|
-kh | --khelp ) helptext;
|
|
|
|
;;
|
|
|
|
* ) helptext
|
|
|
|
exit 1
|
2018-07-24 10:41:10 +02:00
|
|
|
esac
|
|
|
|
shift
|
2014-08-06 11:43:40 +02:00
|
|
|
done
|