CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

chg: [Update + Launcher] use background update + add new options in launcher

pull/342/head
Terrtia 2019-04-12 15:06:24 +02:00
parent 5c4db5f2f1
commit 3a8631d44a
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
3 changed files with 103 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

82
bin/LAUNCH.sh
View File

@ -297,6 +297,30 @@ function checking_ardb {
return $flag_ardb;
}
function wait_until_redis_is_ready {
redis_not_ready=true
while $redis_not_ready; do
if checking_redis; then
redis_not_ready=false;
else
sleep 1
fi
done
echo -e $YELLOW"\t* Redis Launched"$DEFAULT
}
function wait_until_ardb_is_ready {
ardb_not_ready=true;
while $ardb_not_ready; do
if checking_ardb; then
ardb_not_ready=false
else
sleep 3
fi
done
echo -e $YELLOW"\t* ARDB Launched"$DEFAULT
}
function launch_redis {
if [[ ! $isredis ]]; then
launching_redis;
@ -378,7 +402,7 @@ function launch_feeder {
}
function killall {
if [[ $isredis || $isardb || $islogged || $isqueued || $isscripted || $isflasked || $isfeeded ]]; then
if [[ $isredis || $isardb || $islogged || $isqueued || $isscripted || $isflasked || $isfeeded || $iscrawler ]]; then
if [[ $isredis ]]; then
echo -e $GREEN"Gracefully closing redis servers"$DEFAULT
shutting_down_redis;
@ -389,10 +413,10 @@ function killall {
shutting_down_ardb;
fi
echo -e $GREEN"Killing all"$DEFAULT
kill $isredis $isardb $islogged $isqueued $isscripted $isflasked $isfeeded
kill $isredis $isardb $islogged $isqueued $isscripted $isflasked $isfeeded $iscrawler
sleep 0.2
echo -e $ROSE`screen -ls`$DEFAULT
echo -e $GREEN"\t* $isredis $isardb $islogged $isqueued $isscripted killed."$DEFAULT
echo -e $GREEN"\t* $isredis $isardb $islogged $isqueued $isscripted $isflasked $isfeeded $iscrawler killed."$DEFAULT
else
echo -e $RED"\t* No screen to kill"$DEFAULT
fi
@ -511,30 +535,36 @@ function launch_all {
while [ "$1" != "" ]; do
case $1 in
-l | --launchAuto ) launch_all "automatic";
;;
-lr | --launchRedis ) launch_redis;
;;
-la | --launchARDB ) launch_ardb;
;;
-k | --killAll ) killall;
;;
-u | --update ) update;
;;
-t | --thirdpartyUpdate ) update_thirdparty;
;;
-c | --crawler ) launching_crawler;
;;
-f | --launchFeeder ) launch_feeder;
;;
-h | --help ) helptext;
exit
;;
-kh | --khelp ) helptext;
-l | --launchAuto ) launch_all "automatic";
;;
-lr | --launchRedis ) launch_redis;
;;
-la | --launchARDB ) launch_ardb;
;;
-lrv | --launchRedisVerify ) launch_redis;
wait_until_redis_is_ready;
;;
-lav | --launchARDBVerify ) launch_ardb;
wait_until_ardb_is_ready;
;;
-k | --killAll ) killall;
;;
-u | --update ) update;
;;
-t | --thirdpartyUpdate ) update_thirdparty;
;;
-c | --crawler ) launching_crawler;
;;
-f | --launchFeeder ) launch_feeder;
;;
-h | --help ) helptext;
exit
;;
-kh | --khelp ) helptext;
;;
* ) helptext
exit 1
;;
* ) helptext
exit 1
esac
shift
done

23
bin/update-background.py
View File

@ -38,21 +38,20 @@ if __name__ == "__main__":
if ail_version == 'v1.5':
onions_update_status = r_serv.get('v1.5:onions')
if onions_update_status is None:
if int(onions_update_status) != 1:
update_file = os.path.join(os.environ['AIL_HOME'], 'update', 'v1.4', 'Update-ARDB_Onions.py')
process = subprocess.run(['unbuffer', 'python' ,'update_file'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
update_file = os.path.join(os.environ['AIL_HOME'], 'update', 'v1.4', 'Update-ARDB_Onions.py')
process = subprocess.run(['python' ,update_file])
metadata_update_status = r_serv.get('v1.5:metadata')
if metadata_update_status is None:
if int(metadata_update_status) != 1:
pass
#launch update
update_file = os.path.join(os.environ['AIL_HOME'], 'update', 'v1.4', 'Update-ARDB_Metadata.py')
process = subprocess.run(['python' ,update_file])
tags_update_status = r_serv.get('v1.5:tags')
if tags_update_status is None:
if int(tags_update_status) != 1:
pass
#launch update
update_file = os.path.join(os.environ['AIL_HOME'], 'update', 'v1.4', 'Update-ARDB_Tags.py')
process = subprocess.run(['python' ,update_file])
tags_background_update_status = r_serv.get('v1.5:tags_background')
if tags_background_update_status is None:
if int(tags_background_update_status) != 1:
pass
#launch update
update_file = os.path.join(os.environ['AIL_HOME'], 'update', 'v1.4', 'Update-ARDB_Tags_background.py')
process = subprocess.run(['python' ,update_file])

41
update/v1.4/Update.sh
View File

@ -12,17 +12,48 @@ export PATH=$AIL_ARDB:$PATH
export PATH=$AIL_BIN:$PATH
export PATH=$AIL_FLASK:$PATH
GREEN="\\033[1;32m"
DEFAULT="\\033[0;39m"
echo -e $GREEN"Shutting down AIL ..."$DEFAULT
bash ${AIL_BIN}/LAUNCH.sh -k &
wait
echo ""
bash -c "bash ${AIL_HOME}/update/bin/Update_Redis.sh"
#bash -c "bash ${AIL_HOME}/update/bin/Update_Redis.sh"
#bash -c "bash ${AIL_HOME}/update/bin/Update_ARDB.sh"
echo ""
echo "Fixing ARDB ..."
echo -e $GREEN"Update DomainClassifier"$DEFAULT
echo ""
pip3 install --upgrade --force-reinstall git+https://github.com/D4-project/BGP-Ranking.git/@28013297efb039d2ebbce96ee2d89493f6ae56b0#subdirectory=client&egg=pybgpranking
pip3 install --upgrade --force-reinstall git+https://github.com/adulau/DomainClassifier.git
wait
echo ""
bash -c "unbuffer python ${AIL_HOME}/update/v1.5/Update.py"
echo "Shutting down ARDB ..."
bash -c "bash ${AIL_BIN}/LAUNCH.sh -k"
echo ""
echo -e $GREEN"Update Web thirdparty"$DEFAULT
echo ""
bash ${AIL_FLASK}update_thirdparty.sh &
wait
echo ""
bash ${AIL_BIN}LAUNCH.sh -lav &
wait
echo ""
echo ""
echo -e $GREEN"Fixing ARDB ..."$DEFAULT
echo ""
python ${AIL_HOME}/update/v1.4/Update.py &
wait
echo ""
echo ""
echo ""
echo -e $GREEN"Shutting down ARDB ..."$DEFAULT
bash ${AIL_BIN}/LAUNCH.sh -k &
wait
echo ""