From 3a8631d44aaa10337735897fe7b702f5602b0a27 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Fri, 12 Apr 2019 15:06:24 +0200 Subject: [PATCH] chg: [Update + Launcher] use background update + add new options in launcher --- bin/LAUNCH.sh | 82 +++++++++++++++++++++++++++------------- bin/update-background.py | 23 ++++++----- update/v1.4/Update.sh | 41 +++++++++++++++++--- 3 files changed, 103 insertions(+), 43 deletions(-) diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index aa658f4e..5978f0cb 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -297,6 +297,30 @@ function checking_ardb { return $flag_ardb; } +function wait_until_redis_is_ready { + redis_not_ready=true + while $redis_not_ready; do + if checking_redis; then + redis_not_ready=false; + else + sleep 1 + fi + done + echo -e $YELLOW"\t* Redis Launched"$DEFAULT +} + +function wait_until_ardb_is_ready { + ardb_not_ready=true; + while $ardb_not_ready; do + if checking_ardb; then + ardb_not_ready=false + else + sleep 3 + fi + done + echo -e $YELLOW"\t* ARDB Launched"$DEFAULT +} + function launch_redis { if [[ ! $isredis ]]; then launching_redis; @@ -378,7 +402,7 @@ function launch_feeder { } function killall { - if [[ $isredis || $isardb || $islogged || $isqueued || $isscripted || $isflasked || $isfeeded ]]; then + if [[ $isredis || $isardb || $islogged || $isqueued || $isscripted || $isflasked || $isfeeded || $iscrawler ]]; then if [[ $isredis ]]; then echo -e $GREEN"Gracefully closing redis servers"$DEFAULT shutting_down_redis; @@ -389,10 +413,10 @@ function killall { shutting_down_ardb; fi echo -e $GREEN"Killing all"$DEFAULT - kill $isredis $isardb $islogged $isqueued $isscripted $isflasked $isfeeded + kill $isredis $isardb $islogged $isqueued $isscripted $isflasked $isfeeded $iscrawler sleep 0.2 echo -e $ROSE`screen -ls`$DEFAULT - echo -e $GREEN"\t* $isredis $isardb $islogged $isqueued $isscripted killed."$DEFAULT + echo -e $GREEN"\t* $isredis $isardb $islogged $isqueued $isscripted $isflasked $isfeeded $iscrawler killed."$DEFAULT else echo -e $RED"\t* No screen to kill"$DEFAULT fi @@ -511,30 +535,36 @@ function launch_all { while [ "$1" != "" ]; do case $1 in - -l | --launchAuto ) launch_all "automatic"; - ;; - -lr | --launchRedis ) launch_redis; - ;; - -la | --launchARDB ) launch_ardb; - ;; - -k | --killAll ) killall; - ;; - -u | --update ) update; - ;; - -t | --thirdpartyUpdate ) update_thirdparty; - ;; - -c | --crawler ) launching_crawler; - ;; - -f | --launchFeeder ) launch_feeder; - ;; - -h | --help ) helptext; - exit - ;; - -kh | --khelp ) helptext; + -l | --launchAuto ) launch_all "automatic"; + ;; + -lr | --launchRedis ) launch_redis; + ;; + -la | --launchARDB ) launch_ardb; + ;; + -lrv | --launchRedisVerify ) launch_redis; + wait_until_redis_is_ready; + ;; + -lav | --launchARDBVerify ) launch_ardb; + wait_until_ardb_is_ready; + ;; + -k | --killAll ) killall; + ;; + -u | --update ) update; + ;; + -t | --thirdpartyUpdate ) update_thirdparty; + ;; + -c | --crawler ) launching_crawler; + ;; + -f | --launchFeeder ) launch_feeder; + ;; + -h | --help ) helptext; + exit + ;; + -kh | --khelp ) helptext; - ;; - * ) helptext - exit 1 + ;; + * ) helptext + exit 1 esac shift done diff --git a/bin/update-background.py b/bin/update-background.py index ede08c95..86aed2df 100755 --- a/bin/update-background.py +++ b/bin/update-background.py @@ -38,21 +38,20 @@ if __name__ == "__main__": if ail_version == 'v1.5': onions_update_status = r_serv.get('v1.5:onions') if onions_update_status is None: - if int(onions_update_status) != 1: - update_file = os.path.join(os.environ['AIL_HOME'], 'update', 'v1.4', 'Update-ARDB_Onions.py') - process = subprocess.run(['unbuffer', 'python' ,'update_file'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + update_file = os.path.join(os.environ['AIL_HOME'], 'update', 'v1.4', 'Update-ARDB_Onions.py') + process = subprocess.run(['python' ,update_file]) + metadata_update_status = r_serv.get('v1.5:metadata') if metadata_update_status is None: - if int(metadata_update_status) != 1: - pass - #launch update + update_file = os.path.join(os.environ['AIL_HOME'], 'update', 'v1.4', 'Update-ARDB_Metadata.py') + process = subprocess.run(['python' ,update_file]) + tags_update_status = r_serv.get('v1.5:tags') if tags_update_status is None: - if int(tags_update_status) != 1: - pass - #launch update + update_file = os.path.join(os.environ['AIL_HOME'], 'update', 'v1.4', 'Update-ARDB_Tags.py') + process = subprocess.run(['python' ,update_file]) + tags_background_update_status = r_serv.get('v1.5:tags_background') if tags_background_update_status is None: - if int(tags_background_update_status) != 1: - pass - #launch update + update_file = os.path.join(os.environ['AIL_HOME'], 'update', 'v1.4', 'Update-ARDB_Tags_background.py') + process = subprocess.run(['python' ,update_file]) diff --git a/update/v1.4/Update.sh b/update/v1.4/Update.sh index d2500213..b0adf079 100755 --- a/update/v1.4/Update.sh +++ b/update/v1.4/Update.sh @@ -12,17 +12,48 @@ export PATH=$AIL_ARDB:$PATH export PATH=$AIL_BIN:$PATH export PATH=$AIL_FLASK:$PATH +GREEN="\\033[1;32m" +DEFAULT="\\033[0;39m" + +echo -e $GREEN"Shutting down AIL ..."$DEFAULT +bash ${AIL_BIN}/LAUNCH.sh -k & +wait + echo "" -bash -c "bash ${AIL_HOME}/update/bin/Update_Redis.sh" +#bash -c "bash ${AIL_HOME}/update/bin/Update_Redis.sh" #bash -c "bash ${AIL_HOME}/update/bin/Update_ARDB.sh" echo "" -echo "Fixing ARDB ..." +echo -e $GREEN"Update DomainClassifier"$DEFAULT +echo "" +pip3 install --upgrade --force-reinstall git+https://github.com/D4-project/BGP-Ranking.git/@28013297efb039d2ebbce96ee2d89493f6ae56b0#subdirectory=client&egg=pybgpranking +pip3 install --upgrade --force-reinstall git+https://github.com/adulau/DomainClassifier.git +wait echo "" -bash -c "unbuffer python ${AIL_HOME}/update/v1.5/Update.py" -echo "Shutting down ARDB ..." -bash -c "bash ${AIL_BIN}/LAUNCH.sh -k" +echo "" +echo -e $GREEN"Update Web thirdparty"$DEFAULT +echo "" +bash ${AIL_FLASK}update_thirdparty.sh & +wait +echo "" + +bash ${AIL_BIN}LAUNCH.sh -lav & +wait +echo "" + +echo "" +echo -e $GREEN"Fixing ARDB ..."$DEFAULT +echo "" +python ${AIL_HOME}/update/v1.4/Update.py & +wait +echo "" +echo "" + +echo "" +echo -e $GREEN"Shutting down ARDB ..."$DEFAULT +bash ${AIL_BIN}/LAUNCH.sh -k & +wait echo ""