From 2fb03baf509775896f62245339be65f5b604f132 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Fri, 22 Jan 2021 17:03:43 +0100 Subject: [PATCH] fix: [background updater] Don't launch updates on fresh install --- bin/update-background.py | 119 ++++-------------- update/v1.5/Update-ARDB_Onions_screenshots.py | 3 + update/v1.5/Update.py | 7 +- update/v2.4/Update.py | 7 +- update/v2.4/Update_domain.py | 5 + update/v2.6/Update.py | 4 - update/v2.6/Update_screenshots.py | 6 - update/v2.7/Update.py | 4 - update/v2.7/Update_domain_tags.py | 6 - update/v3.4/Update.py | 5 +- update/v3.4/Update_domain.py | 2 + 11 files changed, 40 insertions(+), 128 deletions(-) diff --git a/bin/update-background.py b/bin/update-background.py index 981545c1..d4e7b359 100755 --- a/bin/update-background.py +++ b/bin/update-background.py @@ -17,15 +17,20 @@ import subprocess sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) import ConfigLoader -def launch_background_upgrade(version, script_name): +def launch_background_upgrade(version, l_script_name): if r_serv.sismember('ail:to_update', version): r_serv.delete('ail:update_error') r_serv.set('ail:update_in_progress', version) r_serv.set('ail:current_background_update', version) - r_serv.set('ail:current_background_script', 'domain tags update') - update_file = os.path.join(os.environ['AIL_HOME'], 'update', version, script_name) - process = subprocess.run(['python' ,update_file]) + for script_name in l_script_name: + r_serv.set('ail:current_background_script', script_name) + update_file = os.path.join(os.environ['AIL_HOME'], 'update', version, script_name) + process = subprocess.run(['python' ,update_file]) + update_progress = r_serv.get('ail:current_background_script_stat') + #if update_progress: + # if int(update_progress) != 100: + # r_serv.set('ail:update_error', 'Update {} Failed'.format(version)) update_progress = r_serv.get('ail:current_background_script_stat') if update_progress: @@ -36,6 +41,13 @@ def launch_background_upgrade(version, script_name): r_serv.delete('ail:current_background_update') r_serv.srem('ail:to_update', version) +def clean_update_db(): + r_serv.delete('ail:update_error') + r_serv.delete('ail:update_in_progress') + r_serv.delete('ail:current_background_script') + r_serv.delete('ail:current_background_script_stat') + r_serv.delete('ail:current_background_update') + if __name__ == "__main__": config_loader = ConfigLoader.ConfigLoader() @@ -44,97 +56,12 @@ if __name__ == "__main__": r_serv_onion = config_loader.get_redis_conn("ARDB_Onion") config_loader = None - if r_serv.scard('ail:update_v1.5') != 5: - r_serv.delete('ail:update_error') - r_serv.set('ail:update_in_progress', 'v1.5') - r_serv.set('ail:current_background_update', 'v1.5') - if not r_serv.sismember('ail:update_v1.5', 'onions'): - update_file = os.path.join(os.environ['AIL_HOME'], 'update', 'v1.5', 'Update-ARDB_Onions.py') - process = subprocess.run(['python' ,update_file]) - - if not r_serv.sismember('ail:update_v1.5', 'metadata'): - update_file = os.path.join(os.environ['AIL_HOME'], 'update', 'v1.5', 'Update-ARDB_Metadata.py') - process = subprocess.run(['python' ,update_file]) - - if not r_serv.sismember('ail:update_v1.5', 'tags'): - update_file = os.path.join(os.environ['AIL_HOME'], 'update', 'v1.5', 'Update-ARDB_Tags.py') - process = subprocess.run(['python' ,update_file]) - - if not r_serv.sismember('ail:update_v1.5', 'tags_background'): - update_file = os.path.join(os.environ['AIL_HOME'], 'update', 'v1.5', 'Update-ARDB_Tags_background.py') - process = subprocess.run(['python' ,update_file]) - if not r_serv.sismember('ail:update_v1.5', 'crawled_screenshot'): - update_file = os.path.join(os.environ['AIL_HOME'], 'update', 'v1.5', 'Update-ARDB_Onions_screenshots.py') - process = subprocess.run(['python' ,update_file]) - if r_serv.scard('ail:update_v1.5') != 5: - r_serv.set('ail:update_error', 'Update v1.5 Failed, please relaunch the bin/update-background.py script') - else: - r_serv.delete('ail:update_in_progress') - r_serv.delete('ail:current_background_script') - r_serv.delete('ail:current_background_script_stat') - r_serv.delete('ail:current_background_update') - - if r_serv.get('ail:current_background_update') == 'v2.4': - r_serv.delete('ail:update_error') - r_serv.set('ail:update_in_progress', 'v2.4') - r_serv.set('ail:current_background_update', 'v2.4') - r_serv.set('ail:current_background_script', 'domain update') - - update_file = os.path.join(os.environ['AIL_HOME'], 'update', 'v2.4', 'Update_domain.py') - process = subprocess.run(['python' ,update_file]) + if r_serv.scard('ail:to_update') == 0: + clean_update_db() - if int(r_serv_onion.scard('domain_update_v2.4')) != 0: - r_serv.set('ail:update_error', 'Update v2.4 Failed, please relaunch the bin/update-background.py script') - else: - r_serv.delete('ail:update_in_progress') - r_serv.delete('ail:current_background_script') - r_serv.delete('ail:current_background_script_stat') - r_serv.delete('ail:current_background_update') - r_serv.delete('update:nb_elem_to_convert') - r_serv.delete('update:nb_elem_converted') - - if r_serv.sismember('ail:to_update', 'v2.6'): - new_version = 'v2.6' - r_serv.delete('ail:update_error') - r_serv.delete('ail:current_background_script_stat') - r_serv.set('ail:update_in_progress', new_version) - r_serv.set('ail:current_background_update', new_version) - r_serv.set('ail:current_background_script', 'screenshot update') - - update_file = os.path.join(os.environ['AIL_HOME'], 'update', new_version, 'Update_screenshots.py') - process = subprocess.run(['python' ,update_file]) - - update_progress = r_serv.get('ail:current_background_script_stat') - if update_progress: - if int(update_progress) == 100: - r_serv.delete('ail:update_in_progress') - r_serv.delete('ail:current_background_script') - r_serv.delete('ail:current_background_script_stat') - r_serv.delete('ail:current_background_update') - r_serv.srem('ail:to_update', new_version) - - elif r_serv.sismember('ail:to_update', 'v2.7'): - new_version = 'v2.7' - r_serv.delete('ail:update_error') - r_serv.delete('ail:current_background_script_stat') - r_serv.set('ail:update_in_progress', new_version) - r_serv.set('ail:current_background_update', new_version) - r_serv.set('ail:current_background_script', 'domain tags update') - - update_file = os.path.join(os.environ['AIL_HOME'], 'update', new_version, 'Update_domain_tags.py') - process = subprocess.run(['python' ,update_file]) - - update_progress = r_serv.get('ail:current_background_script_stat') - if update_progress: - if int(update_progress) == 100: - r_serv.delete('ail:update_in_progress') - r_serv.delete('ail:current_background_script') - r_serv.delete('ail:current_background_script_stat') - r_serv.delete('ail:current_background_update') - r_serv.srem('ail:to_update', new_version) - - launch_background_upgrade('v2.6', 'Update_screenshots.py') - launch_background_upgrade('v2.7', 'Update_domain_tags.py') - - launch_background_upgrade('v3.4', 'Update_domain.py') + launch_background_upgrade('v1.5', ['Update-ARDB_Onions.py', 'Update-ARDB_Metadata.py', 'Update-ARDB_Tags.py', 'Update-ARDB_Tags_background.py', 'Update-ARDB_Onions_screenshots.py']) + launch_background_upgrade('v2.4', ['Update_domain.py']) + launch_background_upgrade('v2.6', ['Update_screenshots.py']) + launch_background_upgrade('v2.7', ['Update_domain_tags.py']) + launch_background_upgrade('v3.4', ['Update_domain.py']) diff --git a/update/v1.5/Update-ARDB_Onions_screenshots.py b/update/v1.5/Update-ARDB_Onions_screenshots.py index 41f17c4a..442f979c 100755 --- a/update/v1.5/Update-ARDB_Onions_screenshots.py +++ b/update/v1.5/Update-ARDB_Onions_screenshots.py @@ -111,4 +111,7 @@ if __name__ == '__main__': print() print('Done in {} s'.format(end - start_deb)) + r_serv.set('ail:current_background_script_stat', 100) r_serv.sadd('ail:update_v1.5', 'crawled_screenshot') + if r_serv.scard('ail:update_v1.5') != 5: + r_serv.set('ail:update_error', 'Update v1.5 Failed, please relaunch the bin/update-background.py script') diff --git a/update/v1.5/Update.py b/update/v1.5/Update.py index dee56e44..be80c76d 100755 --- a/update/v1.5/Update.py +++ b/update/v1.5/Update.py @@ -43,13 +43,12 @@ if __name__ == '__main__': print('Updating ARDB_Onion Done => {} paths: {} s'.format(index, end - start)) print() + # Add background update + r_serv.sadd('ail:to_update', 'v1.5') + #Set current ail version r_serv.set('ail:version', 'v1.5') - #Set current update_in_progress - r_serv.set('ail:update_in_progress', 'v1.5') - r_serv.set('ail:current_background_update', 'v1.5') - #Set current ail version r_serv.set('ail:update_date_v1.5', datetime.datetime.now().strftime("%Y%m%d")) diff --git a/update/v2.4/Update.py b/update/v2.4/Update.py index 53456330..7728fab7 100755 --- a/update/v2.4/Update.py +++ b/update/v2.4/Update.py @@ -22,14 +22,13 @@ if __name__ == '__main__': r_serv_onion = config_loader.get_redis_conn("ARDB_Onion") config_loader = None - #Set current update_in_progress - r_serv.set('ail:update_in_progress', new_version) - r_serv.set('ail:current_background_update', new_version) - r_serv_onion.sunionstore('domain_update_v2.4', 'full_onion_up', 'full_regular_up') r_serv.set('update:nb_elem_to_convert', r_serv_onion.scard('domain_update_v2.4')) r_serv.set('update:nb_elem_converted',0) + # Add background update + r_serv.sadd('ail:to_update', new_version) + #Set current ail version r_serv.set('ail:version', new_version) diff --git a/update/v2.4/Update_domain.py b/update/v2.4/Update_domain.py index addaedb6..da4ba01d 100755 --- a/update/v2.4/Update_domain.py +++ b/update/v2.4/Update_domain.py @@ -56,6 +56,8 @@ if __name__ == '__main__': r_serv_onion = config_loader.get_redis_conn("ARDB_Onion") config_loader = None + r_serv.set('ail:current_background_script', 'domain update') + nb_elem_to_update = r_serv_db.get('update:nb_elem_to_convert') if not nb_elem_to_update: nb_elem_to_update = 0 @@ -78,4 +80,7 @@ if __name__ == '__main__': update_update_stats() else: + r_serv.delete('update:nb_elem_to_convert') + r_serv.delete('update:nb_elem_converted') + r_serv_db.set('ail:current_background_script_stat', 100) sys.exit(0) diff --git a/update/v2.6/Update.py b/update/v2.6/Update.py index c8cdb5ac..4245eade 100755 --- a/update/v2.6/Update.py +++ b/update/v2.6/Update.py @@ -21,10 +21,6 @@ if __name__ == '__main__': r_serv = config_loader.get_redis_conn("ARDB_DB") config_loader = None - #Set current update_in_progress - r_serv.set('ail:update_in_progress', new_version) - r_serv.set('ail:current_background_update', new_version) - r_serv.sadd('ail:to_update', new_version) #Set current ail version diff --git a/update/v2.6/Update_screenshots.py b/update/v2.6/Update_screenshots.py index 38ba0366..9716dc67 100755 --- a/update/v2.6/Update_screenshots.py +++ b/update/v2.6/Update_screenshots.py @@ -93,9 +93,3 @@ if __name__ == '__main__': end = time.time() print('ALL screenshot updated: {} in {} s'.format(nb, end - start_deb)) - - r_serv_db.delete('ail:update_in_progress') - r_serv_db.delete('ail:current_background_script') - r_serv_db.delete('ail:current_background_script_stat') - r_serv_db.delete('ail:current_background_update') - r_serv_db.srem('ail:to_update', 'v2.6') diff --git a/update/v2.7/Update.py b/update/v2.7/Update.py index 1f4ead2c..fa6f539c 100755 --- a/update/v2.7/Update.py +++ b/update/v2.7/Update.py @@ -23,10 +23,6 @@ if __name__ == '__main__': r_serv_onion = config_loader.get_redis_conn("ARDB_Onion") config_loader = None - #Set current update_in_progress - r_serv.set('ail:update_in_progress', new_version) - r_serv.set('ail:current_background_update', new_version) - r_serv.sadd('ail:to_update', new_version) #### Update tags #### diff --git a/update/v2.7/Update_domain_tags.py b/update/v2.7/Update_domain_tags.py index 937110af..1f9f5eca 100755 --- a/update/v2.7/Update_domain_tags.py +++ b/update/v2.7/Update_domain_tags.py @@ -116,9 +116,3 @@ if __name__ == '__main__': end = time.time() print('ALL domains tags updated in {} s'.format(end - start_deb)) - - r_serv_db.delete('ail:update_in_progress') - r_serv_db.delete('ail:current_background_script') - r_serv_db.delete('ail:current_background_script_stat') - r_serv_db.delete('ail:current_background_update') - r_serv_db.srem('ail:to_update', update_version) diff --git a/update/v3.4/Update.py b/update/v3.4/Update.py index cc7b739b..943d4b3a 100755 --- a/update/v3.4/Update.py +++ b/update/v3.4/Update.py @@ -22,14 +22,11 @@ if __name__ == '__main__': r_serv_onion = config_loader.get_redis_conn("ARDB_Onion") config_loader = None - #Set current update_in_progress - r_serv.set('ail:update_in_progress', new_version) - r_serv.set('ail:current_background_update', new_version) - r_serv_onion.sunionstore('domain_update_v3.4', 'full_onion_up', 'full_regular_up') r_serv.set('update:nb_elem_to_convert', r_serv_onion.scard('domain_update_v3.4')) r_serv.set('update:nb_elem_converted',0) + # Add background update r_serv.sadd('ail:to_update', new_version) #Set current ail version diff --git a/update/v3.4/Update_domain.py b/update/v3.4/Update_domain.py index a065da0e..72ba166d 100755 --- a/update/v3.4/Update_domain.py +++ b/update/v3.4/Update_domain.py @@ -31,6 +31,8 @@ if __name__ == '__main__': r_serv_onion = config_loader.get_redis_conn("ARDB_Onion") config_loader = None + r_serv.set('ail:current_background_script', 'domain languages update') + nb_elem_to_update = r_serv_db.get('update:nb_elem_to_convert') if not nb_elem_to_update: nb_elem_to_update = 1