mirror of https://github.com/CIRCL/AIL-framework
fix: [background updater] Don't launch updates on fresh install
parent
11d537e2eb
commit
2fb03baf50
|
@ -17,15 +17,20 @@ import subprocess
|
||||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||||
import ConfigLoader
|
import ConfigLoader
|
||||||
|
|
||||||
def launch_background_upgrade(version, script_name):
|
def launch_background_upgrade(version, l_script_name):
|
||||||
if r_serv.sismember('ail:to_update', version):
|
if r_serv.sismember('ail:to_update', version):
|
||||||
r_serv.delete('ail:update_error')
|
r_serv.delete('ail:update_error')
|
||||||
r_serv.set('ail:update_in_progress', version)
|
r_serv.set('ail:update_in_progress', version)
|
||||||
r_serv.set('ail:current_background_update', version)
|
r_serv.set('ail:current_background_update', version)
|
||||||
r_serv.set('ail:current_background_script', 'domain tags update')
|
|
||||||
|
|
||||||
|
for script_name in l_script_name:
|
||||||
|
r_serv.set('ail:current_background_script', script_name)
|
||||||
update_file = os.path.join(os.environ['AIL_HOME'], 'update', version, script_name)
|
update_file = os.path.join(os.environ['AIL_HOME'], 'update', version, script_name)
|
||||||
process = subprocess.run(['python' ,update_file])
|
process = subprocess.run(['python' ,update_file])
|
||||||
|
update_progress = r_serv.get('ail:current_background_script_stat')
|
||||||
|
#if update_progress:
|
||||||
|
# if int(update_progress) != 100:
|
||||||
|
# r_serv.set('ail:update_error', 'Update {} Failed'.format(version))
|
||||||
|
|
||||||
update_progress = r_serv.get('ail:current_background_script_stat')
|
update_progress = r_serv.get('ail:current_background_script_stat')
|
||||||
if update_progress:
|
if update_progress:
|
||||||
|
@ -36,6 +41,13 @@ def launch_background_upgrade(version, script_name):
|
||||||
r_serv.delete('ail:current_background_update')
|
r_serv.delete('ail:current_background_update')
|
||||||
r_serv.srem('ail:to_update', version)
|
r_serv.srem('ail:to_update', version)
|
||||||
|
|
||||||
|
def clean_update_db():
|
||||||
|
r_serv.delete('ail:update_error')
|
||||||
|
r_serv.delete('ail:update_in_progress')
|
||||||
|
r_serv.delete('ail:current_background_script')
|
||||||
|
r_serv.delete('ail:current_background_script_stat')
|
||||||
|
r_serv.delete('ail:current_background_update')
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|
||||||
config_loader = ConfigLoader.ConfigLoader()
|
config_loader = ConfigLoader.ConfigLoader()
|
||||||
|
@ -44,97 +56,12 @@ if __name__ == "__main__":
|
||||||
r_serv_onion = config_loader.get_redis_conn("ARDB_Onion")
|
r_serv_onion = config_loader.get_redis_conn("ARDB_Onion")
|
||||||
config_loader = None
|
config_loader = None
|
||||||
|
|
||||||
if r_serv.scard('ail:update_v1.5') != 5:
|
if r_serv.scard('ail:to_update') == 0:
|
||||||
r_serv.delete('ail:update_error')
|
clean_update_db()
|
||||||
r_serv.set('ail:update_in_progress', 'v1.5')
|
|
||||||
r_serv.set('ail:current_background_update', 'v1.5')
|
|
||||||
if not r_serv.sismember('ail:update_v1.5', 'onions'):
|
|
||||||
update_file = os.path.join(os.environ['AIL_HOME'], 'update', 'v1.5', 'Update-ARDB_Onions.py')
|
|
||||||
process = subprocess.run(['python' ,update_file])
|
|
||||||
|
|
||||||
if not r_serv.sismember('ail:update_v1.5', 'metadata'):
|
|
||||||
update_file = os.path.join(os.environ['AIL_HOME'], 'update', 'v1.5', 'Update-ARDB_Metadata.py')
|
|
||||||
process = subprocess.run(['python' ,update_file])
|
|
||||||
|
|
||||||
if not r_serv.sismember('ail:update_v1.5', 'tags'):
|
|
||||||
update_file = os.path.join(os.environ['AIL_HOME'], 'update', 'v1.5', 'Update-ARDB_Tags.py')
|
|
||||||
process = subprocess.run(['python' ,update_file])
|
|
||||||
|
|
||||||
if not r_serv.sismember('ail:update_v1.5', 'tags_background'):
|
|
||||||
update_file = os.path.join(os.environ['AIL_HOME'], 'update', 'v1.5', 'Update-ARDB_Tags_background.py')
|
|
||||||
process = subprocess.run(['python' ,update_file])
|
|
||||||
if not r_serv.sismember('ail:update_v1.5', 'crawled_screenshot'):
|
|
||||||
update_file = os.path.join(os.environ['AIL_HOME'], 'update', 'v1.5', 'Update-ARDB_Onions_screenshots.py')
|
|
||||||
process = subprocess.run(['python' ,update_file])
|
|
||||||
if r_serv.scard('ail:update_v1.5') != 5:
|
|
||||||
r_serv.set('ail:update_error', 'Update v1.5 Failed, please relaunch the bin/update-background.py script')
|
|
||||||
else:
|
|
||||||
r_serv.delete('ail:update_in_progress')
|
|
||||||
r_serv.delete('ail:current_background_script')
|
|
||||||
r_serv.delete('ail:current_background_script_stat')
|
|
||||||
r_serv.delete('ail:current_background_update')
|
|
||||||
|
|
||||||
if r_serv.get('ail:current_background_update') == 'v2.4':
|
|
||||||
r_serv.delete('ail:update_error')
|
|
||||||
r_serv.set('ail:update_in_progress', 'v2.4')
|
|
||||||
r_serv.set('ail:current_background_update', 'v2.4')
|
|
||||||
r_serv.set('ail:current_background_script', 'domain update')
|
|
||||||
|
|
||||||
update_file = os.path.join(os.environ['AIL_HOME'], 'update', 'v2.4', 'Update_domain.py')
|
|
||||||
process = subprocess.run(['python' ,update_file])
|
|
||||||
|
|
||||||
|
|
||||||
if int(r_serv_onion.scard('domain_update_v2.4')) != 0:
|
launch_background_upgrade('v1.5', ['Update-ARDB_Onions.py', 'Update-ARDB_Metadata.py', 'Update-ARDB_Tags.py', 'Update-ARDB_Tags_background.py', 'Update-ARDB_Onions_screenshots.py'])
|
||||||
r_serv.set('ail:update_error', 'Update v2.4 Failed, please relaunch the bin/update-background.py script')
|
launch_background_upgrade('v2.4', ['Update_domain.py'])
|
||||||
else:
|
launch_background_upgrade('v2.6', ['Update_screenshots.py'])
|
||||||
r_serv.delete('ail:update_in_progress')
|
launch_background_upgrade('v2.7', ['Update_domain_tags.py'])
|
||||||
r_serv.delete('ail:current_background_script')
|
launch_background_upgrade('v3.4', ['Update_domain.py'])
|
||||||
r_serv.delete('ail:current_background_script_stat')
|
|
||||||
r_serv.delete('ail:current_background_update')
|
|
||||||
r_serv.delete('update:nb_elem_to_convert')
|
|
||||||
r_serv.delete('update:nb_elem_converted')
|
|
||||||
|
|
||||||
if r_serv.sismember('ail:to_update', 'v2.6'):
|
|
||||||
new_version = 'v2.6'
|
|
||||||
r_serv.delete('ail:update_error')
|
|
||||||
r_serv.delete('ail:current_background_script_stat')
|
|
||||||
r_serv.set('ail:update_in_progress', new_version)
|
|
||||||
r_serv.set('ail:current_background_update', new_version)
|
|
||||||
r_serv.set('ail:current_background_script', 'screenshot update')
|
|
||||||
|
|
||||||
update_file = os.path.join(os.environ['AIL_HOME'], 'update', new_version, 'Update_screenshots.py')
|
|
||||||
process = subprocess.run(['python' ,update_file])
|
|
||||||
|
|
||||||
update_progress = r_serv.get('ail:current_background_script_stat')
|
|
||||||
if update_progress:
|
|
||||||
if int(update_progress) == 100:
|
|
||||||
r_serv.delete('ail:update_in_progress')
|
|
||||||
r_serv.delete('ail:current_background_script')
|
|
||||||
r_serv.delete('ail:current_background_script_stat')
|
|
||||||
r_serv.delete('ail:current_background_update')
|
|
||||||
r_serv.srem('ail:to_update', new_version)
|
|
||||||
|
|
||||||
elif r_serv.sismember('ail:to_update', 'v2.7'):
|
|
||||||
new_version = 'v2.7'
|
|
||||||
r_serv.delete('ail:update_error')
|
|
||||||
r_serv.delete('ail:current_background_script_stat')
|
|
||||||
r_serv.set('ail:update_in_progress', new_version)
|
|
||||||
r_serv.set('ail:current_background_update', new_version)
|
|
||||||
r_serv.set('ail:current_background_script', 'domain tags update')
|
|
||||||
|
|
||||||
update_file = os.path.join(os.environ['AIL_HOME'], 'update', new_version, 'Update_domain_tags.py')
|
|
||||||
process = subprocess.run(['python' ,update_file])
|
|
||||||
|
|
||||||
update_progress = r_serv.get('ail:current_background_script_stat')
|
|
||||||
if update_progress:
|
|
||||||
if int(update_progress) == 100:
|
|
||||||
r_serv.delete('ail:update_in_progress')
|
|
||||||
r_serv.delete('ail:current_background_script')
|
|
||||||
r_serv.delete('ail:current_background_script_stat')
|
|
||||||
r_serv.delete('ail:current_background_update')
|
|
||||||
r_serv.srem('ail:to_update', new_version)
|
|
||||||
|
|
||||||
launch_background_upgrade('v2.6', 'Update_screenshots.py')
|
|
||||||
launch_background_upgrade('v2.7', 'Update_domain_tags.py')
|
|
||||||
|
|
||||||
launch_background_upgrade('v3.4', 'Update_domain.py')
|
|
||||||
|
|
|
@ -111,4 +111,7 @@ if __name__ == '__main__':
|
||||||
print()
|
print()
|
||||||
print('Done in {} s'.format(end - start_deb))
|
print('Done in {} s'.format(end - start_deb))
|
||||||
|
|
||||||
|
r_serv.set('ail:current_background_script_stat', 100)
|
||||||
r_serv.sadd('ail:update_v1.5', 'crawled_screenshot')
|
r_serv.sadd('ail:update_v1.5', 'crawled_screenshot')
|
||||||
|
if r_serv.scard('ail:update_v1.5') != 5:
|
||||||
|
r_serv.set('ail:update_error', 'Update v1.5 Failed, please relaunch the bin/update-background.py script')
|
||||||
|
|
|
@ -43,13 +43,12 @@ if __name__ == '__main__':
|
||||||
print('Updating ARDB_Onion Done => {} paths: {} s'.format(index, end - start))
|
print('Updating ARDB_Onion Done => {} paths: {} s'.format(index, end - start))
|
||||||
print()
|
print()
|
||||||
|
|
||||||
|
# Add background update
|
||||||
|
r_serv.sadd('ail:to_update', 'v1.5')
|
||||||
|
|
||||||
#Set current ail version
|
#Set current ail version
|
||||||
r_serv.set('ail:version', 'v1.5')
|
r_serv.set('ail:version', 'v1.5')
|
||||||
|
|
||||||
#Set current update_in_progress
|
|
||||||
r_serv.set('ail:update_in_progress', 'v1.5')
|
|
||||||
r_serv.set('ail:current_background_update', 'v1.5')
|
|
||||||
|
|
||||||
#Set current ail version
|
#Set current ail version
|
||||||
r_serv.set('ail:update_date_v1.5', datetime.datetime.now().strftime("%Y%m%d"))
|
r_serv.set('ail:update_date_v1.5', datetime.datetime.now().strftime("%Y%m%d"))
|
||||||
|
|
||||||
|
|
|
@ -22,14 +22,13 @@ if __name__ == '__main__':
|
||||||
r_serv_onion = config_loader.get_redis_conn("ARDB_Onion")
|
r_serv_onion = config_loader.get_redis_conn("ARDB_Onion")
|
||||||
config_loader = None
|
config_loader = None
|
||||||
|
|
||||||
#Set current update_in_progress
|
|
||||||
r_serv.set('ail:update_in_progress', new_version)
|
|
||||||
r_serv.set('ail:current_background_update', new_version)
|
|
||||||
|
|
||||||
r_serv_onion.sunionstore('domain_update_v2.4', 'full_onion_up', 'full_regular_up')
|
r_serv_onion.sunionstore('domain_update_v2.4', 'full_onion_up', 'full_regular_up')
|
||||||
r_serv.set('update:nb_elem_to_convert', r_serv_onion.scard('domain_update_v2.4'))
|
r_serv.set('update:nb_elem_to_convert', r_serv_onion.scard('domain_update_v2.4'))
|
||||||
r_serv.set('update:nb_elem_converted',0)
|
r_serv.set('update:nb_elem_converted',0)
|
||||||
|
|
||||||
|
# Add background update
|
||||||
|
r_serv.sadd('ail:to_update', new_version)
|
||||||
|
|
||||||
#Set current ail version
|
#Set current ail version
|
||||||
r_serv.set('ail:version', new_version)
|
r_serv.set('ail:version', new_version)
|
||||||
|
|
||||||
|
|
|
@ -56,6 +56,8 @@ if __name__ == '__main__':
|
||||||
r_serv_onion = config_loader.get_redis_conn("ARDB_Onion")
|
r_serv_onion = config_loader.get_redis_conn("ARDB_Onion")
|
||||||
config_loader = None
|
config_loader = None
|
||||||
|
|
||||||
|
r_serv.set('ail:current_background_script', 'domain update')
|
||||||
|
|
||||||
nb_elem_to_update = r_serv_db.get('update:nb_elem_to_convert')
|
nb_elem_to_update = r_serv_db.get('update:nb_elem_to_convert')
|
||||||
if not nb_elem_to_update:
|
if not nb_elem_to_update:
|
||||||
nb_elem_to_update = 0
|
nb_elem_to_update = 0
|
||||||
|
@ -78,4 +80,7 @@ if __name__ == '__main__':
|
||||||
update_update_stats()
|
update_update_stats()
|
||||||
|
|
||||||
else:
|
else:
|
||||||
|
r_serv.delete('update:nb_elem_to_convert')
|
||||||
|
r_serv.delete('update:nb_elem_converted')
|
||||||
|
r_serv_db.set('ail:current_background_script_stat', 100)
|
||||||
sys.exit(0)
|
sys.exit(0)
|
||||||
|
|
|
@ -21,10 +21,6 @@ if __name__ == '__main__':
|
||||||
r_serv = config_loader.get_redis_conn("ARDB_DB")
|
r_serv = config_loader.get_redis_conn("ARDB_DB")
|
||||||
config_loader = None
|
config_loader = None
|
||||||
|
|
||||||
#Set current update_in_progress
|
|
||||||
r_serv.set('ail:update_in_progress', new_version)
|
|
||||||
r_serv.set('ail:current_background_update', new_version)
|
|
||||||
|
|
||||||
r_serv.sadd('ail:to_update', new_version)
|
r_serv.sadd('ail:to_update', new_version)
|
||||||
|
|
||||||
#Set current ail version
|
#Set current ail version
|
||||||
|
|
|
@ -93,9 +93,3 @@ if __name__ == '__main__':
|
||||||
|
|
||||||
end = time.time()
|
end = time.time()
|
||||||
print('ALL screenshot updated: {} in {} s'.format(nb, end - start_deb))
|
print('ALL screenshot updated: {} in {} s'.format(nb, end - start_deb))
|
||||||
|
|
||||||
r_serv_db.delete('ail:update_in_progress')
|
|
||||||
r_serv_db.delete('ail:current_background_script')
|
|
||||||
r_serv_db.delete('ail:current_background_script_stat')
|
|
||||||
r_serv_db.delete('ail:current_background_update')
|
|
||||||
r_serv_db.srem('ail:to_update', 'v2.6')
|
|
||||||
|
|
|
@ -23,10 +23,6 @@ if __name__ == '__main__':
|
||||||
r_serv_onion = config_loader.get_redis_conn("ARDB_Onion")
|
r_serv_onion = config_loader.get_redis_conn("ARDB_Onion")
|
||||||
config_loader = None
|
config_loader = None
|
||||||
|
|
||||||
#Set current update_in_progress
|
|
||||||
r_serv.set('ail:update_in_progress', new_version)
|
|
||||||
r_serv.set('ail:current_background_update', new_version)
|
|
||||||
|
|
||||||
r_serv.sadd('ail:to_update', new_version)
|
r_serv.sadd('ail:to_update', new_version)
|
||||||
|
|
||||||
#### Update tags ####
|
#### Update tags ####
|
||||||
|
|
|
@ -116,9 +116,3 @@ if __name__ == '__main__':
|
||||||
|
|
||||||
end = time.time()
|
end = time.time()
|
||||||
print('ALL domains tags updated in {} s'.format(end - start_deb))
|
print('ALL domains tags updated in {} s'.format(end - start_deb))
|
||||||
|
|
||||||
r_serv_db.delete('ail:update_in_progress')
|
|
||||||
r_serv_db.delete('ail:current_background_script')
|
|
||||||
r_serv_db.delete('ail:current_background_script_stat')
|
|
||||||
r_serv_db.delete('ail:current_background_update')
|
|
||||||
r_serv_db.srem('ail:to_update', update_version)
|
|
||||||
|
|
|
@ -22,14 +22,11 @@ if __name__ == '__main__':
|
||||||
r_serv_onion = config_loader.get_redis_conn("ARDB_Onion")
|
r_serv_onion = config_loader.get_redis_conn("ARDB_Onion")
|
||||||
config_loader = None
|
config_loader = None
|
||||||
|
|
||||||
#Set current update_in_progress
|
|
||||||
r_serv.set('ail:update_in_progress', new_version)
|
|
||||||
r_serv.set('ail:current_background_update', new_version)
|
|
||||||
|
|
||||||
r_serv_onion.sunionstore('domain_update_v3.4', 'full_onion_up', 'full_regular_up')
|
r_serv_onion.sunionstore('domain_update_v3.4', 'full_onion_up', 'full_regular_up')
|
||||||
r_serv.set('update:nb_elem_to_convert', r_serv_onion.scard('domain_update_v3.4'))
|
r_serv.set('update:nb_elem_to_convert', r_serv_onion.scard('domain_update_v3.4'))
|
||||||
r_serv.set('update:nb_elem_converted',0)
|
r_serv.set('update:nb_elem_converted',0)
|
||||||
|
|
||||||
|
# Add background update
|
||||||
r_serv.sadd('ail:to_update', new_version)
|
r_serv.sadd('ail:to_update', new_version)
|
||||||
|
|
||||||
#Set current ail version
|
#Set current ail version
|
||||||
|
|
|
@ -31,6 +31,8 @@ if __name__ == '__main__':
|
||||||
r_serv_onion = config_loader.get_redis_conn("ARDB_Onion")
|
r_serv_onion = config_loader.get_redis_conn("ARDB_Onion")
|
||||||
config_loader = None
|
config_loader = None
|
||||||
|
|
||||||
|
r_serv.set('ail:current_background_script', 'domain languages update')
|
||||||
|
|
||||||
nb_elem_to_update = r_serv_db.get('update:nb_elem_to_convert')
|
nb_elem_to_update = r_serv_db.get('update:nb_elem_to_convert')
|
||||||
if not nb_elem_to_update:
|
if not nb_elem_to_update:
|
||||||
nb_elem_to_update = 1
|
nb_elem_to_update = 1
|
||||||
|
|
Loading…
Reference in New Issue