CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity
AIL-framework/bin/Update.py

474 lines
20 KiB
Python
Executable File
Raw Permalink Blame History

#!/usr/bin/env python3
# -*-coding:UTF-8 -*
"""
Update AIL
============================
Update AIL clone and fork
"""
import configparser
import os
import sys
import argparse
import subprocess
sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
# TODO: move other functions
from packages import git_status
UPDATER_FILENAME = os.path.join(os.environ['AIL_BIN'], 'Update.py')
UPDATER_LAST_MODIFICATION = float(os.stat(UPDATER_FILENAME).st_mtime)
def auto_update_enabled(cfg):
auto_update = cfg.get('Update', 'auto_update')
if auto_update == 'True' or auto_update == 'true':
return True
else:
return False
# check if files are modify locally
def check_if_files_modified():
# return True
process = subprocess.run(['git', 'ls-files' ,'-m'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if process.returncode == 0:
modified_files = process.stdout
if modified_files:
l_modified_files = []
for modified_file in modified_files.decode().split('\n'):
if modified_file:
if modified_file.split('/')[0] != 'configs':
l_modified_files.append(modified_file)
if l_modified_files:
print('Modified Files:')
for modified_file in l_modified_files:
print('{}{}{}'.format(TERMINAL_BLUE, modified_file, TERMINAL_DEFAULT))
print()
return False
else:
return True
else:
return True
else:
print('{}{}{}'.format(TERMINAL_RED, process.stderr.decode(), TERMINAL_DEFAULT))
sys.exit(1)
def repo_is_fork():
# return False
print('Check if this repository is a fork:')
process = subprocess.run(['git', 'remote', '-v'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if process.returncode == 0:
res = process.stdout.decode()
if 'origin {}'.format(AIL_REPO) in res:
print(' This repository is a {}clone of {}{}'.format(TERMINAL_BLUE, AIL_REPO, TERMINAL_DEFAULT))
return False
elif 'origin {}'.format(OLD_AIL_REPO) in res:
print(' old AIL repository, Updating remote origin...')
res = git_status.set_default_remote(AIL_REPO, verbose=False)
if res:
return False
else:
return True
else:
print(' This repository is a {}fork{}'.format(TERMINAL_BLUE, TERMINAL_DEFAULT))
print()
return True
else:
print('{}{}{}'.format(TERMINAL_RED, process.stderr.decode(), TERMINAL_DEFAULT))
aborting_update()
sys.exit(0)
def is_upstream_created(upstream):
process = subprocess.run(['git', 'remote', '-v'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if process.returncode == 0:
output = process.stdout.decode()
if upstream in output:
return True
else:
return False
else:
print('{}{}{}'.format(TERMINAL_RED, process.stderr.decode(), TERMINAL_DEFAULT))
aborting_update()
sys.exit(0)
def create_fork_upstream(upstream):
print('{}... Creating upstream ...{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT))
print('git remote add {} {}'.format(upstream, AIL_REPO))
process = subprocess.run(['git', 'remote', 'add', upstream, AIL_REPO],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if process.returncode == 0:
print(process.stdout.decode())
if is_upstream_created(upstream):
print('Fork upstream created')
print('{}... ...{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT))
else:
print('Fork not created')
aborting_update()
sys.exit(0)
else:
print('{}{}{}'.format(TERMINAL_RED, process.stderr.decode(), TERMINAL_DEFAULT))
aborting_update()
sys.exit(0)
def update_fork():
print('{}... Updating fork ...{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT))
if cfg.get('Update', 'update-fork') == 'True' or cfg.get('Update', 'update-fork') == 'true':
upstream = cfg.get('Update', 'upstream')
if not is_upstream_created(upstream):
create_fork_upstream(upstream)
print('{}git fetch {}:{}'.format(TERMINAL_YELLOW, upstream, TERMINAL_DEFAULT))
process = subprocess.run(['git', 'fetch', upstream], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if process.returncode == 0:
print(process.stdout.decode())
print('{}git checkout master:{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT))
process = subprocess.run(['git', 'checkout', 'master'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if process.returncode == 0:
print(process.stdout.decode())
print('{}git merge {}/master:{}'.format(TERMINAL_YELLOW, upstream, TERMINAL_DEFAULT))
process = subprocess.run(['git', 'merge', '{}/master'.format(upstream)],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if process.returncode == 0:
print(process.stdout.decode())
print('{}... ...{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT))
else:
print('{}{}{}'.format(TERMINAL_RED, process.stderr.decode(), TERMINAL_DEFAULT))
aborting_update()
sys.exit(1)
else:
print('{}{}{}'.format(TERMINAL_RED, process.stderr.decode(), TERMINAL_DEFAULT))
aborting_update()
sys.exit(0)
else:
print('{}{}{}'.format(TERMINAL_RED, process.stderr.decode(), TERMINAL_DEFAULT))
aborting_update()
sys.exit(0)
else:
print('{}Fork Auto-Update disabled in config file{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT))
aborting_update()
sys.exit(0)
def get_git_current_tag(path_current_version):
try:
with open(path_current_version, 'r') as version_content:
version = version_content.read()
except FileNotFoundError:
version = 'v5.0' # TODO Replace with VERSION.json
with open(path_current_version, 'w') as version_content:
version_content.write(version)
version = version.replace(" ", "").splitlines()[0]
if version[0] != 'v':
version = 'v{}'.format(version)
return version
def get_git_upper_tags_remote(current_tag, is_fork):
# keep only first dot
nb_dot = current_tag.count('.')
if nb_dot > 0:
nb_dot = nb_dot - 1
current_tag_val = current_tag.rsplit('.', nb_dot)
current_tag_val = ''.join(current_tag_val)
if is_fork:
process = subprocess.run(['git', 'tag'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if process.returncode == 0:
list_all_tags = process.stdout.decode().splitlines()
list_upper_tags = []
if list_all_tags[-1][1:] == current_tag:
list_upper_tags.append((list_all_tags[-1], None))
# force update order
list_upper_tags.sort()
return list_upper_tags
for tag in list_all_tags:
if float(tag[1:]) >= float(current_tag_val):
list_upper_tags.append((tag, None))
# force update order
list_upper_tags.sort()
return list_upper_tags
else:
print('{}{}{}'.format(TERMINAL_RED, process.stderr.decode(), TERMINAL_DEFAULT))
aborting_update()
sys.exit(0)
else:
process = subprocess.run(['git', 'ls-remote', '--tags'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if process.returncode == 0:
list_all_tags = process.stdout.decode().splitlines()
last_tag = list_all_tags[-1].split('\trefs/tags/')
last_commit = last_tag[0]
last_tag = last_tag[1].split('^{}')[0]
list_upper_tags = []
if last_tag[1:] == current_tag:
list_upper_tags.append((last_tag, last_commit))
# force update order
list_upper_tags.sort()
return list_upper_tags
else:
dict_tags_commit = {}
for mess_tag in list_all_tags:
commit, tag = mess_tag.split('\trefs/tags/')
tag = tag.replace('^{}', '')
# remove 'v' version
tag = tag.replace('v', '')
# keep only first dot
nb_dot = tag.count('.')
if nb_dot > 0:
nb_dot = nb_dot - 1
tag_val = tag.rsplit('.', nb_dot)
tag_val = ''.join(tag_val)
# check if tag is float
try:
tag_val = float(tag_val)
except ValueError:
continue
if float(current_tag) < 5.0:
# add tag with last commit
if float(current_tag_val) <= float(tag_val) < float(5.0):
dict_tags_commit[tag] = commit
else:
# add tag with last commit
if float(tag_val) >= float(current_tag_val):
dict_tags_commit[tag] = commit
list_upper_tags = [('v{}'.format(key), dict_tags_commit[key]) for key in dict_tags_commit]
# force update order
list_upper_tags.sort()
return list_upper_tags
else:
print('{}{}{}'.format(TERMINAL_RED, process.stderr.decode(), TERMINAL_DEFAULT))
aborting_update()
sys.exit(0)
def update_submodules():
print('{}git submodule update:{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT))
process = subprocess.run(['git', 'submodule', 'update'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if process.returncode == 0:
print(process.stdout.decode())
print()
else:
print('{}{}{}'.format(TERMINAL_RED, process.stderr.decode(), TERMINAL_DEFAULT))
def update_ail(current_tag, list_upper_tags_remote, current_version_path, is_fork):
print('{}git checkout master:{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT))
process = subprocess.run(['git', 'checkout', 'master'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
# process = subprocess.run(['ls'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if process.returncode == 0:
print(process.stdout.decode())
print()
update_submodules()
temp_current_tag = current_tag.replace('v', '')
if temp_current_tag.count('.') > 1:
temp_current_tag = temp_current_tag.rsplit('.', 1)
temp_current_tag = ''.join(temp_current_tag)
if float(temp_current_tag) < 5.0:
roll_back_update('2c65194b94dab95df9b8da19c88d65239f398355')
pulled = True
else:
print('{}git pull:{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT))
process = subprocess.run(['git', 'pull'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if process.returncode == 0:
output = process.stdout.decode()
print(output)
pulled = True
else:
print('{}{}{}'.format(TERMINAL_RED, process.stderr.decode(), TERMINAL_DEFAULT))
aborting_update()
pulled = False
sys.exit(1)
if pulled:
# CHECK IF UPDATER Update
if float(os.stat(UPDATER_FILENAME).st_mtime) > UPDATER_LAST_MODIFICATION:
# request updater relaunch
print(f'{TERMINAL_RED} Relaunch Launcher {TERMINAL_DEFAULT}')
sys.exit(3)
if len(list_upper_tags_remote) == 1:
# additional update (between 2 commits on the same version)
additional_update_path = os.path.join(os.environ['AIL_HOME'], 'update', current_tag, 'additional_update.sh')
if os.path.isfile(additional_update_path):
print()
print(f'{TERMINAL_YELLOW}------------------------------------------------------------------')
print('- Launching Additional Update: -')
print(f'-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --{TERMINAL_DEFAULT}')
process = subprocess.run(['bash', additional_update_path],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if process.returncode == 0:
output = process.stdout.decode()
print(output)
else:
print('{}{}{}'.format(TERMINAL_RED, process.stderr.decode(), TERMINAL_DEFAULT))
aborting_update()
sys.exit(1)
print()
print(f'{TERMINAL_YELLOW}**************** AIL Successfully Updated *****************{TERMINAL_DEFAULT}')
print()
exit(0)
else:
# map version with roll back commit
list_update = []
previous_commit = list_upper_tags_remote[0][1]
for row_tuple in list_upper_tags_remote[1:]:
tag = row_tuple[0]
list_update.append((tag, previous_commit))
previous_commit = row_tuple[1]
for update in list_update:
launch_update_version(update[0], update[1], current_version_path, is_fork)
# Success
print(f'{TERMINAL_YELLOW}**************** AIL Successfully Updated *****************{TERMINAL_DEFAULT}')
print()
sys.exit(0)
else:
print('{}{}{}'.format(TERMINAL_RED, process.stderr.decode(), TERMINAL_DEFAULT))
aborting_update()
sys.exit(0)
def launch_update_version(version, roll_back_commit, current_version_path, is_fork):
update_path = os.path.join(os.environ['AIL_HOME'], 'update', str(version), 'Update.sh')
print()
print(f'{TERMINAL_YELLOW}------------------------------------------------------------------')
print(f'- Launching Update: {TERMINAL_BLUE}{version}{TERMINAL_YELLOW} -')
print(f'-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --{TERMINAL_DEFAULT}')
if not os.path.isfile(update_path):
update_path = os.path.join(os.environ['AIL_HOME'], 'update', 'default_update', 'Update.sh')
process = subprocess.Popen(['bash', update_path, version], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
else:
process = subprocess.Popen(['bash', update_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
while True:
output = process.stdout.readline().decode()
if output == '' and process.poll() is not None:
break
if output:
print(output.strip())
if process.returncode == 0:
# output = process.stdout.decode()
# print(output)
with open(current_version_path, 'w') as version_content:
version_content.write(version)
print('{}-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --'.format(TERMINAL_YELLOW))
print('- Successfully Updated: {}{}{} -'.format(TERMINAL_BLUE, version, TERMINAL_YELLOW))
print('------------------------------------------------------------------{}'.format(TERMINAL_DEFAULT))
print()
else:
# print(process.stdout.read().decode())
print('{}{}{}'.format(TERMINAL_RED, process.stderr.read().decode(), TERMINAL_DEFAULT))
print('------------------------------------------------------------------')
print(' {}Update Error: {}{}{}'.format(TERMINAL_RED, TERMINAL_BLUE, version, TERMINAL_DEFAULT))
print('------------------------------------------------------------------')
if not is_fork:
roll_back_update(roll_back_commit)
else:
aborting_update()
sys.exit(1)
def roll_back_update(roll_back_commit):
print('Rolling back to safe commit: {}{}{}'.format(TERMINAL_BLUE, roll_back_commit, TERMINAL_DEFAULT))
process = subprocess.run(['git', 'checkout', roll_back_commit], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if process.returncode == 0:
output = process.stdout
print(output)
sys.exit(0)
else:
print(TERMINAL_RED+process.stderr.decode()+TERMINAL_DEFAULT)
aborting_update()
sys.exit(1)
def aborting_update():
print()
print('{}Aborting ...{}'.format(TERMINAL_RED, TERMINAL_DEFAULT))
print('{}******************************************************************'.format(TERMINAL_RED))
print('* AIL Not Updated *')
print('******************************************************************{}'.format(TERMINAL_DEFAULT))
print()
if __name__ == "__main__":
TERMINAL_RED = '\033[91m'
TERMINAL_YELLOW = '\33[93m'
TERMINAL_BLUE = '\33[94m'
TERMINAL_BLINK = '\33[6m'
TERMINAL_DEFAULT = '\033[0m'
AIL_REPO = 'https://github.com/ail-project/ail-framework'
OLD_AIL_REPO = 'https://github.com/CIRCL/AIL-framework.git'
configfile = os.path.join(os.environ['AIL_HOME'], 'configs/update.cfg')
if not os.path.exists(configfile):
raise Exception('Unable to find the configuration file. \
Did you set environment variables? \
Or activate the virtualenv.')
cfg = configparser.ConfigParser()
cfg.read(configfile)
current_version_path = os.path.join(os.environ['AIL_HOME'], 'update/current_version')
print('{}******************************************************************'.format(TERMINAL_YELLOW))
print('* Updating AIL ... *')
print('******************************************************************{}'.format(TERMINAL_DEFAULT))
# manual updates
parser = argparse.ArgumentParser()
parser.add_argument("--manual", nargs='?', const=True, default=False)
args = parser.parse_args()
manual_update = args.manual
if auto_update_enabled(cfg) or manual_update:
if check_if_files_modified():
is_fork = repo_is_fork()
if is_fork:
update_fork()
current_tag = get_git_current_tag(current_version_path)
print()
print('Current Version: {}{}{}'.format(TERMINAL_YELLOW, current_tag, TERMINAL_DEFAULT))
print()
list_upper_tags_remote = get_git_upper_tags_remote(current_tag.replace('v', ''), is_fork)
# new release
if len(list_upper_tags_remote) > 1:
print('New Releases:')
if is_fork:
for upper_tag in list_upper_tags_remote:
print(' {}{}{}'.format(TERMINAL_BLUE, upper_tag[0], TERMINAL_DEFAULT))
else:
for upper_tag in list_upper_tags_remote:
print(' {}{}{}: {}'.format(TERMINAL_BLUE, upper_tag[0], TERMINAL_DEFAULT, upper_tag[1]))
print()
update_ail(current_tag, list_upper_tags_remote, current_version_path, is_fork)
else:
print('Please, commit your changes or stash them before you can update AIL')
aborting_update()
sys.exit(0)
else:
print(' {}AIL Auto update is disabled{}'.format(TERMINAL_RED, TERMINAL_DEFAULT))
aborting_update()
sys.exit(0)
# r = get_git_upper_tags_remote('4.2.1', False)
# print(r)
Reference in New Issue View Git Blame Copy Permalink