From 60ff0b9cf7a62886640cff44c827151927c76e05 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Mon, 12 Nov 2018 17:10:31 +0100 Subject: [PATCH 01/16] chg: [Update] add update script --- bin/Update.py | 297 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 297 insertions(+) create mode 100755 bin/Update.py diff --git a/bin/Update.py b/bin/Update.py new file mode 100755 index 00000000..73bec058 --- /dev/null +++ b/bin/Update.py @@ -0,0 +1,297 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import configparser +import os +import sys + +import subprocess + +def auto_update_enabled(cfg): + auto_update = cfg.get('Update', 'auto_update') + if auto_update == 'True' or auto_update == 'true': + return True + else: + return False + +# check if files are modify locally +def check_if_files_modified(): + process = subprocess.run(['git', 'ls-files' ,'-m'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + if process.returncode == 0: + modified_files = process.stdout + if modified_files: + return False + else: + return True + else: + print(TERMINAL_RED+process.stderr.decode()+TERMINAL_DEFAULT) + return False + +def repo_is_fork(): + process = subprocess.run(['git', 'ls-remote', '--tags'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + if process.returncode == 0: + # remove url origin + local_remote = process.stdout + process = subprocess.run(['git', 'ls-remote' ,'--tags', AIL_REPO], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + if process.returncode == 0: + ail_remote = process.stdout + print(local_remote) + print(ail_remote) + if local_remote == ail_remote: + return False + else: + return True + else: + print(TERMINAL_RED+process.stderr.decode()+TERMINAL_DEFAULT) + return False + +def is_upstream_created(upstream): + process = subprocess.run(['git', 'remote', '-v'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if process.returncode == 0: + output = process.stdout.decode() + if upstream in output: + return True + else: + return False + else: + print(process.stderr.decode()) + return None + +def create_fork_upstream(upstream): + process = subprocess.run(['git', 'remote', 'add', upstream, AIL_REPO], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if process.returncode == 0: + print(process.stdout.decode()) + if is_upstream_created(): + print('fork created') + else: + print('error, fork not created') + else: + print(process.stderr.decode()) + return None + +def update_fork(): + if cfg.get('Update', 'update-fork') == 'True' or cfg.get('Update', 'update-fork') == 'true': + upstream = cfg.get('Update', 'upstream') + if not is_upstream_created(upstream): + create_fork_upstream(upstream) + process = subprocess.run(['git', 'fetch', upstream], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if process.returncode == 0: + print(process.stdout.decode()) + process = subprocess.run(['git', 'checkout', 'master'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if process.returncode == 0: + print(process.stdout.decode()) + process = subprocess.run(['git', 'merge', '{}/master'.format(upstream)], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if process.returncode == 0: + print(process.stdout.decode()) + else: + print(process.stderr.decode()) + return None + else: + print(process.stderr.decode()) + return None + else: + print(process.stderr.decode()) + return None + + else: + print('auto update fork disabled, you can active it in ...') + + +def get_git_current_tag(current_version_path): + with open(current_version_path, 'r') as version_content: + version = version_content.read() + version = version.replace(" ", "").splitlines() + return version[0] + + ''' + process = subprocess.run(['git', 'describe' ,'--tags'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + if process.returncode == 0: + current_tag = process.stdout + current_tag = current_tag.split(b'-')[0] + return current_tag.decode() + else: + print(process.stderr.decode()) + return None + ''' + +def get_git_upper_tags_remote(current_tag): + process = subprocess.run(['git', 'ls-remote' ,'--tags'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + if process.returncode == 0: + list_all_tags = process.stdout.decode().splitlines() + list_all_tags.append('aaaaaaaaaaaaaaaaaaaaaaaaaaaaa\trefs/tags/v1.5') + list_all_tags.append('eeeeeeeeeeeeeeeeeeeeeeeeeeee\trefs/tags/v1.5^{}') + list_all_tags.append('bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\trefs/tags/v1.6') + list_all_tags.append('bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\trefs/tags/v1.6^{}') + list_all_tags.append('zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz\trefs/tags/v1.7') + last_tag = list_all_tags[-1].split('\trefs/tags/') + last_commit = last_tag[0] + last_tag = last_tag[1].split('^{}')[0] + list_upper_tags = [] + if last_tag[1:] == current_tag: + list_upper_tags.append( (last_tag, last_commit) ) + return list_upper_tags + else: + for mess_tag in list_all_tags: + commit, tag = mess_tag.split('\trefs/tags/') + + # add tag with last commit + if float(tag.split('^{}')[0][1:]) >= float(current_tag): + if '^{}' in tag: + list_upper_tags.append( (tag.split('^{}')[0], commit) ) + # add last commit + if last_tag not in list_upper_tags[-1][0]: + list_upper_tags.append( (last_tag, last_commit) ) + return list_upper_tags + + else: + print(TERMINAL_RED+process.stderr.decode()+TERMINAL_DEFAULT) + return None + +def update_ail(current_tag, list_upper_tags_remote, current_version_path): + print('git checkout master:') + process = subprocess.run(['git', 'checkout', 'master'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + if process.returncode == 0: + print('git pull:') + process = subprocess.run(['git', 'pull'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + if process.returncode == 0: + output = process.stdout.decode() + print(output) + + if len(list_upper_tags_remote) == 1: + print('AIL Updated') + # # FIXME: # TODO: exit sucess + + else: + # map version with roll back commit + list_update = [] + previous_commit = list_upper_tags_remote[0][1] + for tuple in list_upper_tags_remote[1:]: + tag = tuple[0] + list_update.append( (tag, previous_commit) ) + previous_commit = tuple[1] + print(list_update) + + for update in list_update: + launch_update_version(update[0], update[1], current_version_path, is_fork) + else: + print(TERMINAL_RED+process.stderr.decode()+TERMINAL_DEFAULT) + return None + else: + print(TERMINAL_RED+process.stderr.decode()+TERMINAL_DEFAULT) + return None + +def launch_update_version(version, roll_back_commit, current_version_path, is_fork): + update_path = os.path.join(os.environ['AIL_HOME'], 'update', version, 'Update.sh') + print('------------------------------------------------------------------') + print('- Launching Update: {} -'.format(version)) + print('------------------------------------------------------------------') + process = subprocess.run(['bash', update_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if process.returncode == 0: + output = process.stdout + print(output) + + with open(current_version_path, 'w') as version_content: + version_content.write(version) + + else: + print(TERMINAL_RED+process.stderr.decode()+TERMINAL_DEFAULT) + if not is_fork: + roll_back_update(roll_back_commit) + +def roll_back_update(roll_back_commit): + process = subprocess.run(['git', 'checkout', roll_back_commit], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if process.returncode == 0: + output = process.stdout + print(output) + sys.exit() + else: + print(TERMINAL_RED+process.stderr.decode()+TERMINAL_DEFAULT) + sys.exit(1) + +''' + + if len(sys.argv) != 2: + print('usage:', 'Update-conf.py', 'Automatic (boolean)') + exit(1) + else: + automatic = sys.argv[1] + if automatic == 'True': + automatic = True + else: + automatic = False + + + if automatic: + resp = 'y' + else: + resp = input("Do you want to auto fix it? [y/n] ") + + if resp != 'y': + return False + else: + if automatic: + resp2 = 'y' + else: + resp2 = input("Do you want to keep a backup of the old configuration file? [y/n] ") +''' + +if __name__ == "__main__": + + TERMINAL_RED = '\033[91m' + TERMINAL_YELLOW = '\33[93m' + TERMINAL_DEFAULT = '\033[0m' + + AIL_REPO = 'https://github.com/CIRCL/AIL-framework.git' + + configfile = os.path.join(os.environ['AIL_HOME'], 'configs/update.cfg') + if not os.path.exists(configfile): + raise Exception('Unable to find the configuration file. \ + Did you set environment variables? \ + Or activate the virtualenv.') + cfg = configparser.ConfigParser() + cfg.read(configfile) + + current_version_path = os.path.join(os.environ['AIL_HOME'], 'update/current_version') + + print('******************************************************************') + print('* Updating AIL ... *') + print('******************************************************************') + + if auto_update_enabled(cfg): + if check_if_files_modified(): + is_fork = repo_is_fork() + if is_fork: + update_fork() + + current_tag = get_git_current_tag(current_version_path) + print('Current Version: {}'.format(current_tag)) + print() + list_upper_tags_remote = get_git_upper_tags_remote(current_tag[1:]) + # new realease + if len(list_upper_tags_remote) > 1: + print('New Releases:') + for upper_tag in list_upper_tags_remote: + print(' {}{}{}: {}'.format(TERMINAL_YELLOW, upper_tag[0], TERMINAL_DEFAULT, upper_tag[1])) + print() + update_ail(current_tag, list_upper_tags_remote, current_version_path, is_fork) + #else: + # print('your fork is outdated') + else: + print('please commit your change') + else: + print(' AIL Auto update is disabled') + print(' AIL not Updated') + print('******************************************************************') + ''' + if main(): + sys.exit() + else: + sys.exit(1) + ''' From 7aff45c50764406003e8dcea58f4cbb9160a4680 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 13 Nov 2018 14:54:56 +0100 Subject: [PATCH 02/16] chg [Update] update AIL clone and fork --- bin/Update.py | 291 +++++++++++++++++++++++++++++--------------------- 1 file changed, 170 insertions(+), 121 deletions(-) diff --git a/bin/Update.py b/bin/Update.py index 73bec058..d525d228 100755 --- a/bin/Update.py +++ b/bin/Update.py @@ -1,6 +1,14 @@ #!/usr/bin/env python3 # -*-coding:UTF-8 -* +""" +Update AIL +============================ + +Update AIL clone and fork + +""" + import configparser import os import sys @@ -21,14 +29,18 @@ def check_if_files_modified(): if process.returncode == 0: modified_files = process.stdout if modified_files: + print('Modified Files:') + print('{}{}{}'.format(TERMINAL_BLUE, modified_files.decode(), TERMINAL_DEFAULT)) return False + #return True else: return True else: - print(TERMINAL_RED+process.stderr.decode()+TERMINAL_DEFAULT) - return False + print('{}{}{}'.format(TERMINAL_RED, process.stderr.decode(), TERMINAL_DEFAULT)) + sys.exit(1) def repo_is_fork(): + print('Check if this repository is a fork:') process = subprocess.run(['git', 'ls-remote', '--tags'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) if process.returncode == 0: @@ -38,15 +50,21 @@ def repo_is_fork(): if process.returncode == 0: ail_remote = process.stdout - print(local_remote) - print(ail_remote) if local_remote == ail_remote: + print(' This repository is a {}clone of {}{}'.format(TERMINAL_BLUE, AIL_REPO, TERMINAL_DEFAULT)) return False else: + print(' This repository is a {}fork{}'.format(TERMINAL_BLUE, TERMINAL_DEFAULT)) + print() return True + else: + print('{}{}{}'.format(TERMINAL_RED, process.stderr.decode(), TERMINAL_DEFAULT)) + aborting_update() + sys.exit(0) else: - print(TERMINAL_RED+process.stderr.decode()+TERMINAL_DEFAULT) - return False + print('{}{}{}'.format(TERMINAL_RED, process.stderr.decode(), TERMINAL_DEFAULT)) + aborting_update() + sys.exit(0) def is_upstream_created(upstream): process = subprocess.run(['git', 'remote', '-v'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -57,47 +75,64 @@ def is_upstream_created(upstream): else: return False else: - print(process.stderr.decode()) - return None + print('{}{}{}'.format(TERMINAL_RED, process.stderr.decode(), TERMINAL_DEFAULT)) + aborting_update() + sys.exit(0) def create_fork_upstream(upstream): + print('{}... Creating upstream ...{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT)) + print('git remote add {} {}'.format(upstream, AIL_REPO)) process = subprocess.run(['git', 'remote', 'add', upstream, AIL_REPO], stdout=subprocess.PIPE, stderr=subprocess.PIPE) if process.returncode == 0: print(process.stdout.decode()) - if is_upstream_created(): - print('fork created') + if is_upstream_created(upstream): + print('Fork upstream created') + print('{}... ...{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT)) else: - print('error, fork not created') + print('Fork not created') + aborting_update() + sys.exit(0) else: - print(process.stderr.decode()) - return None + print('{}{}{}'.format(TERMINAL_RED, process.stderr.decode(), TERMINAL_DEFAULT)) + aborting_update() + sys.exit(0) def update_fork(): + print('{}... Updating fork ...{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT)) if cfg.get('Update', 'update-fork') == 'True' or cfg.get('Update', 'update-fork') == 'true': upstream = cfg.get('Update', 'upstream') if not is_upstream_created(upstream): create_fork_upstream(upstream) + print('{}git fetch {}:{}'.format(TERMINAL_YELLOW, upstream, TERMINAL_DEFAULT)) process = subprocess.run(['git', 'fetch', upstream], stdout=subprocess.PIPE, stderr=subprocess.PIPE) if process.returncode == 0: print(process.stdout.decode()) + print('{}git checkout master:{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT)) process = subprocess.run(['git', 'checkout', 'master'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) if process.returncode == 0: print(process.stdout.decode()) + print('{}git merge {}/master:{}'.format(TERMINAL_YELLOW, upstream, TERMINAL_DEFAULT)) process = subprocess.run(['git', 'merge', '{}/master'.format(upstream)], stdout=subprocess.PIPE, stderr=subprocess.PIPE) if process.returncode == 0: print(process.stdout.decode()) + print('{}... ...{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT)) else: - print(process.stderr.decode()) - return None + print('{}{}{}'.format(TERMINAL_RED, process.stderr.decode(), TERMINAL_DEFAULT)) + aborting_update() + sys.exit(1) else: - print(process.stderr.decode()) - return None + print('{}{}{}'.format(TERMINAL_RED, process.stderr.decode(), TERMINAL_DEFAULT)) + aborting_update() + sys.exit(0) else: - print(process.stderr.decode()) - return None + print('{}{}{}'.format(TERMINAL_RED, process.stderr.decode(), TERMINAL_DEFAULT)) + aborting_update() + sys.exit(0) else: - print('auto update fork disabled, you can active it in ...') + print('{}Fork Auto-Update disabled in config file{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT)) + aborting_update() + sys.exit(0) def get_git_current_tag(current_version_path): @@ -106,58 +141,68 @@ def get_git_current_tag(current_version_path): version = version.replace(" ", "").splitlines() return version[0] - ''' - process = subprocess.run(['git', 'describe' ,'--tags'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) +def get_git_upper_tags_remote(current_tag, is_fork): + if is_fork: + process = subprocess.run(['git', 'tag'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if process.returncode == 0: + list_all_tags = process.stdout.decode().splitlines() - if process.returncode == 0: - current_tag = process.stdout - current_tag = current_tag.split(b'-')[0] - return current_tag.decode() - else: - print(process.stderr.decode()) - return None - ''' - -def get_git_upper_tags_remote(current_tag): - process = subprocess.run(['git', 'ls-remote' ,'--tags'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - - if process.returncode == 0: - list_all_tags = process.stdout.decode().splitlines() - list_all_tags.append('aaaaaaaaaaaaaaaaaaaaaaaaaaaaa\trefs/tags/v1.5') - list_all_tags.append('eeeeeeeeeeeeeeeeeeeeeeeeeeee\trefs/tags/v1.5^{}') - list_all_tags.append('bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\trefs/tags/v1.6') - list_all_tags.append('bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\trefs/tags/v1.6^{}') - list_all_tags.append('zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz\trefs/tags/v1.7') - last_tag = list_all_tags[-1].split('\trefs/tags/') - last_commit = last_tag[0] - last_tag = last_tag[1].split('^{}')[0] - list_upper_tags = [] - if last_tag[1:] == current_tag: - list_upper_tags.append( (last_tag, last_commit) ) + list_upper_tags = [] + if list_all_tags[-1][1:] == current_tag: + list_upper_tags.append( (list_all_tags[-1], None) ) + return list_upper_tags + for tag in list_all_tags: + if float(tag[1:]) >= float(current_tag): + list_upper_tags.append( (tag, None) ) return list_upper_tags else: - for mess_tag in list_all_tags: - commit, tag = mess_tag.split('\trefs/tags/') - - # add tag with last commit - if float(tag.split('^{}')[0][1:]) >= float(current_tag): - if '^{}' in tag: - list_upper_tags.append( (tag.split('^{}')[0], commit) ) - # add last commit - if last_tag not in list_upper_tags[-1][0]: - list_upper_tags.append( (last_tag, last_commit) ) - return list_upper_tags - + print('{}{}{}'.format(TERMINAL_RED, process.stderr.decode(), TERMINAL_DEFAULT)) + aborting_update() + sys.exit(0) else: - print(TERMINAL_RED+process.stderr.decode()+TERMINAL_DEFAULT) - return None + process = subprocess.run(['git', 'ls-remote' ,'--tags'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) -def update_ail(current_tag, list_upper_tags_remote, current_version_path): - print('git checkout master:') + if process.returncode == 0: + list_all_tags = process.stdout.decode().splitlines() + list_all_tags.append('aaaaaaaaaaaaaaaaaaaaaaaaaaaaa\trefs/tags/v1.5') + list_all_tags.append('eeeeeeeeeeeeeeeeeeeeeeeeeeee\trefs/tags/v1.5^{}') + list_all_tags.append('bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\trefs/tags/v1.6') + list_all_tags.append('bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\trefs/tags/v1.6^{}') + #list_all_tags.append('zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz\trefs/tags/v1.7') + last_tag = list_all_tags[-1].split('\trefs/tags/') + last_commit = last_tag[0] + last_tag = last_tag[1].split('^{}')[0] + list_upper_tags = [] + if last_tag[1:] == current_tag: + list_upper_tags.append( (last_tag, last_commit) ) + return list_upper_tags + else: + for mess_tag in list_all_tags: + commit, tag = mess_tag.split('\trefs/tags/') + + # add tag with last commit + if float(tag.split('^{}')[0][1:]) >= float(current_tag): + if '^{}' in tag: + list_upper_tags.append( (tag.split('^{}')[0], commit) ) + # add last commit + if last_tag not in list_upper_tags[-1][0]: + list_upper_tags.append( (last_tag, last_commit) ) + return list_upper_tags + + else: + print('{}{}{}'.format(TERMINAL_RED, process.stderr.decode(), TERMINAL_DEFAULT)) + aborting_update() + sys.exit(0) + +def update_ail(current_tag, list_upper_tags_remote, current_version_path, is_fork): + print('{}git checkout master:{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT)) process = subprocess.run(['git', 'checkout', 'master'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + #process = subprocess.run(['git', 'status'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) if process.returncode == 0: - print('git pull:') + print(process.stdout.decode()) + print() + print('{}git pull:{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT)) process = subprocess.run(['git', 'pull'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) if process.returncode == 0: @@ -165,8 +210,10 @@ def update_ail(current_tag, list_upper_tags_remote, current_version_path): print(output) if len(list_upper_tags_remote) == 1: - print('AIL Updated') - # # FIXME: # TODO: exit sucess + print() + print('{}**************** AIL Sucessfully Updated *****************{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT)) + print() + exit(0) else: # map version with roll back commit @@ -176,76 +223,78 @@ def update_ail(current_tag, list_upper_tags_remote, current_version_path): tag = tuple[0] list_update.append( (tag, previous_commit) ) previous_commit = tuple[1] - print(list_update) for update in list_update: launch_update_version(update[0], update[1], current_version_path, is_fork) + # Sucess + print('{}**************** AIL Sucessfully Updated *****************{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT)) + print() + sys.exit(0) else: - print(TERMINAL_RED+process.stderr.decode()+TERMINAL_DEFAULT) - return None + print('{}{}{}'.format(TERMINAL_RED, process.stderr.decode(), TERMINAL_DEFAULT)) + aborting_update() + sys.exit(1) else: - print(TERMINAL_RED+process.stderr.decode()+TERMINAL_DEFAULT) - return None + print('{}{}{}'.format(TERMINAL_RED, process.stderr.decode(), TERMINAL_DEFAULT)) + aborting_update() + sys.exit(0) def launch_update_version(version, roll_back_commit, current_version_path, is_fork): update_path = os.path.join(os.environ['AIL_HOME'], 'update', version, 'Update.sh') - print('------------------------------------------------------------------') - print('- Launching Update: {} -'.format(version)) - print('------------------------------------------------------------------') + print() + print('{}------------------------------------------------------------------'.format(TERMINAL_YELLOW)) + print('- Launching Update: {}{}{} -'.format(TERMINAL_BLUE, version, TERMINAL_YELLOW)) + print('-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --{}'.format(TERMINAL_DEFAULT)) process = subprocess.run(['bash', update_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE) if process.returncode == 0: - output = process.stdout + output = process.stdout.decode() print(output) with open(current_version_path, 'w') as version_content: version_content.write(version) + print('{}-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --'.format(TERMINAL_YELLOW)) + print('- Sucessfully Updated: {}{}{} -'.format(TERMINAL_BLUE, version, TERMINAL_YELLOW)) + print('------------------------------------------------------------------{}'.format(TERMINAL_DEFAULT)) + print() else: - print(TERMINAL_RED+process.stderr.decode()+TERMINAL_DEFAULT) + print(process.stdout.decode()) + print('{}{}{}'.format(TERMINAL_RED, process.stderr.decode(), TERMINAL_DEFAULT)) + print('------------------------------------------------------------------') + print(' {}Update Error: {}{}{}'.format(TERMINAL_RED, TERMINAL_BLUE, version, TERMINAL_DEFAULT)) + print('------------------------------------------------------------------') if not is_fork: roll_back_update(roll_back_commit) + else: + aborting_update() + sys.exit(1) def roll_back_update(roll_back_commit): + print('Rolling back to safe commit: {}{}{}'.format(TERMINAL_BLUE ,roll_back_commit, TERMINAL_DEFAULT)) process = subprocess.run(['git', 'checkout', roll_back_commit], stdout=subprocess.PIPE, stderr=subprocess.PIPE) if process.returncode == 0: output = process.stdout print(output) - sys.exit() + sys.exit(0) else: print(TERMINAL_RED+process.stderr.decode()+TERMINAL_DEFAULT) + aborting_update() sys.exit(1) -''' - - if len(sys.argv) != 2: - print('usage:', 'Update-conf.py', 'Automatic (boolean)') - exit(1) - else: - automatic = sys.argv[1] - if automatic == 'True': - automatic = True - else: - automatic = False - - - if automatic: - resp = 'y' - else: - resp = input("Do you want to auto fix it? [y/n] ") - - if resp != 'y': - return False - else: - if automatic: - resp2 = 'y' - else: - resp2 = input("Do you want to keep a backup of the old configuration file? [y/n] ") -''' +def aborting_update(): + print() + print('{}Aborting ...{}'.format(TERMINAL_RED, TERMINAL_DEFAULT)) + print('{}******************************************************************'.format(TERMINAL_RED)) + print('* AIL Not Updated *') + print('******************************************************************{}'.format(TERMINAL_DEFAULT)) + print() if __name__ == "__main__": TERMINAL_RED = '\033[91m' TERMINAL_YELLOW = '\33[93m' + TERMINAL_BLUE = '\33[94m' + TERMINAL_BLINK = '\33[6m' TERMINAL_DEFAULT = '\033[0m' AIL_REPO = 'https://github.com/CIRCL/AIL-framework.git' @@ -260,9 +309,9 @@ if __name__ == "__main__": current_version_path = os.path.join(os.environ['AIL_HOME'], 'update/current_version') - print('******************************************************************') + print('{}******************************************************************'.format(TERMINAL_YELLOW)) print('* Updating AIL ... *') - print('******************************************************************') + print('******************************************************************{}'.format(TERMINAL_DEFAULT)) if auto_update_enabled(cfg): if check_if_files_modified(): @@ -271,27 +320,27 @@ if __name__ == "__main__": update_fork() current_tag = get_git_current_tag(current_version_path) - print('Current Version: {}'.format(current_tag)) print() - list_upper_tags_remote = get_git_upper_tags_remote(current_tag[1:]) + print('Current Version: {}{}{}'.format( TERMINAL_YELLOW, current_tag, TERMINAL_DEFAULT)) + print() + list_upper_tags_remote = get_git_upper_tags_remote(current_tag[1:], is_fork) # new realease if len(list_upper_tags_remote) > 1: print('New Releases:') - for upper_tag in list_upper_tags_remote: - print(' {}{}{}: {}'.format(TERMINAL_YELLOW, upper_tag[0], TERMINAL_DEFAULT, upper_tag[1])) + if is_fork: + for upper_tag in list_upper_tags_remote: + print(' {}{}{}'.format(TERMINAL_BLUE, upper_tag[0], TERMINAL_DEFAULT)) + else: + for upper_tag in list_upper_tags_remote: + print(' {}{}{}: {}'.format(TERMINAL_BLUE, upper_tag[0], TERMINAL_DEFAULT, upper_tag[1])) print() update_ail(current_tag, list_upper_tags_remote, current_version_path, is_fork) - #else: - # print('your fork is outdated') + else: - print('please commit your change') + print('Please, commit your changes or stash them before you can update AIL') + aborting_update() + sys.exit(0) else: - print(' AIL Auto update is disabled') - print(' AIL not Updated') - print('******************************************************************') - ''' - if main(): - sys.exit() - else: - sys.exit(1) - ''' + print(' {}AIL Auto update is disabled{}'.format(TERMINAL_RED, TERMINAL_DEFAULT)) + aborting_update() + sys.exit(0) From 347986a2718a7f0bb3fac526c093ceb5c374a060 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 13 Nov 2018 15:45:22 +0100 Subject: [PATCH 03/16] chg: [LAUNCH] add AIL update by default --- bin/LAUNCH.sh | 18 +++++++++++++++++- configs/update.cfg | 4 ++++ update/current_version | 1 + update/v1.5/Update.sh | 4 ++++ 4 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 configs/update.cfg create mode 100644 update/current_version create mode 100755 update/v1.5/Update.sh diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index 684af83b..58497e01 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -382,6 +382,16 @@ function shutdown { bash -c "./Shutdown.py" } +function update() { + bash -c "./Update.py" + + exitStatus=$? + if [ $exitStatus -ge 1 ]; then + echo -e $RED"\t* Update Error"$DEFAULT + exit + fi +} + function update_thirdparty { echo -e "\t* Updating thirdparty..." bash -c "(cd ${AIL_FLASK}; ./update_thirdparty.sh)" @@ -395,6 +405,7 @@ function update_thirdparty { } function launch_all { + update; launch_redis; launch_ardb; launch_logs; @@ -408,7 +419,7 @@ function launch_all { helptext; - options=("Redis" "Ardb" "Logs" "Queues" "Scripts" "Flask" "Killall" "Shutdown" "Update-config" "Update-thirdparty") + options=("Redis" "Ardb" "Logs" "Queues" "Scripts" "Flask" "Killall" "Shutdown" "Update" "Update-config" "Update-thirdparty") menu() { echo "What do you want to Launch?:" @@ -459,6 +470,9 @@ function launch_all { Shutdown) shutdown; ;; + Update) + update; + ;; Update-config) checking_configuration "manual"; ;; @@ -478,6 +492,8 @@ while [ "$1" != "" ]; do ;; -k | --killAll ) killall; ;; + -u | --update ) update; + ;; -t | --thirdpartyUpdate ) update_thirdparty; ;; -c | --crawler ) launching_crawler; diff --git a/configs/update.cfg b/configs/update.cfg new file mode 100644 index 00000000..a72d8bc0 --- /dev/null +++ b/configs/update.cfg @@ -0,0 +1,4 @@ +[Update] +auto_update = True +upstream = upstream +update-fork = False diff --git a/update/current_version b/update/current_version new file mode 100644 index 00000000..b1f74215 --- /dev/null +++ b/update/current_version @@ -0,0 +1 @@ +v1.4 diff --git a/update/v1.5/Update.sh b/update/v1.5/Update.sh new file mode 100755 index 00000000..e69e3b31 --- /dev/null +++ b/update/v1.5/Update.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +echo $AIL_HOME + From 94fcf66d2027380938990bca55f3f3d86d5f5d14 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 13 Nov 2018 15:52:33 +0100 Subject: [PATCH 04/16] fix: [Update] cleaning --- bin/Update.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/bin/Update.py b/bin/Update.py index d525d228..fe29c936 100755 --- a/bin/Update.py +++ b/bin/Update.py @@ -32,7 +32,6 @@ def check_if_files_modified(): print('Modified Files:') print('{}{}{}'.format(TERMINAL_BLUE, modified_files.decode(), TERMINAL_DEFAULT)) return False - #return True else: return True else: @@ -164,11 +163,6 @@ def get_git_upper_tags_remote(current_tag, is_fork): if process.returncode == 0: list_all_tags = process.stdout.decode().splitlines() - list_all_tags.append('aaaaaaaaaaaaaaaaaaaaaaaaaaaaa\trefs/tags/v1.5') - list_all_tags.append('eeeeeeeeeeeeeeeeeeeeeeeeeeee\trefs/tags/v1.5^{}') - list_all_tags.append('bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\trefs/tags/v1.6') - list_all_tags.append('bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\trefs/tags/v1.6^{}') - #list_all_tags.append('zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz\trefs/tags/v1.7') last_tag = list_all_tags[-1].split('\trefs/tags/') last_commit = last_tag[0] last_tag = last_tag[1].split('^{}')[0] @@ -197,7 +191,6 @@ def get_git_upper_tags_remote(current_tag, is_fork): def update_ail(current_tag, list_upper_tags_remote, current_version_path, is_fork): print('{}git checkout master:{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT)) process = subprocess.run(['git', 'checkout', 'master'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - #process = subprocess.run(['git', 'status'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) if process.returncode == 0: print(process.stdout.decode()) From 912b977bb4809a818282bfdfd98ea724cf7e18fd Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 13 Nov 2018 16:54:39 +0100 Subject: [PATCH 05/16] chg: [LAUNCH] update: launch + check BDD --- bin/LAUNCH.sh | 45 ++++++--------------------------------------- bin/check_ardb.sh | 27 +++++++++++++++++++++++++++ bin/check_redis.sh | 39 +++++++++++++++++++++++++++++++++++++++ bin/launch_ardb.sh | 21 +++++++++++++++++++++ bin/launch_lvldb.sh | 29 ----------------------------- bin/launch_redis.sh | 24 ++++++++++++------------ 6 files changed, 105 insertions(+), 80 deletions(-) create mode 100755 bin/check_ardb.sh create mode 100755 bin/check_redis.sh create mode 100755 bin/launch_ardb.sh delete mode 100755 bin/launch_lvldb.sh diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index 58497e01..7e6fdfc8 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -64,27 +64,11 @@ function helptext { } function launching_redis { - conf_dir="${AIL_HOME}/configs/" - - screen -dmS "Redis_AIL" - sleep 0.1 - echo -e $GREEN"\t* Launching Redis servers"$DEFAULT - screen -S "Redis_AIL" -X screen -t "6379" bash -c 'redis-server '$conf_dir'6379.conf ; read x' - sleep 0.1 - screen -S "Redis_AIL" -X screen -t "6380" bash -c 'redis-server '$conf_dir'6380.conf ; read x' - sleep 0.1 - screen -S "Redis_AIL" -X screen -t "6381" bash -c 'redis-server '$conf_dir'6381.conf ; read x' + bash -c "bash ${AIL_BIN}/launch_redis.sh" } function launching_ardb { - conf_dir="${AIL_HOME}/configs/" - - screen -dmS "ARDB_AIL" - sleep 0.1 - echo -e $GREEN"\t* Launching ARDB servers"$DEFAULT - - sleep 0.1 - screen -S "ARDB_AIL" -X screen -t "6382" bash -c 'cd '${AIL_HOME}'; ardb-server '$conf_dir'6382.conf ; read x' + bash -c "bash ${AIL_BIN}/launch_ardb.sh" } function launching_logs { @@ -245,36 +229,18 @@ function shutting_down_ardb { function checking_redis { flag_redis=0 - redis_dir=${AIL_HOME}/redis/src/ - bash -c $redis_dir'redis-cli -p 6379 PING | grep "PONG" &> /dev/null' + bash -c "bash ${AIL_BIN}/check_redis.sh" if [ ! $? == 0 ]; then - echo -e $RED"\t6379 not ready"$DEFAULT flag_redis=1 fi - sleep 0.1 - bash -c $redis_dir'redis-cli -p 6380 PING | grep "PONG" &> /dev/null' - if [ ! $? == 0 ]; then - echo -e $RED"\t6380 not ready"$DEFAULT - flag_redis=1 - fi - sleep 0.1 - bash -c $redis_dir'redis-cli -p 6381 PING | grep "PONG" &> /dev/null' - if [ ! $? == 0 ]; then - echo -e $RED"\t6381 not ready"$DEFAULT - flag_redis=1 - fi - sleep 0.1 return $flag_redis; } function checking_ardb { flag_ardb=0 - redis_dir=${AIL_HOME}/redis/src/ - sleep 0.2 - bash -c $redis_dir'redis-cli -p 6382 PING | grep "PONG" &> /dev/null' + bash -c "bash ${AIL_BIN}/check_ardb.sh" if [ ! $? == 0 ]; then - echo -e $RED"\t6382 ARDB not ready"$DEFAULT flag_ardb=1 fi @@ -383,8 +349,9 @@ function shutdown { } function update() { - bash -c "./Update.py" + bin_dir=${AIL_HOME}/bin + bash -c "python3 $bin_dir/Update.py" exitStatus=$? if [ $exitStatus -ge 1 ]; then echo -e $RED"\t* Update Error"$DEFAULT diff --git a/bin/check_ardb.sh b/bin/check_ardb.sh new file mode 100755 index 00000000..d73ac3e9 --- /dev/null +++ b/bin/check_ardb.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +GREEN="\\033[1;32m" +DEFAULT="\\033[0;39m" +RED="\\033[1;31m" +ROSE="\\033[1;35m" +BLUE="\\033[1;34m" +WHITE="\\033[0;02m" +YELLOW="\\033[1;33m" +CYAN="\\033[1;36m" + +[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; + +flag_ardb=0 +redis_dir=${AIL_HOME}/redis/src/ +sleep 0.2 +bash -c $redis_dir'redis-cli -p 6382 PING | grep "PONG" &> /dev/null' +if [ ! $? == 0 ]; then + echo -e $RED"\t6382 ARDB not ready"$DEFAULT + flag_ardb=1 +fi + +if [ $flag_ardb == 0 ]; then + exit 0 +else + exit 1 +fi diff --git a/bin/check_redis.sh b/bin/check_redis.sh new file mode 100755 index 00000000..0223e1eb --- /dev/null +++ b/bin/check_redis.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +GREEN="\\033[1;32m" +DEFAULT="\\033[0;39m" +RED="\\033[1;31m" +ROSE="\\033[1;35m" +BLUE="\\033[1;34m" +WHITE="\\033[0;02m" +YELLOW="\\033[1;33m" +CYAN="\\033[1;36m" + +[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; + +flag_redis=0 +redis_dir=${AIL_HOME}/redis/src/ +bash -c $redis_dir'redis-cli -p 6379 PING | grep "PONG" &> /dev/null' +if [ ! $? == 0 ]; then + echo -e $RED"\t6379 not ready"$DEFAULT + flag_redis=1 +fi +sleep 0.1 +bash -c $redis_dir'redis-cli -p 6380 PING | grep "PONG" &> /dev/null' +if [ ! $? == 0 ]; then + echo -e $RED"\t6380 not ready"$DEFAULT + flag_redis=1 +fi +sleep 0.1 +bash -c $redis_dir'redis-cli -p 6381 PING | grep "PONG" &> /dev/null' +if [ ! $? == 0 ]; then + echo -e $RED"\t6381 not ready"$DEFAULT + flag_redis=1 +fi +sleep 0.1 + +if [ $flag_redis == 0 ]; then + exit 0 +else + exit 1 +fi diff --git a/bin/launch_ardb.sh b/bin/launch_ardb.sh new file mode 100755 index 00000000..9d6ea1e7 --- /dev/null +++ b/bin/launch_ardb.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +GREEN="\\033[1;32m" +DEFAULT="\\033[0;39m" +RED="\\033[1;31m" +ROSE="\\033[1;35m" +BLUE="\\033[1;34m" +WHITE="\\033[0;02m" +YELLOW="\\033[1;33m" +CYAN="\\033[1;36m" + +[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; + +conf_dir="${AIL_HOME}/configs/" + +screen -dmS "ARDB_AIL" +sleep 0.1 +echo -e $GREEN"\t* Launching ARDB servers"$DEFAULT + +sleep 0.1 +screen -S "ARDB_AIL" -X screen -t "6382" bash -c 'cd '${AIL_HOME}'; ardb-server '$conf_dir'6382.conf ; read x' diff --git a/bin/launch_lvldb.sh b/bin/launch_lvldb.sh deleted file mode 100755 index a534e7ae..00000000 --- a/bin/launch_lvldb.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -set -e -set -x - -[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_LEVELDB" ] && echo "Needs the env var AIL_LEVELDB. Run the script from the virtual environment." && exit 1; - -lvdbhost='127.0.0.1' -lvdbdir="${AIL_HOME}/LEVEL_DB_DATA/" -nb_db=13 - -db_y=`date +%Y` -#Verify that a dir with the correct year exists, create it otherwise -if [ ! -d "$lvdbdir$db_y" ]; then - mkdir -p "$db_y" -fi - -screen -dmS "LevelDB" -sleep 0.1 -echo -e $GREEN"\t* Launching Levels DB servers"$DEFAULT - -#Launch a DB for each dir -for pathDir in $lvdbdir*/ ; do - yDir=$(basename "$pathDir") - sleep 0.1 - screen -S "LevelDB" -X screen -t "$yDir" bash -c 'redis-leveldb -H '$lvdbhost' -D '$pathDir'/ -P '$yDir' -M '$nb_db'; read x' -done diff --git a/bin/launch_redis.sh b/bin/launch_redis.sh index 69910927..91c35f4d 100755 --- a/bin/launch_redis.sh +++ b/bin/launch_redis.sh @@ -1,23 +1,23 @@ #!/bin/bash -set -e -set -x +GREEN="\\033[1;32m" +DEFAULT="\\033[0;39m" +RED="\\033[1;31m" +ROSE="\\033[1;35m" +BLUE="\\033[1;34m" +WHITE="\\033[0;02m" +YELLOW="\\033[1;33m" +CYAN="\\033[1;36m" [ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_LEVELDB" ] && echo "Needs the env var AIL_LEVELDB. Run the script from the virtual environment." && exit 1; conf_dir="${AIL_HOME}/configs/" -screen -dmS "Redis" +screen -dmS "Redis_AIL" sleep 0.1 echo -e $GREEN"\t* Launching Redis servers"$DEFAULT -screen -S "Redis" -X screen -t "6379" bash -c '../redis/src/redis-server '$conf_dir'6379.conf ; read x' +screen -S "Redis_AIL" -X screen -t "6379" bash -c 'redis-server '$conf_dir'6379.conf ; read x' sleep 0.1 -screen -S "Redis" -X screen -t "6380" bash -c '../redis/src/redis-server '$conf_dir'6380.conf ; read x' +screen -S "Redis_AIL" -X screen -t "6380" bash -c 'redis-server '$conf_dir'6380.conf ; read x' sleep 0.1 -screen -S "Redis" -X screen -t "6381" bash -c '../redis/src/redis-server '$conf_dir'6381.conf ; read x' - -# For Words and curves -sleep 0.1 -screen -S "Redis" -X screen -t "6382" bash -c '../redis/src/redis-server '$conf_dir'6382.conf ; read x' +screen -S "Redis_AIL" -X screen -t "6381" bash -c 'redis-server '$conf_dir'6381.conf ; read x' From aed8d65aefb2bd2773f524560dd88d0ec0bde8d2 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Wed, 14 Nov 2018 15:17:56 +0100 Subject: [PATCH 06/16] fix: [LAUNCH] add update in helper --- bin/LAUNCH.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index 7e6fdfc8..0a0e7fbb 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -57,6 +57,7 @@ function helptext { LAUNCH.sh [-l | --launchAuto] [-k | --killAll] + [-u | --update] [-c | --configUpdate] [-t | --thirdpartyUpdate] [-h | --help] From 3ae47a8659fb0b5c025662e06885d8b2023c51f0 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Thu, 15 Nov 2018 10:43:19 +0100 Subject: [PATCH 07/16] chg: [gitignore] add update version --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 2d276111..8dfd23b7 100644 --- a/.gitignore +++ b/.gitignore @@ -34,6 +34,7 @@ var/www/submitted bin/packages/config.cfg bin/packages/config.cfg.backup configs/keys +update/current_version files # installed files From f6e86582c84ea33f3df6988896e7fe86260cc648 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Thu, 15 Nov 2018 13:48:44 +0100 Subject: [PATCH 08/16] chg: [Update] generate current version --- bin/Update.py | 13 ++++++++++--- update/current_version | 1 - 2 files changed, 10 insertions(+), 4 deletions(-) delete mode 100644 update/current_version diff --git a/bin/Update.py b/bin/Update.py index fe29c936..ab3abf26 100755 --- a/bin/Update.py +++ b/bin/Update.py @@ -31,7 +31,8 @@ def check_if_files_modified(): if modified_files: print('Modified Files:') print('{}{}{}'.format(TERMINAL_BLUE, modified_files.decode(), TERMINAL_DEFAULT)) - return False + #return False + return True else: return True else: @@ -135,8 +136,14 @@ def update_fork(): def get_git_current_tag(current_version_path): - with open(current_version_path, 'r') as version_content: - version = version_content.read() + try: + with open(current_version_path, 'r') as version_content: + version = version_content.read() + except FileNotFoundError: + version = 'v1.4' + with open(current_version_path, 'w') as version_content: + version_content.write(version) + version = version.replace(" ", "").splitlines() return version[0] diff --git a/update/current_version b/update/current_version deleted file mode 100644 index b1f74215..00000000 --- a/update/current_version +++ /dev/null @@ -1 +0,0 @@ -v1.4 From aaa277b8a0b686666fa1783013dcca515b07e006 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Thu, 15 Nov 2018 14:26:41 +0100 Subject: [PATCH 09/16] chg: [Update] add additonal update --- bin/Update.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/bin/Update.py b/bin/Update.py index ab3abf26..cc7cb7d1 100755 --- a/bin/Update.py +++ b/bin/Update.py @@ -31,8 +31,7 @@ def check_if_files_modified(): if modified_files: print('Modified Files:') print('{}{}{}'.format(TERMINAL_BLUE, modified_files.decode(), TERMINAL_DEFAULT)) - #return False - return True + return False else: return True else: @@ -210,6 +209,22 @@ def update_ail(current_tag, list_upper_tags_remote, current_version_path, is_for print(output) if len(list_upper_tags_remote) == 1: + # additional update (between 2 commits on the same version) + additional_update_path = os.path.join(os.environ['AIL_HOME'], 'update', current_tag, 'additional_update.sh') + if os.path.isfile(additional_update_path): + print() + print('{}------------------------------------------------------------------'.format(TERMINAL_YELLOW)) + print('- Launching Additional Update: -') + print('-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --{}'.format(TERMINAL_DEFAULT)) + process = subprocess.run(['bash', additional_update_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if process.returncode == 0: + output = process.stdout.decode() + print(output) + else: + print('{}{}{}'.format(TERMINAL_RED, process.stderr.decode(), TERMINAL_DEFAULT)) + aborting_update() + sys.exit(1) + print() print('{}**************** AIL Sucessfully Updated *****************{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT)) print() From 3272ffa71486a1802e76f8a2dba00115d793bc5e Mon Sep 17 00:00:00 2001 From: Terrtia Date: Wed, 21 Nov 2018 17:07:29 +0100 Subject: [PATCH 10/16] chg: [Update] add 1.5 update, remove absolute paths from DB --- update/v1.5/Update.py | 160 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 160 insertions(+) create mode 100755 update/v1.5/Update.py diff --git a/update/v1.5/Update.py b/update/v1.5/Update.py new file mode 100755 index 00000000..ae52f4a0 --- /dev/null +++ b/update/v1.5/Update.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import sys +import time +import redis +import configparser + +if __name__ == '__main__': + + start_deb = time.time() + + configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') + if not os.path.exists(configfile): + raise Exception('Unable to find the configuration file. \ + Did you set environment variables? \ + Or activate the virtualenv.') + cfg = configparser.ConfigParser() + cfg.read(configfile) + + PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], cfg.get("Directories", "pastes")) + '/' + + r_serv_metadata = redis.StrictRedis( + host=cfg.get("ARDB_Metadata", "host"), + port=cfg.getint("ARDB_Metadata", "port"), + db=cfg.getint("ARDB_Metadata", "db"), + decode_responses=True) + + r_serv_tag = redis.StrictRedis( + host=cfg.get("ARDB_Tags", "host"), + port=cfg.getint("ARDB_Tags", "port"), + db=cfg.getint("ARDB_Tags", "db"), + decode_responses=True) + + r_serv_onion = redis.StrictRedis( + host=cfg.get("ARDB_Onion", "host"), + port=cfg.getint("ARDB_Onion", "port"), + db=cfg.getint("ARDB_Onion", "db"), + decode_responses=True) + + ## Update metadata ## + print('Updating ARDB_Metadata ...') + index = 0 + start = time.time() + for key in r_serv_metadata.scan_iter('*'): + + list_data = r_serv_metadata.hscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000) + while list_data[1]: + for hash_key, value in list_data[1].items(): + r_serv_metadata.hdel(key, hash_key) + if PASTES_FOLDER in hash_key: + new_hash = hash_key.replace(PASTES_FOLDER, '', 1) + else: + new_hash = hash_key + if PASTES_FOLDER in value: + new_value = value.replace(PASTES_FOLDER, '', 1) + else: + new_value = value + index = index +1 + r_serv_metadata.hset(key, new_hash, new_value) + + list_data = r_serv_metadata.sscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000) + + list_data = r_serv_metadata.zscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000) + while list_data[1]: + for elem in list_data[1]: + zset_key = elem[0] + value = int(elem[1]) + r_serv_metadata.zrem(key, zset_key) + new_key = zset_key.replace(PASTES_FOLDER, '', 1) + index = index +1 + r_serv_metadata.zincrby(key, new_key, value) + + list_data = r_serv_metadata.zscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000) + + if not 'dup:' in key: + list_data = r_serv_metadata.sscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000) + while list_data[1]: + for set_value in list_data[1]: + r_serv_metadata.srem(key, set_value) + r_serv_metadata.sadd(key, set_value.replace(PASTES_FOLDER, '', 1)) + index = index + 1 + + list_data = r_serv_metadata.sscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000) + + if PASTES_FOLDER in key: + new_key = key.replace(PASTES_FOLDER, '', 1) + + # a set with this key already exist + if r_serv_metadata.exists(new_key): + # save data + for new_key_value in r_serv_metadata.smembers(new_key): + r_serv_metadata.sadd(key, new_key_value) + r_serv_metadata.rename(key, new_key) + index = index + 1 + + end = time.time() + + + print('Updating ARDB_Metadata Done => {} paths: {} s'.format(index, end - start)) + + print() + print('Updating ARDB_Tags ...') + index = 0 + start = time.time() + + tags_list = r_serv_tag.smembers('list_tags') + for tag in tags_list: + res = False + + list_pastes = r_serv_tag.sscan(tag, 0, '*{}*'.format(PASTES_FOLDER), 1000) + while list_pastes[1]: + for paste in list_pastes[1]: + r_serv_tag.srem(tag, paste) + r_serv_tag.sadd(tag, paste.replace(PASTES_FOLDER, '', 1)) + index = index + 1 + + list_pastes = r_serv_tag.sscan(tag, 0, '*{}*'.format(PASTES_FOLDER), 1000) + + end = time.time() + print('Updating ARDB_Tags Done => {} paths: {} s'.format(index, end - start)) + + print() + print('Updating ARDB_Onion ...') + index = 0 + start = time.time() + for key in r_serv_onion.scan_iter('*'): + + if key != 'mess_onion': + list_data = r_serv_onion.hscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000) + while list_data[1]: + for hash_key, value in list_data[1].items(): + r_serv_onion.hdel(key, hash_key) + if PASTES_FOLDER in hash_key: + new_hash = hash_key.replace(PASTES_FOLDER, '', 1) + else: + new_hash = hash_key + if PASTES_FOLDER in value: + new_value = value.replace(PASTES_FOLDER, '', 1) + else: + new_value = value + index = index +1 + r_serv_onion.hset(key, new_hash, new_value) + + list_data = r_serv_onion.sscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000) + + list_data = r_serv_onion.sscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000) + while list_data[1]: + for set_value in list_data[1]: + r_serv_onion.srem(key, set_value) + r_serv_onion.sadd(key, set_value.replace(PASTES_FOLDER, '', 1)) + index = index + 1 + + list_data = r_serv_onion.sscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000) + + end = time.time() + print('Updating ARDB_Onion Done => {} paths: {} s'.format(index, end - start)) + print() + print('Done in {} s'.format(end - start_deb)) From 309e150b8b82382e5dcf4a4f0b5b9995fc9a00bb Mon Sep 17 00:00:00 2001 From: Terrtia Date: Thu, 22 Nov 2018 15:24:13 +0100 Subject: [PATCH 11/16] fix: [update 1.4] typo --- update/v1.5/Update.py | 99 +++++++++++++++++++++---------------------- 1 file changed, 48 insertions(+), 51 deletions(-) diff --git a/update/v1.5/Update.py b/update/v1.5/Update.py index ae52f4a0..2cf62d74 100755 --- a/update/v1.5/Update.py +++ b/update/v1.5/Update.py @@ -44,57 +44,60 @@ if __name__ == '__main__': index = 0 start = time.time() for key in r_serv_metadata.scan_iter('*'): - - list_data = r_serv_metadata.hscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000) - while list_data[1]: - for hash_key, value in list_data[1].items(): - r_serv_metadata.hdel(key, hash_key) - if PASTES_FOLDER in hash_key: - new_hash = hash_key.replace(PASTES_FOLDER, '', 1) - else: - new_hash = hash_key - if PASTES_FOLDER in value: - new_value = value.replace(PASTES_FOLDER, '', 1) - else: - new_value = value - index = index +1 - r_serv_metadata.hset(key, new_hash, new_value) - - list_data = r_serv_metadata.sscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000) - - list_data = r_serv_metadata.zscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000) - while list_data[1]: - for elem in list_data[1]: - zset_key = elem[0] - value = int(elem[1]) - r_serv_metadata.zrem(key, zset_key) - new_key = zset_key.replace(PASTES_FOLDER, '', 1) - index = index +1 - r_serv_metadata.zincrby(key, new_key, value) - - list_data = r_serv_metadata.zscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000) - if not 'dup:' in key: - list_data = r_serv_metadata.sscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000) - while list_data[1]: - for set_value in list_data[1]: - r_serv_metadata.srem(key, set_value) - r_serv_metadata.sadd(key, set_value.replace(PASTES_FOLDER, '', 1)) - index = index + 1 - - list_data = r_serv_metadata.sscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000) - if PASTES_FOLDER in key: new_key = key.replace(PASTES_FOLDER, '', 1) # a set with this key already exist if r_serv_metadata.exists(new_key): # save data - for new_key_value in r_serv_metadata.smembers(new_key): - r_serv_metadata.sadd(key, new_key_value) - r_serv_metadata.rename(key, new_key) + for new_key_value in r_serv_metadata.smembers(key): + r_serv_metadata.sadd(new_key, new_key_value) + r_serv_metadata.delete(key) index = index + 1 + type = r_serv_metadata.type(key) + print(type) + if type == 'hash': + list_data = r_serv_metadata.hscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000) + print(key) + while list_data[1]: + print(list_data[1]) + for hash_key, value in list_data[1].items(): + print('-----------------------------') + print(key) + print(hash_key) + print(value) + r_serv_metadata.hdel(key, hash_key) + new_hash = hash_key.replace(PASTES_FOLDER, '', 1) + new_value = value.replace(PASTES_FOLDER, '', 1) + index = index +1 + r_serv_metadata.hset(key, new_hash, new_value) + + list_data = r_serv_metadata.hscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000) + elif type == 'zset': + list_data = r_serv_metadata.zscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000) + while list_data[1]: + for elem in list_data[1]: + zset_key = elem[0] + value = int(elem[1]) + r_serv_metadata.zrem(key, zset_key) + new_key = zset_key.replace(PASTES_FOLDER, '', 1) + index = index +1 + r_serv_metadata.zincrby(key, new_key, value) + + list_data = r_serv_metadata.zscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000) + + elif type == 'set': + list_data = r_serv_metadata.sscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000) + while list_data[1]: + for set_value in list_data[1]: + r_serv_metadata.srem(key, set_value) + r_serv_metadata.sadd(key, set_value.replace(PASTES_FOLDER, '', 1)) + index = index + 1 + + list_data = r_serv_metadata.sscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000) + end = time.time() @@ -132,18 +135,12 @@ if __name__ == '__main__': while list_data[1]: for hash_key, value in list_data[1].items(): r_serv_onion.hdel(key, hash_key) - if PASTES_FOLDER in hash_key: - new_hash = hash_key.replace(PASTES_FOLDER, '', 1) - else: - new_hash = hash_key - if PASTES_FOLDER in value: - new_value = value.replace(PASTES_FOLDER, '', 1) - else: - new_value = value + new_hash = hash_key.replace(PASTES_FOLDER, '', 1) + new_value = value.replace(PASTES_FOLDER, '', 1) index = index +1 r_serv_onion.hset(key, new_hash, new_value) - list_data = r_serv_onion.sscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000) + list_data = r_serv_onion.hscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000) list_data = r_serv_onion.sscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000) while list_data[1]: From 0280b0b6474afa5f02afcd2c778ca7a968e834b3 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Wed, 19 Dec 2018 09:28:46 +0100 Subject: [PATCH 12/16] fix: [DB fix] performance --- update/v1.5/Update.py | 201 ++++++++++++++++++++++++++++++------------ 1 file changed, 145 insertions(+), 56 deletions(-) diff --git a/update/v1.5/Update.py b/update/v1.5/Update.py index 2cf62d74..b330672d 100755 --- a/update/v1.5/Update.py +++ b/update/v1.5/Update.py @@ -43,63 +43,153 @@ if __name__ == '__main__': print('Updating ARDB_Metadata ...') index = 0 start = time.time() - for key in r_serv_metadata.scan_iter('*'): - if not 'dup:' in key: - if PASTES_FOLDER in key: - new_key = key.replace(PASTES_FOLDER, '', 1) - # a set with this key already exist - if r_serv_metadata.exists(new_key): - # save data - for new_key_value in r_serv_metadata.smembers(key): - r_serv_metadata.sadd(new_key, new_key_value) - r_serv_metadata.delete(key) - index = index + 1 + string_keys_to_rename = ['misp_events:{}*'.format(PASTES_FOLDER), 'hive_cases:{}*'.format(PASTES_FOLDER)] + for key_to_rename in string_keys_to_rename: - type = r_serv_metadata.type(key) - print(type) - if type == 'hash': - list_data = r_serv_metadata.hscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000) + keys_to_rename = [] + for key in r_serv_metadata.scan_iter(key_to_rename): + new_key = key.replace(PASTES_FOLDER, '', 1) + keys_to_rename.append( (key, new_key) ) + index = index + 1 + for key, new_key in keys_to_rename: + r_serv_metadata.rename(key, new_key) + + keys_to_rename = None + + set_keys_to_rename = ['tag:{}*'.format(PASTES_FOLDER), 'hash_paste:{}*'.format(PASTES_FOLDER), 'base64_paste:{}*'.format(PASTES_FOLDER), 'binary_paste:{}*'.format(PASTES_FOLDER), 'hexadecimal_paste:{}*'.format(PASTES_FOLDER), 'paste_regular_external_links:{}*'.format(PASTES_FOLDER), 'paste_onion_external_links:{}*'.format(PASTES_FOLDER), 'paste_children:{}*'.format(PASTES_FOLDER)] + for key_to_rename in set_keys_to_rename: + + keys_to_remove = [] + keys_to_rename = [] + for key in r_serv_metadata.scan_iter(key_to_rename): + new_key = key.replace(PASTES_FOLDER, '', 1) + # a set with this key already exist + if r_serv_metadata.exists(new_key): + # save data + for new_key_value in r_serv_metadata.smembers(key): + r_serv_metadata.sadd(new_key, new_key_value) + keys_to_remove.append(key) + else: + keys_to_rename.append( (key, new_key) ) + index = index + 1 + for key in keys_to_remove: + r_serv_metadata.delete(key) + for key, new_key in keys_to_rename: + r_serv_metadata.rename(key, new_key) + + keys_to_remove = None + keys_to_rename = None + + + zset_keys_to_rename = ['nb_seen_hash:*', 'base64_hash:*', 'binary_hash:*'] + for key_to_rename in zset_keys_to_rename: + + keys_to_remove = [] + zkeys_to_remove = [] + keys_to_add = [] + for key in r_serv_metadata.scan_iter(key_to_rename): + temp = [] + for zset_key, value in r_serv_metadata.zscan_iter(key, '*{}*'.format(PASTES_FOLDER)): + #print(key) + #print(zset_key) + #print(value) + new_key = zset_key.replace(PASTES_FOLDER, '', 1) + index = index +1 + temp.append((key, zset_key)) + keys_to_add.append((key, new_key, value)) + if 0 < len(temp) < r_serv_metadata.zcard(key): + #print(key) + #print(len(temp)) + #print(temp) + #print(r_serv_metadata.zcard(key)) + #print('---------------') + zkeys_to_remove.extend(temp) + else: + keys_to_remove.append(key) + for key in keys_to_remove: + r_serv_metadata.delete(key) + for key, zset_key in zkeys_to_remove: + r_serv_metadata.zrem(key, zset_key) + for key, new_key, value in keys_to_add: + r_serv_metadata.zincrby(key, new_key, int(value)) + keys_to_remove = None + zkeys_to_remove = None + keys_to_add = None + + set_keys_to_rename = ['paste_children:*'] + for key_to_rename in set_keys_to_rename: + keys_to_remove = [] + skeys_to_remove = [] + keys_to_add = [] + for key in r_serv_metadata.scan_iter(key_to_rename): + temp = [] + for set_key in r_serv_metadata.sscan_iter(key, '*{}*'.format(PASTES_FOLDER)): + new_key = set_key.replace(PASTES_FOLDER, '', 1) + index = index +1 + temp.append((key, set_key)) + keys_to_add.append((key, new_key)) + if 0 < len(temp) < r_serv_metadata.scard(key): + skeys_to_remove.extend(temp) + else: + keys_to_remove.append(key) + for key in keys_to_remove: + r_serv_metadata.delete(key) + for key, set_key in skeys_to_remove: + r_serv_metadata.srem(key, set_key) + for key, new_key in keys_to_add: + r_serv_metadata.sadd(key, new_key) + keys_to_remove = None + skeys_to_remove = None + keys_to_add = None + + hset_keys_to_rename = ['paste_metadata:{}*'.format(PASTES_FOLDER)] + for key_to_rename in hset_keys_to_rename: + + keys_to_rename = [] + for key in r_serv_metadata.scan_iter(key_to_rename): + new_key = key.replace(PASTES_FOLDER, '', 1) + # a hset with this key already exist + if r_serv_metadata.exists(new_key): print(key) - while list_data[1]: - print(list_data[1]) - for hash_key, value in list_data[1].items(): - print('-----------------------------') - print(key) - print(hash_key) - print(value) - r_serv_metadata.hdel(key, hash_key) - new_hash = hash_key.replace(PASTES_FOLDER, '', 1) - new_value = value.replace(PASTES_FOLDER, '', 1) - index = index +1 - r_serv_metadata.hset(key, new_hash, new_value) + else: + keys_to_rename.append((key, new_key)) + index = index + 1 + for key, new_key in keys_to_rename: + r_serv_metadata.rename(key, new_key) + keys_to_rename = None - list_data = r_serv_metadata.hscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000) - elif type == 'zset': - list_data = r_serv_metadata.zscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000) - while list_data[1]: - for elem in list_data[1]: - zset_key = elem[0] - value = int(elem[1]) - r_serv_metadata.zrem(key, zset_key) - new_key = zset_key.replace(PASTES_FOLDER, '', 1) - index = index +1 - r_serv_metadata.zincrby(key, new_key, value) + # to verify 120/100 try with scan + hset_keys_to_rename = ['paste_metadata:*'] + for key_to_rename in hset_keys_to_rename: + for key in r_serv_metadata.scan_iter(key_to_rename): + father = r_serv_metadata.hget(key, 'father') + super_father = r_serv_metadata.hget(key, 'super_father') - list_data = r_serv_metadata.zscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000) + if father: + if PASTES_FOLDER in father: + index = index + 1 + r_serv_metadata.hdel(key, 'father') + r_serv_metadata.hset(key, 'father', father.replace(PASTES_FOLDER, '', 1)) - elif type == 'set': - list_data = r_serv_metadata.sscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000) - while list_data[1]: - for set_value in list_data[1]: - r_serv_metadata.srem(key, set_value) - r_serv_metadata.sadd(key, set_value.replace(PASTES_FOLDER, '', 1)) - index = index + 1 + if super_father: + if PASTES_FOLDER in super_father: + index = index + 1 + r_serv_metadata.hdel(key, 'super_father') + r_serv_metadata.hset(key, 'super_father', super_father.replace(PASTES_FOLDER, '', 1)) + + keys_to_rename = None - list_data = r_serv_metadata.sscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000) end = time.time() + '''' + for key in r_serv_metadata.scan_iter('*{}*'.format(PASTES_FOLDER)): + if not 'dup:' in key: + if not 'paste_i2p_external_links:' in key: + if not 'base64:' in key: + print(key) + ''' print('Updating ARDB_Metadata Done => {} paths: {} s'.format(index, end - start)) @@ -128,9 +218,10 @@ if __name__ == '__main__': print('Updating ARDB_Onion ...') index = 0 start = time.time() - for key in r_serv_onion.scan_iter('*'): - if key != 'mess_onion': + hset_keys_to_rename = ['onion_metadata:*'] + for key_to_rename in hset_keys_to_rename: + for key in r_serv_onion.scan_iter(key_to_rename): list_data = r_serv_onion.hscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000) while list_data[1]: for hash_key, value in list_data[1].items(): @@ -142,14 +233,12 @@ if __name__ == '__main__': list_data = r_serv_onion.hscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000) - list_data = r_serv_onion.sscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000) - while list_data[1]: - for set_value in list_data[1]: - r_serv_onion.srem(key, set_value) - r_serv_onion.sadd(key, set_value.replace(PASTES_FOLDER, '', 1)) - index = index + 1 + for elem in r_serv_onion.smembers('onion_crawler_queue'): + if PASTES_FOLDER in elem: + r_serv_onion.srem('onion_crawler_queue', elem) + r_serv_onion.sadd('onion_crawler_queue', elem.replace(PASTES_FOLDER, '', 1)) + index = index +1 - list_data = r_serv_onion.sscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000) end = time.time() print('Updating ARDB_Onion Done => {} paths: {} s'.format(index, end - start)) From 198ee97d90697ac579de856ce7e2ac42a2f50a83 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Wed, 19 Dec 2018 11:41:01 +0100 Subject: [PATCH 13/16] chg: [update 1.5] add update bash --- bin/LAUNCH.sh | 15 +++++++++------ update/v1.5/Update.py | 16 ---------------- update/v1.5/Update.sh | 39 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 47 insertions(+), 23 deletions(-) diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index 5467396f..2e6dd6e6 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -16,7 +16,6 @@ export AIL_HOME="${DIR}" cd ${AIL_HOME} if [ -e "${DIR}/AILENV/bin/python" ]; then - echo "AIL-framework virtualenv seems to exist, good" ENV_PY="${DIR}/AILENV/bin/python" else echo "Please make sure you have a AIL-framework environment, au revoir" @@ -348,11 +347,15 @@ function launch_feeder { function killall { if [[ $isredis || $isardb || $islogged || $isqueued || $isscripted || $isflasked || $isfeeded ]]; then - echo -e $GREEN"Gracefully closing redis servers"$DEFAULT - shutting_down_redis; - sleep 0.2 - echo -e $GREEN"Gracefully closing ardb servers"$DEFAULT - shutting_down_ardb; + if [[ $isredis ]]; then + echo -e $GREEN"Gracefully closing redis servers"$DEFAULT + shutting_down_redis; + sleep 0.2 + fi + if [[ $isardb ]]; then + echo -e $GREEN"Gracefully closing ardb servers"$DEFAULT + shutting_down_ardb; + fi echo -e $GREEN"Killing all"$DEFAULT kill $isredis $isardb $islogged $isqueued $isscripted $isflasked $isfeeded sleep 0.2 diff --git a/update/v1.5/Update.py b/update/v1.5/Update.py index b330672d..2622976f 100755 --- a/update/v1.5/Update.py +++ b/update/v1.5/Update.py @@ -91,19 +91,11 @@ if __name__ == '__main__': for key in r_serv_metadata.scan_iter(key_to_rename): temp = [] for zset_key, value in r_serv_metadata.zscan_iter(key, '*{}*'.format(PASTES_FOLDER)): - #print(key) - #print(zset_key) - #print(value) new_key = zset_key.replace(PASTES_FOLDER, '', 1) index = index +1 temp.append((key, zset_key)) keys_to_add.append((key, new_key, value)) if 0 < len(temp) < r_serv_metadata.zcard(key): - #print(key) - #print(len(temp)) - #print(temp) - #print(r_serv_metadata.zcard(key)) - #print('---------------') zkeys_to_remove.extend(temp) else: keys_to_remove.append(key) @@ -183,14 +175,6 @@ if __name__ == '__main__': end = time.time() - '''' - for key in r_serv_metadata.scan_iter('*{}*'.format(PASTES_FOLDER)): - if not 'dup:' in key: - if not 'paste_i2p_external_links:' in key: - if not 'base64:' in key: - print(key) - ''' - print('Updating ARDB_Metadata Done => {} paths: {} s'.format(index, end - start)) print() diff --git a/update/v1.5/Update.sh b/update/v1.5/Update.sh index e69e3b31..f329c7c5 100755 --- a/update/v1.5/Update.sh +++ b/update/v1.5/Update.sh @@ -1,4 +1,41 @@ #!/bin/bash -echo $AIL_HOME +[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1; +export PATH=$AIL_HOME:$PATH +export PATH=$AIL_REDIS:$PATH +export PATH=$AIL_ARDB:$PATH +export PATH=$AIL_BIN:$PATH +export PATH=$AIL_FLASK:$PATH + +echo "Killing all screens ..." +bash -c "bash ${AIL_BIN}/LAUNCH.sh -k" +echo "" +echo "Starting ARDB ..." +bash -c "bash ${AIL_BIN}/launch_ardb.sh" + +flag_ardb=true +while $flag_ardb; do + sleep 1 + bash -c "bash ${AIL_BIN}/check_ardb.sh" + if [ $? == 0 ]; then + flag_ardb=false + else + echo "ARDB not available, waiting 5s before retry" + sleep 5 + fi +done + +echo "" +bash -c "python ${AIL_HOME}/update/v1.5/Update.py" + +echo "Shutting down ARDB ..." +bash -c "bash ${AIL_BIN}/LAUNCH.sh -k" + +echo "" + +exit 0 From ca47764836e44480d1807e0c07f1e3815c49556e Mon Sep 17 00:00:00 2001 From: Terrtia Date: Wed, 19 Dec 2018 15:16:52 +0100 Subject: [PATCH 14/16] fix: [update v1.5] --- bin/Update.py | 19 ++++++++++++------- update/v1.5/Update.py | 7 ++----- update/v1.5/Update.sh | 2 ++ 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/bin/Update.py b/bin/Update.py index cc7cb7d1..f9cfbd9e 100755 --- a/bin/Update.py +++ b/bin/Update.py @@ -25,7 +25,6 @@ def auto_update_enabled(cfg): # check if files are modify locally def check_if_files_modified(): process = subprocess.run(['git', 'ls-files' ,'-m'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - if process.returncode == 0: modified_files = process.stdout if modified_files: @@ -197,7 +196,7 @@ def get_git_upper_tags_remote(current_tag, is_fork): def update_ail(current_tag, list_upper_tags_remote, current_version_path, is_fork): print('{}git checkout master:{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT)) process = subprocess.run(['git', 'checkout', 'master'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - + #process = subprocess.run(['ls'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) if process.returncode == 0: print(process.stdout.decode()) print() @@ -260,10 +259,16 @@ def launch_update_version(version, roll_back_commit, current_version_path, is_fo print('{}------------------------------------------------------------------'.format(TERMINAL_YELLOW)) print('- Launching Update: {}{}{} -'.format(TERMINAL_BLUE, version, TERMINAL_YELLOW)) print('-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --{}'.format(TERMINAL_DEFAULT)) - process = subprocess.run(['bash', update_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + process = subprocess.Popen(['bash', update_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + while True: + output = process.stdout.readline().decode() + if output == '' and process.poll() is not None: + break + if output: + print(output.strip()) if process.returncode == 0: - output = process.stdout.decode() - print(output) + #output = process.stdout.decode() + #print(output) with open(current_version_path, 'w') as version_content: version_content.write(version) @@ -273,8 +278,8 @@ def launch_update_version(version, roll_back_commit, current_version_path, is_fo print('------------------------------------------------------------------{}'.format(TERMINAL_DEFAULT)) print() else: - print(process.stdout.decode()) - print('{}{}{}'.format(TERMINAL_RED, process.stderr.decode(), TERMINAL_DEFAULT)) + #print(process.stdout.read().decode()) + print('{}{}{}'.format(TERMINAL_RED, process.stderr.read().decode(), TERMINAL_DEFAULT)) print('------------------------------------------------------------------') print(' {}Update Error: {}{}{}'.format(TERMINAL_RED, TERMINAL_BLUE, version, TERMINAL_DEFAULT)) print('------------------------------------------------------------------') diff --git a/update/v1.5/Update.py b/update/v1.5/Update.py index 2622976f..6a75b47a 100755 --- a/update/v1.5/Update.py +++ b/update/v1.5/Update.py @@ -142,11 +142,8 @@ if __name__ == '__main__': for key in r_serv_metadata.scan_iter(key_to_rename): new_key = key.replace(PASTES_FOLDER, '', 1) # a hset with this key already exist - if r_serv_metadata.exists(new_key): - print(key) - else: - keys_to_rename.append((key, new_key)) - index = index + 1 + keys_to_rename.append((key, new_key)) + index = index + 1 for key, new_key in keys_to_rename: r_serv_metadata.rename(key, new_key) keys_to_rename = None diff --git a/update/v1.5/Update.sh b/update/v1.5/Update.sh index f329c7c5..3cc45f01 100755 --- a/update/v1.5/Update.sh +++ b/update/v1.5/Update.sh @@ -30,6 +30,8 @@ while $flag_ardb; do fi done +echo "" +echo "Fixing ARDB ..." echo "" bash -c "python ${AIL_HOME}/update/v1.5/Update.py" From cee2b2486f0e7fc5d62b6105e8324753246e3663 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 26 Mar 2019 16:45:01 +0100 Subject: [PATCH 15/16] chg: [Launch] remove old launchers --- bin/LAUNCH.sh | 51 ++++++++++++++++++++++++---- bin/launch_ardb.sh | 21 ------------ bin/launch_logs.sh | 16 --------- bin/launch_queues.sh | 15 --------- bin/launch_redis.sh | 23 ------------- bin/launch_scripts.sh | 77 ------------------------------------------- 6 files changed, 45 insertions(+), 158 deletions(-) delete mode 100755 bin/launch_ardb.sh delete mode 100755 bin/launch_logs.sh delete mode 100755 bin/launch_queues.sh delete mode 100755 bin/launch_redis.sh delete mode 100755 bin/launch_scripts.sh diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index 2e6dd6e6..b281557e 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -82,11 +82,27 @@ function helptext { } function launching_redis { - bash -c "bash ${AIL_BIN}/launch_redis.sh" + conf_dir="${AIL_HOME}/configs/" + + screen -dmS "Redis_AIL" + sleep 0.1 + echo -e $GREEN"\t* Launching Redis servers"$DEFAULT + screen -S "Redis_AIL" -X screen -t "6379" bash -c 'redis-server '$conf_dir'6379.conf ; read x' + sleep 0.1 + screen -S "Redis_AIL" -X screen -t "6380" bash -c 'redis-server '$conf_dir'6380.conf ; read x' + sleep 0.1 + screen -S "Redis_AIL" -X screen -t "6381" bash -c 'redis-server '$conf_dir'6381.conf ; read x' } function launching_ardb { - bash -c "bash ${AIL_BIN}/launch_ardb.sh" + conf_dir="${AIL_HOME}/configs/" + + screen -dmS "ARDB_AIL" + sleep 0.1 + echo -e $GREEN"\t* Launching ARDB servers"$DEFAULT + + sleep 0.1 + screen -S "ARDB_AIL" -X screen -t "6382" bash -c 'cd '${AIL_HOME}'; ardb-server '$conf_dir'6382.conf ; read x' } function launching_logs { @@ -247,19 +263,37 @@ function shutting_down_ardb { function checking_redis { flag_redis=0 - bash -c "bash ${AIL_BIN}/check_redis.sh" + redis_dir=${AIL_HOME}/redis/src/ + bash -c $redis_dir'redis-cli -p 6379 PING | grep "PONG" &> /dev/null' if [ ! $? == 0 ]; then - flag_redis=1 + echo -e $RED"\t6379 not ready"$DEFAULT + flag_redis=1 fi + sleep 0.1 + bash -c $redis_dir'redis-cli -p 6380 PING | grep "PONG" &> /dev/null' + if [ ! $? == 0 ]; then + echo -e $RED"\t6380 not ready"$DEFAULT + flag_redis=1 + fi + sleep 0.1 + bash -c $redis_dir'redis-cli -p 6381 PING | grep "PONG" &> /dev/null' + if [ ! $? == 0 ]; then + echo -e $RED"\t6381 not ready"$DEFAULT + flag_redis=1 + fi + sleep 0.1 return $flag_redis; } function checking_ardb { flag_ardb=0 - bash -c "bash ${AIL_BIN}/check_ardb.sh" + redis_dir=${AIL_HOME}/redis/src/ + sleep 0.2 + bash -c $redis_dir'redis-cli -p 6382 PING | grep "PONG" &> /dev/null' if [ ! $? == 0 ]; then - flag_ardb=1 + echo -e $RED"\t6382 ARDB not ready"$DEFAULT + flag_ardb=1 fi return $flag_ardb; @@ -475,6 +509,8 @@ function launch_all { exit } +echo "$@" + while [ "$1" != "" ]; do case $1 in -l | --launchAuto ) launch_all "automatic"; @@ -491,6 +527,9 @@ while [ "$1" != "" ]; do ;; -h | --help ) helptext; exit + ;; + -kh | --khelp ) helptext; + ;; * ) helptext exit 1 diff --git a/bin/launch_ardb.sh b/bin/launch_ardb.sh deleted file mode 100755 index 9d6ea1e7..00000000 --- a/bin/launch_ardb.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -GREEN="\\033[1;32m" -DEFAULT="\\033[0;39m" -RED="\\033[1;31m" -ROSE="\\033[1;35m" -BLUE="\\033[1;34m" -WHITE="\\033[0;02m" -YELLOW="\\033[1;33m" -CYAN="\\033[1;36m" - -[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; - -conf_dir="${AIL_HOME}/configs/" - -screen -dmS "ARDB_AIL" -sleep 0.1 -echo -e $GREEN"\t* Launching ARDB servers"$DEFAULT - -sleep 0.1 -screen -S "ARDB_AIL" -X screen -t "6382" bash -c 'cd '${AIL_HOME}'; ardb-server '$conf_dir'6382.conf ; read x' diff --git a/bin/launch_logs.sh b/bin/launch_logs.sh deleted file mode 100755 index ed08ea48..00000000 --- a/bin/launch_logs.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -set -e -set -x - -[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_LEVELDB" ] && echo "Needs the env var AIL_LEVELDB. Run the script from the virtual environment." && exit 1; - -screen -dmS "Logging" -sleep 0.1 -echo -e $GREEN"\t* Launching logging process"$DEFAULT -screen -S "Logging" -X screen -t "LogQueue" bash -c 'log_subscriber -p 6380 -c Queuing -l ../logs/; read x' -sleep 0.1 -screen -S "Logging" -X screen -t "LogScript" bash -c 'log_subscriber -p 6380 -c Script -l ../logs/; read x' - diff --git a/bin/launch_queues.sh b/bin/launch_queues.sh deleted file mode 100755 index 2ddd2b7d..00000000 --- a/bin/launch_queues.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -set -e -set -x - -[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_LEVELDB" ] && echo "Needs the env var AIL_LEVELDB. Run the script from the virtual environment." && exit 1; - -screen -dmS "Queue" -sleep 0.1 - -echo -e $GREEN"\t* Launching all the queues"$DEFAULT -screen -S "Queue" -X screen -t "Queues" bash -c './launch_queues.py; read x' - diff --git a/bin/launch_redis.sh b/bin/launch_redis.sh deleted file mode 100755 index 91c35f4d..00000000 --- a/bin/launch_redis.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -GREEN="\\033[1;32m" -DEFAULT="\\033[0;39m" -RED="\\033[1;31m" -ROSE="\\033[1;35m" -BLUE="\\033[1;34m" -WHITE="\\033[0;02m" -YELLOW="\\033[1;33m" -CYAN="\\033[1;36m" - -[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; - -conf_dir="${AIL_HOME}/configs/" - -screen -dmS "Redis_AIL" -sleep 0.1 -echo -e $GREEN"\t* Launching Redis servers"$DEFAULT -screen -S "Redis_AIL" -X screen -t "6379" bash -c 'redis-server '$conf_dir'6379.conf ; read x' -sleep 0.1 -screen -S "Redis_AIL" -X screen -t "6380" bash -c 'redis-server '$conf_dir'6380.conf ; read x' -sleep 0.1 -screen -S "Redis_AIL" -X screen -t "6381" bash -c 'redis-server '$conf_dir'6381.conf ; read x' diff --git a/bin/launch_scripts.sh b/bin/launch_scripts.sh deleted file mode 100755 index 0dd29c2f..00000000 --- a/bin/launch_scripts.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/bin/bash - -set -e -set -x - -[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_LEVELDB" ] && echo "Needs the env var AIL_LEVELDB. Run the script from the virtual environment." && exit 1; - -echo -e "\t* Checking configuration" -bash -c "./Update-conf.py" -exitStatus=$? -if [ $exitStatus -ge 1 ]; then - echo -e $RED"\t* Configuration not up-to-date"$DEFAULT - exit -fi -echo -e $GREEN"\t* Configuration up-to-date"$DEFAULT - -screen -dmS "Script" -sleep 0.1 -echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT - -screen -S "Script" -X screen -t "ModuleInformation" bash -c './ModulesInformationV2.py -k 0 -c 1; read x' -sleep 0.1 -screen -S "Script" -X screen -t "Mixer" bash -c './Mixer.py; read x' -sleep 0.1 -screen -S "Script" -X screen -t "Global" bash -c './Global.py; read x' -sleep 0.1 -screen -S "Script" -X screen -t "Duplicates" bash -c './Duplicates.py; read x' -sleep 0.1 -screen -S "Script" -X screen -t "Attributes" bash -c './Attributes.py; read x' -sleep 0.1 -screen -S "Script" -X screen -t "Lines" bash -c './Lines.py; read x' -sleep 0.1 -screen -S "Script" -X screen -t "DomClassifier" bash -c './DomClassifier.py; read x' -sleep 0.1 -screen -S "Script" -X screen -t "Categ" bash -c './Categ.py; read x' -sleep 0.1 -screen -S "Script" -X screen -t "Tokenize" bash -c './Tokenize.py; read x' -sleep 0.1 -screen -S "Script" -X screen -t "CreditCards" bash -c './CreditCards.py; read x' -sleep 0.1 -screen -S "Script" -X screen -t "Onion" bash -c './Onion.py; read x' -sleep 0.1 -screen -S "Script" -X screen -t "Mail" bash -c './Mail.py; read x' -sleep 0.1 -screen -S "Script" -X screen -t "Web" bash -c './Web.py; read x' -sleep 0.1 -screen -S "Script" -X screen -t "Credential" bash -c './Credential.py; read x' -sleep 0.1 -screen -S "Script" -X screen -t "Curve" bash -c './Curve.py; read x' -sleep 0.1 -screen -S "Script" -X screen -t "CurveManageTopSets" bash -c './CurveManageTopSets.py; read x' -sleep 0.1 -screen -S "Script" -X screen -t "RegexForTermsFrequency" bash -c './RegexForTermsFrequency.py; read x' -sleep 0.1 -screen -S "Script" -X screen -t "SetForTermsFrequency" bash -c './SetForTermsFrequency.py; read x' -sleep 0.1 -screen -S "Script" -X screen -t "Indexer" bash -c './Indexer.py; read x' -sleep 0.1 -screen -S "Script" -X screen -t "Keys" bash -c './Keys.py; read x' -sleep 0.1 -screen -S "Script" -X screen -t "Phone" bash -c './Phone.py; read x' -sleep 0.1 -screen -S "Script" -X screen -t "Release" bash -c './Release.py; read x' -sleep 0.1 -screen -S "Script" -X screen -t "Cve" bash -c './Cve.py; read x' -sleep 0.1 -screen -S "Script" -X screen -t "WebStats" bash -c './WebStats.py; read x' -sleep 0.1 -screen -S "Script" -X screen -t "ModuleStats" bash -c './ModuleStats.py; read x' -sleep 0.1 -screen -S "Script" -X screen -t "SQLInjectionDetection" bash -c './SQLInjectionDetection.py; read x' -sleep 0.1 -screen -S "Script" -X screen -t "alertHandler" bash -c './alertHandler.py; read x' -sleep 0.1 -screen -S "Script" -X screen -t "SentimentAnalysis" bash -c './SentimentAnalysis.py; read x' From 044933aa336d5ddd1561dd9022e5a2e45f06f0a7 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 26 Mar 2019 16:47:10 +0100 Subject: [PATCH 16/16] chg: [Launch] remove old launchers --- bin/check_ardb.sh | 27 --------------------------- bin/check_redis.sh | 39 --------------------------------------- 2 files changed, 66 deletions(-) delete mode 100755 bin/check_ardb.sh delete mode 100755 bin/check_redis.sh diff --git a/bin/check_ardb.sh b/bin/check_ardb.sh deleted file mode 100755 index d73ac3e9..00000000 --- a/bin/check_ardb.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash - -GREEN="\\033[1;32m" -DEFAULT="\\033[0;39m" -RED="\\033[1;31m" -ROSE="\\033[1;35m" -BLUE="\\033[1;34m" -WHITE="\\033[0;02m" -YELLOW="\\033[1;33m" -CYAN="\\033[1;36m" - -[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; - -flag_ardb=0 -redis_dir=${AIL_HOME}/redis/src/ -sleep 0.2 -bash -c $redis_dir'redis-cli -p 6382 PING | grep "PONG" &> /dev/null' -if [ ! $? == 0 ]; then - echo -e $RED"\t6382 ARDB not ready"$DEFAULT - flag_ardb=1 -fi - -if [ $flag_ardb == 0 ]; then - exit 0 -else - exit 1 -fi diff --git a/bin/check_redis.sh b/bin/check_redis.sh deleted file mode 100755 index 0223e1eb..00000000 --- a/bin/check_redis.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash - -GREEN="\\033[1;32m" -DEFAULT="\\033[0;39m" -RED="\\033[1;31m" -ROSE="\\033[1;35m" -BLUE="\\033[1;34m" -WHITE="\\033[0;02m" -YELLOW="\\033[1;33m" -CYAN="\\033[1;36m" - -[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; - -flag_redis=0 -redis_dir=${AIL_HOME}/redis/src/ -bash -c $redis_dir'redis-cli -p 6379 PING | grep "PONG" &> /dev/null' -if [ ! $? == 0 ]; then - echo -e $RED"\t6379 not ready"$DEFAULT - flag_redis=1 -fi -sleep 0.1 -bash -c $redis_dir'redis-cli -p 6380 PING | grep "PONG" &> /dev/null' -if [ ! $? == 0 ]; then - echo -e $RED"\t6380 not ready"$DEFAULT - flag_redis=1 -fi -sleep 0.1 -bash -c $redis_dir'redis-cli -p 6381 PING | grep "PONG" &> /dev/null' -if [ ! $? == 0 ]; then - echo -e $RED"\t6381 not ready"$DEFAULT - flag_redis=1 -fi -sleep 0.1 - -if [ $flag_redis == 0 ]; then - exit 0 -else - exit 1 -fi