From 3272ffa71486a1802e76f8a2dba00115d793bc5e Mon Sep 17 00:00:00 2001 From: Terrtia Date: Wed, 21 Nov 2018 17:07:29 +0100 Subject: [PATCH] chg: [Update] add 1.5 update, remove absolute paths from DB --- update/v1.5/Update.py | 160 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 160 insertions(+) create mode 100755 update/v1.5/Update.py diff --git a/update/v1.5/Update.py b/update/v1.5/Update.py new file mode 100755 index 00000000..ae52f4a0 --- /dev/null +++ b/update/v1.5/Update.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import sys +import time +import redis +import configparser + +if __name__ == '__main__': + + start_deb = time.time() + + configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') + if not os.path.exists(configfile): + raise Exception('Unable to find the configuration file. \ + Did you set environment variables? \ + Or activate the virtualenv.') + cfg = configparser.ConfigParser() + cfg.read(configfile) + + PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], cfg.get("Directories", "pastes")) + '/' + + r_serv_metadata = redis.StrictRedis( + host=cfg.get("ARDB_Metadata", "host"), + port=cfg.getint("ARDB_Metadata", "port"), + db=cfg.getint("ARDB_Metadata", "db"), + decode_responses=True) + + r_serv_tag = redis.StrictRedis( + host=cfg.get("ARDB_Tags", "host"), + port=cfg.getint("ARDB_Tags", "port"), + db=cfg.getint("ARDB_Tags", "db"), + decode_responses=True) + + r_serv_onion = redis.StrictRedis( + host=cfg.get("ARDB_Onion", "host"), + port=cfg.getint("ARDB_Onion", "port"), + db=cfg.getint("ARDB_Onion", "db"), + decode_responses=True) + + ## Update metadata ## + print('Updating ARDB_Metadata ...') + index = 0 + start = time.time() + for key in r_serv_metadata.scan_iter('*'): + + list_data = r_serv_metadata.hscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000) + while list_data[1]: + for hash_key, value in list_data[1].items(): + r_serv_metadata.hdel(key, hash_key) + if PASTES_FOLDER in hash_key: + new_hash = hash_key.replace(PASTES_FOLDER, '', 1) + else: + new_hash = hash_key + if PASTES_FOLDER in value: + new_value = value.replace(PASTES_FOLDER, '', 1) + else: + new_value = value + index = index +1 + r_serv_metadata.hset(key, new_hash, new_value) + + list_data = r_serv_metadata.sscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000) + + list_data = r_serv_metadata.zscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000) + while list_data[1]: + for elem in list_data[1]: + zset_key = elem[0] + value = int(elem[1]) + r_serv_metadata.zrem(key, zset_key) + new_key = zset_key.replace(PASTES_FOLDER, '', 1) + index = index +1 + r_serv_metadata.zincrby(key, new_key, value) + + list_data = r_serv_metadata.zscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000) + + if not 'dup:' in key: + list_data = r_serv_metadata.sscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000) + while list_data[1]: + for set_value in list_data[1]: + r_serv_metadata.srem(key, set_value) + r_serv_metadata.sadd(key, set_value.replace(PASTES_FOLDER, '', 1)) + index = index + 1 + + list_data = r_serv_metadata.sscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000) + + if PASTES_FOLDER in key: + new_key = key.replace(PASTES_FOLDER, '', 1) + + # a set with this key already exist + if r_serv_metadata.exists(new_key): + # save data + for new_key_value in r_serv_metadata.smembers(new_key): + r_serv_metadata.sadd(key, new_key_value) + r_serv_metadata.rename(key, new_key) + index = index + 1 + + end = time.time() + + + print('Updating ARDB_Metadata Done => {} paths: {} s'.format(index, end - start)) + + print() + print('Updating ARDB_Tags ...') + index = 0 + start = time.time() + + tags_list = r_serv_tag.smembers('list_tags') + for tag in tags_list: + res = False + + list_pastes = r_serv_tag.sscan(tag, 0, '*{}*'.format(PASTES_FOLDER), 1000) + while list_pastes[1]: + for paste in list_pastes[1]: + r_serv_tag.srem(tag, paste) + r_serv_tag.sadd(tag, paste.replace(PASTES_FOLDER, '', 1)) + index = index + 1 + + list_pastes = r_serv_tag.sscan(tag, 0, '*{}*'.format(PASTES_FOLDER), 1000) + + end = time.time() + print('Updating ARDB_Tags Done => {} paths: {} s'.format(index, end - start)) + + print() + print('Updating ARDB_Onion ...') + index = 0 + start = time.time() + for key in r_serv_onion.scan_iter('*'): + + if key != 'mess_onion': + list_data = r_serv_onion.hscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000) + while list_data[1]: + for hash_key, value in list_data[1].items(): + r_serv_onion.hdel(key, hash_key) + if PASTES_FOLDER in hash_key: + new_hash = hash_key.replace(PASTES_FOLDER, '', 1) + else: + new_hash = hash_key + if PASTES_FOLDER in value: + new_value = value.replace(PASTES_FOLDER, '', 1) + else: + new_value = value + index = index +1 + r_serv_onion.hset(key, new_hash, new_value) + + list_data = r_serv_onion.sscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000) + + list_data = r_serv_onion.sscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000) + while list_data[1]: + for set_value in list_data[1]: + r_serv_onion.srem(key, set_value) + r_serv_onion.sadd(key, set_value.replace(PASTES_FOLDER, '', 1)) + index = index + 1 + + list_data = r_serv_onion.sscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000) + + end = time.time() + print('Updating ARDB_Onion Done => {} paths: {} s'.format(index, end - start)) + print() + print('Done in {} s'.format(end - start_deb))