chg: [Update] add 1.5 update, remove absolute paths from DB

update/v1.5/Update.py

@@ -0,0 +1,160 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import os
+import sys
+import time
+import redis
+import configparser
+
+if __name__ == '__main__':
+
+    start_deb = time.time()
+
+    configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
+    if not os.path.exists(configfile):
+        raise Exception('Unable to find the configuration file. \
+                        Did you set environment variables? \
+                        Or activate the virtualenv.')
+    cfg = configparser.ConfigParser()
+    cfg.read(configfile)
+
+    PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], cfg.get("Directories", "pastes")) + '/'
+
+    r_serv_metadata = redis.StrictRedis(
+        host=cfg.get("ARDB_Metadata", "host"),
+        port=cfg.getint("ARDB_Metadata", "port"),
+        db=cfg.getint("ARDB_Metadata", "db"),
+        decode_responses=True)
+
+    r_serv_tag = redis.StrictRedis(
+        host=cfg.get("ARDB_Tags", "host"),
+        port=cfg.getint("ARDB_Tags", "port"),
+        db=cfg.getint("ARDB_Tags", "db"),
+        decode_responses=True)
+
+    r_serv_onion = redis.StrictRedis(
+        host=cfg.get("ARDB_Onion", "host"),
+        port=cfg.getint("ARDB_Onion", "port"),
+        db=cfg.getint("ARDB_Onion", "db"),
+        decode_responses=True)
+
+    ## Update metadata ##
+    print('Updating ARDB_Metadata ...')
+    index = 0
+    start = time.time()
+    for key in r_serv_metadata.scan_iter('*'):
+
+        list_data = r_serv_metadata.hscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000)
+        while list_data[1]:
+            for hash_key, value in list_data[1].items():
+                r_serv_metadata.hdel(key, hash_key)
+                if PASTES_FOLDER in hash_key:
+                    new_hash = hash_key.replace(PASTES_FOLDER, '', 1)
+                else:
+                    new_hash = hash_key
+                if PASTES_FOLDER in value:
+                    new_value = value.replace(PASTES_FOLDER, '', 1)
+                else:
+                    new_value = value
+                index = index +1
+                r_serv_metadata.hset(key, new_hash, new_value)
+
+            list_data = r_serv_metadata.sscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000)
+
+        list_data = r_serv_metadata.zscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000)
+        while list_data[1]:
+            for elem in list_data[1]:
+                zset_key = elem[0]
+                value =  int(elem[1])
+                r_serv_metadata.zrem(key, zset_key)
+                new_key = zset_key.replace(PASTES_FOLDER, '', 1)
+                index = index +1
+                r_serv_metadata.zincrby(key, new_key, value)
+
+            list_data = r_serv_metadata.zscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000)
+
+        if not 'dup:' in key:
+            list_data = r_serv_metadata.sscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000)
+            while list_data[1]:
+                for set_value in list_data[1]:
+                    r_serv_metadata.srem(key, set_value)
+                    r_serv_metadata.sadd(key, set_value.replace(PASTES_FOLDER, '', 1))
+                    index = index + 1
+
+                list_data = r_serv_metadata.sscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000)
+
+            if PASTES_FOLDER in key:
+                new_key = key.replace(PASTES_FOLDER, '', 1)
+
+                # a set with this key already exist
+                if r_serv_metadata.exists(new_key):
+                    # save data
+                    for new_key_value in r_serv_metadata.smembers(new_key):
+                        r_serv_metadata.sadd(key, new_key_value)
+                r_serv_metadata.rename(key, new_key)
+                index = index + 1
+
+    end = time.time()
+
+
+    print('Updating ARDB_Metadata Done => {} paths: {} s'.format(index, end - start))
+
+    print()
+    print('Updating ARDB_Tags ...')
+    index = 0
+    start = time.time()
+
+    tags_list = r_serv_tag.smembers('list_tags')
+    for tag in tags_list:
+        res = False
+
+        list_pastes = r_serv_tag.sscan(tag, 0, '*{}*'.format(PASTES_FOLDER), 1000)
+        while list_pastes[1]:
+            for paste in list_pastes[1]:
+                r_serv_tag.srem(tag, paste)
+                r_serv_tag.sadd(tag, paste.replace(PASTES_FOLDER, '', 1))
+                index = index + 1
+
+            list_pastes = r_serv_tag.sscan(tag, 0, '*{}*'.format(PASTES_FOLDER), 1000)
+
+    end = time.time()
+    print('Updating ARDB_Tags Done => {} paths: {} s'.format(index, end - start))
+
+    print()
+    print('Updating ARDB_Onion ...')
+    index = 0
+    start = time.time()
+    for key in r_serv_onion.scan_iter('*'):
+
+        if key != 'mess_onion':
+            list_data = r_serv_onion.hscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000)
+            while list_data[1]:
+                for hash_key, value in list_data[1].items():
+                    r_serv_onion.hdel(key, hash_key)
+                    if PASTES_FOLDER in hash_key:
+                        new_hash = hash_key.replace(PASTES_FOLDER, '', 1)
+                    else:
+                        new_hash = hash_key
+                    if PASTES_FOLDER in value:
+                        new_value = value.replace(PASTES_FOLDER, '', 1)
+                    else:
+                        new_value = value
+                    index = index +1
+                    r_serv_onion.hset(key, new_hash, new_value)
+
+                list_data = r_serv_onion.sscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000)
+
+            list_data = r_serv_onion.sscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000)
+            while list_data[1]:
+                for set_value in list_data[1]:
+                    r_serv_onion.srem(key, set_value)
+                    r_serv_onion.sadd(key, set_value.replace(PASTES_FOLDER, '', 1))
+                    index = index + 1
+
+                list_data = r_serv_onion.sscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000)
+
+    end = time.time()
+    print('Updating ARDB_Onion Done => {} paths: {} s'.format(index, end - start))
+    print()
+    print('Done in {} s'.format(end - start_deb))