mirror of https://github.com/CIRCL/AIL-framework
chg: [Update] add 1.5 update, remove absolute paths from DB
Terrtia
parent
aaa277b8a0
commit
3272ffa714
|
|
@ -0,0 +1,160 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import redis
|
||||
import configparser
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
start_deb = time.time()
|
||||
|
||||
configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
|
||||
if not os.path.exists(configfile):
|
||||
raise Exception('Unable to find the configuration file. \
|
||||
Did you set environment variables? \
|
||||
Or activate the virtualenv.')
|
||||
cfg = configparser.ConfigParser()
|
||||
cfg.read(configfile)
|
||||
|
||||
PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], cfg.get("Directories", "pastes")) + '/'
|
||||
|
||||
r_serv_metadata = redis.StrictRedis(
|
||||
host=cfg.get("ARDB_Metadata", "host"),
|
||||
port=cfg.getint("ARDB_Metadata", "port"),
|
||||
db=cfg.getint("ARDB_Metadata", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
r_serv_tag = redis.StrictRedis(
|
||||
host=cfg.get("ARDB_Tags", "host"),
|
||||
port=cfg.getint("ARDB_Tags", "port"),
|
||||
db=cfg.getint("ARDB_Tags", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
r_serv_onion = redis.StrictRedis(
|
||||
host=cfg.get("ARDB_Onion", "host"),
|
||||
port=cfg.getint("ARDB_Onion", "port"),
|
||||
db=cfg.getint("ARDB_Onion", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
## Update metadata ##
|
||||
print('Updating ARDB_Metadata ...')
|
||||
index = 0
|
||||
start = time.time()
|
||||
for key in r_serv_metadata.scan_iter('*'):
|
||||
|
||||
list_data = r_serv_metadata.hscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000)
|
||||
while list_data[1]:
|
||||
for hash_key, value in list_data[1].items():
|
||||
r_serv_metadata.hdel(key, hash_key)
|
||||
if PASTES_FOLDER in hash_key:
|
||||
new_hash = hash_key.replace(PASTES_FOLDER, '', 1)
|
||||
else:
|
||||
new_hash = hash_key
|
||||
if PASTES_FOLDER in value:
|
||||
new_value = value.replace(PASTES_FOLDER, '', 1)
|
||||
else:
|
||||
new_value = value
|
||||
index = index +1
|
||||
r_serv_metadata.hset(key, new_hash, new_value)
|
||||
|
||||
list_data = r_serv_metadata.sscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000)
|
||||
|
||||
list_data = r_serv_metadata.zscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000)
|
||||
while list_data[1]:
|
||||
for elem in list_data[1]:
|
||||
zset_key = elem[0]
|
||||
value = int(elem[1])
|
||||
r_serv_metadata.zrem(key, zset_key)
|
||||
new_key = zset_key.replace(PASTES_FOLDER, '', 1)
|
||||
index = index +1
|
||||
r_serv_metadata.zincrby(key, new_key, value)
|
||||
|
||||
list_data = r_serv_metadata.zscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000)
|
||||
|
||||
if not 'dup:' in key:
|
||||
list_data = r_serv_metadata.sscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000)
|
||||
while list_data[1]:
|
||||
for set_value in list_data[1]:
|
||||
r_serv_metadata.srem(key, set_value)
|
||||
r_serv_metadata.sadd(key, set_value.replace(PASTES_FOLDER, '', 1))
|
||||
index = index + 1
|
||||
|
||||
list_data = r_serv_metadata.sscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000)
|
||||
|
||||
if PASTES_FOLDER in key:
|
||||
new_key = key.replace(PASTES_FOLDER, '', 1)
|
||||
|
||||
# a set with this key already exist
|
||||
if r_serv_metadata.exists(new_key):
|
||||
# save data
|
||||
for new_key_value in r_serv_metadata.smembers(new_key):
|
||||
r_serv_metadata.sadd(key, new_key_value)
|
||||
r_serv_metadata.rename(key, new_key)
|
||||
index = index + 1
|
||||
|
||||
end = time.time()
|
||||
|
||||
|
||||
print('Updating ARDB_Metadata Done => {} paths: {} s'.format(index, end - start))
|
||||
|
||||
print()
|
||||
print('Updating ARDB_Tags ...')
|
||||
index = 0
|
||||
start = time.time()
|
||||
|
||||
tags_list = r_serv_tag.smembers('list_tags')
|
||||
for tag in tags_list:
|
||||
res = False
|
||||
|
||||
list_pastes = r_serv_tag.sscan(tag, 0, '*{}*'.format(PASTES_FOLDER), 1000)
|
||||
while list_pastes[1]:
|
||||
for paste in list_pastes[1]:
|
||||
r_serv_tag.srem(tag, paste)
|
||||
r_serv_tag.sadd(tag, paste.replace(PASTES_FOLDER, '', 1))
|
||||
index = index + 1
|
||||
|
||||
list_pastes = r_serv_tag.sscan(tag, 0, '*{}*'.format(PASTES_FOLDER), 1000)
|
||||
|
||||
end = time.time()
|
||||
print('Updating ARDB_Tags Done => {} paths: {} s'.format(index, end - start))
|
||||
|
||||
print()
|
||||
print('Updating ARDB_Onion ...')
|
||||
index = 0
|
||||
start = time.time()
|
||||
for key in r_serv_onion.scan_iter('*'):
|
||||
|
||||
if key != 'mess_onion':
|
||||
list_data = r_serv_onion.hscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000)
|
||||
while list_data[1]:
|
||||
for hash_key, value in list_data[1].items():
|
||||
r_serv_onion.hdel(key, hash_key)
|
||||
if PASTES_FOLDER in hash_key:
|
||||
new_hash = hash_key.replace(PASTES_FOLDER, '', 1)
|
||||
else:
|
||||
new_hash = hash_key
|
||||
if PASTES_FOLDER in value:
|
||||
new_value = value.replace(PASTES_FOLDER, '', 1)
|
||||
else:
|
||||
new_value = value
|
||||
index = index +1
|
||||
r_serv_onion.hset(key, new_hash, new_value)
|
||||
|
||||
list_data = r_serv_onion.sscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000)
|
||||
|
||||
list_data = r_serv_onion.sscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000)
|
||||
while list_data[1]:
|
||||
for set_value in list_data[1]:
|
||||
r_serv_onion.srem(key, set_value)
|
||||
r_serv_onion.sadd(key, set_value.replace(PASTES_FOLDER, '', 1))
|
||||
index = index + 1
|
||||
|
||||
list_data = r_serv_onion.sscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000)
|
||||
|
||||
end = time.time()
|
||||
print('Updating ARDB_Onion Done => {} paths: {} s'.format(index, end - start))
|
||||
print()
|
||||
print('Done in {} s'.format(end - start_deb))
|
||||
Loading…
Reference in New Issue