CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

fix: [DB fix] performance

pull/292/head
Terrtia 2018-12-19 09:28:46 +01:00
parent 309e150b8b
commit 0280b0b647
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
1 changed files with 145 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

201
update/v1.5/Update.py
View File

@ -43,63 +43,153 @@ if __name__ == '__main__':
print('Updating ARDB_Metadata ...') print('Updating ARDB_Metadata ...')
index = 0 index = 0
start = time.time() start = time.time()
for key in r_serv_metadata.scan_iter('*'):
if not 'dup:' in key:
if PASTES_FOLDER in key:
new_key = key.replace(PASTES_FOLDER, '', 1)
# a set with this key already exist string_keys_to_rename = ['misp_events:{}*'.format(PASTES_FOLDER), 'hive_cases:{}*'.format(PASTES_FOLDER)]
if r_serv_metadata.exists(new_key): for key_to_rename in string_keys_to_rename:
# save data
for new_key_value in r_serv_metadata.smembers(key):
r_serv_metadata.sadd(new_key, new_key_value)
r_serv_metadata.delete(key)
index = index + 1
type = r_serv_metadata.type(key) keys_to_rename = []
print(type) for key in r_serv_metadata.scan_iter(key_to_rename):
if type == 'hash': new_key = key.replace(PASTES_FOLDER, '', 1)
list_data = r_serv_metadata.hscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000) keys_to_rename.append( (key, new_key) )
index = index + 1
for key, new_key in keys_to_rename:
r_serv_metadata.rename(key, new_key)
keys_to_rename = None
set_keys_to_rename = ['tag:{}*'.format(PASTES_FOLDER), 'hash_paste:{}*'.format(PASTES_FOLDER), 'base64_paste:{}*'.format(PASTES_FOLDER), 'binary_paste:{}*'.format(PASTES_FOLDER), 'hexadecimal_paste:{}*'.format(PASTES_FOLDER), 'paste_regular_external_links:{}*'.format(PASTES_FOLDER), 'paste_onion_external_links:{}*'.format(PASTES_FOLDER), 'paste_children:{}*'.format(PASTES_FOLDER)]
for key_to_rename in set_keys_to_rename:
keys_to_remove = []
keys_to_rename = []
for key in r_serv_metadata.scan_iter(key_to_rename):
new_key = key.replace(PASTES_FOLDER, '', 1)
# a set with this key already exist
if r_serv_metadata.exists(new_key):
# save data
for new_key_value in r_serv_metadata.smembers(key):
r_serv_metadata.sadd(new_key, new_key_value)
keys_to_remove.append(key)
else:
keys_to_rename.append( (key, new_key) )
index = index + 1
for key in keys_to_remove:
r_serv_metadata.delete(key)
for key, new_key in keys_to_rename:
r_serv_metadata.rename(key, new_key)
keys_to_remove = None
keys_to_rename = None
zset_keys_to_rename = ['nb_seen_hash:*', 'base64_hash:*', 'binary_hash:*']
for key_to_rename in zset_keys_to_rename:
keys_to_remove = []
zkeys_to_remove = []
keys_to_add = []
for key in r_serv_metadata.scan_iter(key_to_rename):
temp = []
for zset_key, value in r_serv_metadata.zscan_iter(key, '*{}*'.format(PASTES_FOLDER)):
#print(key)
#print(zset_key)
#print(value)
new_key = zset_key.replace(PASTES_FOLDER, '', 1)
index = index +1
temp.append((key, zset_key))
keys_to_add.append((key, new_key, value))
if 0 < len(temp) < r_serv_metadata.zcard(key):
#print(key)
#print(len(temp))
#print(temp)
#print(r_serv_metadata.zcard(key))
#print('---------------')
zkeys_to_remove.extend(temp)
else:
keys_to_remove.append(key)
for key in keys_to_remove:
r_serv_metadata.delete(key)
for key, zset_key in zkeys_to_remove:
r_serv_metadata.zrem(key, zset_key)
for key, new_key, value in keys_to_add:
r_serv_metadata.zincrby(key, new_key, int(value))
keys_to_remove = None
zkeys_to_remove = None
keys_to_add = None
set_keys_to_rename = ['paste_children:*']
for key_to_rename in set_keys_to_rename:
keys_to_remove = []
skeys_to_remove = []
keys_to_add = []
for key in r_serv_metadata.scan_iter(key_to_rename):
temp = []
for set_key in r_serv_metadata.sscan_iter(key, '*{}*'.format(PASTES_FOLDER)):
new_key = set_key.replace(PASTES_FOLDER, '', 1)
index = index +1
temp.append((key, set_key))
keys_to_add.append((key, new_key))
if 0 < len(temp) < r_serv_metadata.scard(key):
skeys_to_remove.extend(temp)
else:
keys_to_remove.append(key)
for key in keys_to_remove:
r_serv_metadata.delete(key)
for key, set_key in skeys_to_remove:
r_serv_metadata.srem(key, set_key)
for key, new_key in keys_to_add:
r_serv_metadata.sadd(key, new_key)
keys_to_remove = None
skeys_to_remove = None
keys_to_add = None
hset_keys_to_rename = ['paste_metadata:{}*'.format(PASTES_FOLDER)]
for key_to_rename in hset_keys_to_rename:
keys_to_rename = []
for key in r_serv_metadata.scan_iter(key_to_rename):
new_key = key.replace(PASTES_FOLDER, '', 1)
# a hset with this key already exist
if r_serv_metadata.exists(new_key):
print(key) print(key)
while list_data[1]: else:
print(list_data[1]) keys_to_rename.append((key, new_key))
for hash_key, value in list_data[1].items(): index = index + 1
print('-----------------------------') for key, new_key in keys_to_rename:
print(key) r_serv_metadata.rename(key, new_key)
print(hash_key) keys_to_rename = None
print(value)
r_serv_metadata.hdel(key, hash_key)
new_hash = hash_key.replace(PASTES_FOLDER, '', 1)
new_value = value.replace(PASTES_FOLDER, '', 1)
index = index +1
r_serv_metadata.hset(key, new_hash, new_value)
list_data = r_serv_metadata.hscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000) # to verify 120/100 try with scan
elif type == 'zset': hset_keys_to_rename = ['paste_metadata:*']
list_data = r_serv_metadata.zscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000) for key_to_rename in hset_keys_to_rename:
while list_data[1]: for key in r_serv_metadata.scan_iter(key_to_rename):
for elem in list_data[1]: father = r_serv_metadata.hget(key, 'father')
zset_key = elem[0] super_father = r_serv_metadata.hget(key, 'super_father')
value = int(elem[1])
r_serv_metadata.zrem(key, zset_key)
new_key = zset_key.replace(PASTES_FOLDER, '', 1)
index = index +1
r_serv_metadata.zincrby(key, new_key, value)
list_data = r_serv_metadata.zscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000) if father:
if PASTES_FOLDER in father:
index = index + 1
r_serv_metadata.hdel(key, 'father')
r_serv_metadata.hset(key, 'father', father.replace(PASTES_FOLDER, '', 1))
elif type == 'set': if super_father:
list_data = r_serv_metadata.sscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000) if PASTES_FOLDER in super_father:
while list_data[1]: index = index + 1
for set_value in list_data[1]: r_serv_metadata.hdel(key, 'super_father')
r_serv_metadata.srem(key, set_value) r_serv_metadata.hset(key, 'super_father', super_father.replace(PASTES_FOLDER, '', 1))
r_serv_metadata.sadd(key, set_value.replace(PASTES_FOLDER, '', 1))
index = index + 1 keys_to_rename = None
list_data = r_serv_metadata.sscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000)
end = time.time() end = time.time()
''''
for key in r_serv_metadata.scan_iter('*{}*'.format(PASTES_FOLDER)):
if not 'dup:' in key:
if not 'paste_i2p_external_links:' in key:
if not 'base64:' in key:
print(key)
'''
print('Updating ARDB_Metadata Done => {} paths: {} s'.format(index, end - start)) print('Updating ARDB_Metadata Done => {} paths: {} s'.format(index, end - start))
@ -128,9 +218,10 @@ if __name__ == '__main__':
print('Updating ARDB_Onion ...') print('Updating ARDB_Onion ...')
index = 0 index = 0
start = time.time() start = time.time()
for key in r_serv_onion.scan_iter('*'):
if key != 'mess_onion': hset_keys_to_rename = ['onion_metadata:*']
for key_to_rename in hset_keys_to_rename:
for key in r_serv_onion.scan_iter(key_to_rename):
list_data = r_serv_onion.hscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000) list_data = r_serv_onion.hscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000)
while list_data[1]: while list_data[1]:
for hash_key, value in list_data[1].items(): for hash_key, value in list_data[1].items():
@ -142,14 +233,12 @@ if __name__ == '__main__':
list_data = r_serv_onion.hscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000) list_data = r_serv_onion.hscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000)
list_data = r_serv_onion.sscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000) for elem in r_serv_onion.smembers('onion_crawler_queue'):
while list_data[1]: if PASTES_FOLDER in elem:
for set_value in list_data[1]: r_serv_onion.srem('onion_crawler_queue', elem)
r_serv_onion.srem(key, set_value) r_serv_onion.sadd('onion_crawler_queue', elem.replace(PASTES_FOLDER, '', 1))
r_serv_onion.sadd(key, set_value.replace(PASTES_FOLDER, '', 1)) index = index +1
index = index + 1
list_data = r_serv_onion.sscan(key, 0, '*{}*'.format(PASTES_FOLDER), 1000)
end = time.time() end = time.time()
print('Updating ARDB_Onion Done => {} paths: {} s'.format(index, end - start)) print('Updating ARDB_Onion Done => {} paths: {} s'.format(index, end - start))