CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

fix: [Update DB] remove indexErrors

pull/342/head
Terrtia 2019-04-12 17:32:17 +02:00
parent bf92a2f22f
commit a3167a740a
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
4 changed files with 27 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
bin/update-background.py
View File

@ -32,23 +32,22 @@ if __name__ == "__main__":
decode_responses=True) decode_responses=True)
if r_serv.exists('ail:update_v1.5'): if r_serv.exists('ail:update_v1.5'):
if ail_version == 'v1.5': onions_update_status = r_serv.get('v1.5:onions')
onions_update_status = r_serv.get('v1.5:onions') if onions_update_status is None:
if onions_update_status is None: update_file = os.path.join(os.environ['AIL_HOME'], 'update', 'v1.4', 'Update-ARDB_Onions.py')
update_file = os.path.join(os.environ['AIL_HOME'], 'update', 'v1.4', 'Update-ARDB_Onions.py') process = subprocess.run(['python' ,update_file])
process = subprocess.run(['python' ,update_file])
metadata_update_status = r_serv.get('v1.5:metadata') metadata_update_status = r_serv.get('v1.5:metadata')
if metadata_update_status is None: if metadata_update_status is None:
update_file = os.path.join(os.environ['AIL_HOME'], 'update', 'v1.4', 'Update-ARDB_Metadata.py') update_file = os.path.join(os.environ['AIL_HOME'], 'update', 'v1.4', 'Update-ARDB_Metadata.py')
process = subprocess.run(['python' ,update_file]) process = subprocess.run(['python' ,update_file])
tags_update_status = r_serv.get('v1.5:tags') tags_update_status = r_serv.get('v1.5:tags')
if tags_update_status is None: if tags_update_status is None:
update_file = os.path.join(os.environ['AIL_HOME'], 'update', 'v1.4', 'Update-ARDB_Tags.py') update_file = os.path.join(os.environ['AIL_HOME'], 'update', 'v1.4', 'Update-ARDB_Tags.py')
process = subprocess.run(['python' ,update_file]) process = subprocess.run(['python' ,update_file])
tags_background_update_status = r_serv.get('v1.5:tags_background') tags_background_update_status = r_serv.get('v1.5:tags_background')
if tags_background_update_status is None: if tags_background_update_status is None:
update_file = os.path.join(os.environ['AIL_HOME'], 'update', 'v1.4', 'Update-ARDB_Tags_background.py') update_file = os.path.join(os.environ['AIL_HOME'], 'update', 'v1.4', 'Update-ARDB_Tags_background.py')
process = subprocess.run(['python' ,update_file]) process = subprocess.run(['python' ,update_file])

3
update/v1.4/Update-ARDB_Metadata.py
View File

@ -31,7 +31,6 @@ def update_hash_item(has_type):
if r_serv_metadata.exists(hash_key): if r_serv_metadata.exists(hash_key):
new_hash_key = hash_key.replace(PASTES_FOLDER, '', 1) new_hash_key = hash_key.replace(PASTES_FOLDER, '', 1)
res = r_serv_metadata.renamenx(hash_key, new_hash_key) res = r_serv_metadata.renamenx(hash_key, new_hash_key)
print(res)
if res == 0: if res == 0:
print('same key, double name: {}'.format(item_path)) print('same key, double name: {}'.format(item_path))
# fusion # fusion
@ -106,7 +105,7 @@ if __name__ == '__main__':
# update domain port # update domain port
domain = r_serv_metadata.hget(new_item_metadata, 'domain') domain = r_serv_metadata.hget(new_item_metadata, 'domain')
if domain: if domain:
if domain[-3:] ?= ':80': if domain[-3:] != ':80':
r_serv_metadata.hset(new_item_metadata, 'domain', '{}:80'.format(domain)) r_serv_metadata.hset(new_item_metadata, 'domain', '{}:80'.format(domain))
super_father = r_serv_metadata.hget(new_item_metadata, 'super_father') super_father = r_serv_metadata.hget(new_item_metadata, 'super_father')
if super_father: if super_father:

6
update/v1.4/Update-ARDB_Onions.py
View File

@ -107,7 +107,11 @@ if __name__ == '__main__':
print('onion_history:{}:{}'.format(onion_domain, date_history)) print('onion_history:{}:{}'.format(onion_domain, date_history))
item_father = r_serv_onion.lrange('onion_history:{}:{}'.format(onion_domain, date_history), 0, 0) item_father = r_serv_onion.lrange('onion_history:{}:{}'.format(onion_domain, date_history), 0, 0)
print('item_father: {}'.format(item_father)) print('item_father: {}'.format(item_father))
item_father = item_father[0] try:
item_father = item_father[0]
except IndexError:
r_serv_onion.delete('onion_history:{}:{}'.format(onion_domain, date_history))
continue
#print(item_father) #print(item_father)
# delete old history # delete old history
r_serv_onion.delete('onion_history:{}:{}'.format(onion_domain, date_history)) r_serv_onion.delete('onion_history:{}:{}'.format(onion_domain, date_history))

6
update/v1.4/Update-ARDB_Tags.py
View File

@ -85,7 +85,11 @@ if __name__ == '__main__':
splitted_item_path = item_path.split('/') splitted_item_path = item_path.split('/')
#print(tag) #print(tag)
#print(item_path) #print(item_path)
item_date = int( ''.join([splitted_item_path[-4], splitted_item_path[-3], splitted_item_path[-2]]) ) try:
item_date = int( ''.join([splitted_item_path[-4], splitted_item_path[-3], splitted_item_path[-2]]) )
except IndexError:
r_serv_tag.srem(tag, item_path)
continue
# remove absolute path # remove absolute path
new_path = item_path.replace(PASTES_FOLDER, '', 1) new_path = item_path.replace(PASTES_FOLDER, '', 1)