CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

fix: [tags] remove invalid tags

dev
terrtia 2024-01-08 14:51:40 +01:00
parent d7c8262653
commit 8bf67cf3b6
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
1 changed files with 19 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
bin/lib/Tag.py
View File

@ -635,7 +635,7 @@ def update_tag_metadata(tag, date, delete=False): # # TODO: delete Tags
# r_tags.smembers(f'{tag}:{date}') # r_tags.smembers(f'{tag}:{date}')
# r_tags.smembers(f'{obj_type}:{tag}') # r_tags.smembers(f'{obj_type}:{tag}')
def get_tag_objects(tag, obj_type, subtype='', date=''): def get_tag_objects(tag, obj_type, subtype='', date=''):
if obj_type == 'item': if obj_type == 'item' or obj_type == 'message':
return r_tags.smembers(f'{obj_type}:{subtype}:{tag}:{date}') return r_tags.smembers(f'{obj_type}:{subtype}:{tag}:{date}')
else: else:
return r_tags.smembers(f'{obj_type}:{subtype}:{tag}') return r_tags.smembers(f'{obj_type}:{subtype}:{tag}')
@ -716,6 +716,12 @@ def delete_object_tag(tag, obj_type, id, subtype=''):
date = item_basic.get_item_date(id) date = item_basic.get_item_date(id)
r_tags.srem(f'{obj_type}:{subtype}:{tag}:{date}', id) r_tags.srem(f'{obj_type}:{subtype}:{tag}:{date}', id)
update_tag_metadata(tag, date, delete=True)
elif obj_type == 'message':
timestamp = id.split('/')[1]
date = datetime.datetime.fromtimestamp(float(timestamp)).strftime('%Y%m%d')
r_tags.srem(f'{obj_type}:{subtype}:{tag}:{date}', id)
update_tag_metadata(tag, date, delete=True) update_tag_metadata(tag, date, delete=True)
else: else:
r_tags.srem(f'{obj_type}:{subtype}:{tag}', id) r_tags.srem(f'{obj_type}:{subtype}:{tag}', id)
@ -1458,6 +1464,18 @@ def get_list_of_solo_tags_to_export_by_type(export_type): # by type
return None return None
#r_serv_db.smembers('whitelist_hive') #r_serv_db.smembers('whitelist_hive')
def _fix_tag_obj_id():
for obj_type in ail_core.get_all_objects():
for tag in get_all_obj_tags(obj_type):
if ';' in tag:
new_tag = tag.split(';')[0]
for raw in get_obj_by_tags(obj_type, [new_tag], nb_obj=500000):
for global_id in raw.get('tagged_obj', []):
obj_type, subtype, obj_id = global_id.split(':', 2)
delete_object_tag(tag, obj_type, obj_id, subtype=subtype)
add_object_tag(new_tag, obj_type, obj_id, subtype=subtype)
# if __name__ == '__main__': # if __name__ == '__main__':
# taxo = 'accessnow' # taxo = 'accessnow'
# # taxo = TAXONOMIES.get(taxo) # # taxo = TAXONOMIES.get(taxo)