CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

chg: [update v2.7] sort domain full_onion_up and full_regular_up

pull/449/head
Terrtia 2020-01-13 11:24:03 +01:00
parent fb320ca5d5
commit d677b32da6
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
1 changed files with 6 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
update/v2.7/Update_domain_tags.py
View File

@ -52,7 +52,7 @@ def update_domain_tags(domain):
domain_tags = Tag.get_obj_tag(domain) domain_tags = Tag.get_obj_tag(domain)
for tag in domain_tags: for tag in domain_tags:
# delete incorrect tags # delete incorrect tags
if tag == 'infoleak:submission="crawler"' and tag == 'infoleak:submission="manual"': if tag == 'infoleak:submission="crawler"' or tag == 'infoleak:submission="manual"':
r_serv_metadata.srem('tag:{}'.format(domain), tag) r_serv_metadata.srem('tag:{}'.format(domain), tag)
else: else:
Tag.add_global_tag(tag, object_type='domain') Tag.add_global_tag(tag, object_type='domain')
@ -84,6 +84,10 @@ def update_db():
r_serv_db.set('ail:current_background_script', 'tags: remove deprecated keys') r_serv_db.set('ail:current_background_script', 'tags: remove deprecated keys')
delete_domain_tag_daterange() delete_domain_tag_daterange()
# sort all crawled domain
r_serv_onion.sort('full_onion_up', alpha=True)
r_serv_onion.sort('full_regular_up', alpha=True)
if __name__ == '__main__': if __name__ == '__main__':
start_deb = time.time() start_deb = time.time()
@ -94,6 +98,7 @@ if __name__ == '__main__':
r_serv_db = config_loader.get_redis_conn("ARDB_DB") r_serv_db = config_loader.get_redis_conn("ARDB_DB")
r_serv_tags = config_loader.get_redis_conn("ARDB_Tags") r_serv_tags = config_loader.get_redis_conn("ARDB_Tags")
r_serv_onion = config_loader.get_redis_conn("ARDB_Onion") r_serv_onion = config_loader.get_redis_conn("ARDB_Onion")
r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
config_loader = None config_loader = None
update_version = 'v2.7' update_version = 'v2.7'