CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

fix: [v3.7] add missing file

pull/569/head
Terrtia 2021-08-27 18:18:44 +02:00
parent 564280935b
commit fb73c25675
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
1 changed files with 5 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
update/v3.7/Update_trackers.py Executable file → Normal file
View File

@ -10,7 +10,7 @@ import datetime
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader import ConfigLoader
import Domain import Tracker
def update_update_stats(): def update_update_stats():
nb_updated = int(r_serv_db.get('update:nb_elem_converted')) nb_updated = int(r_serv_db.get('update:nb_elem_converted'))
@ -18,17 +18,12 @@ def update_update_stats():
print('{}/{} updated {}%'.format(nb_updated, nb_elem_to_update, progress)) print('{}/{} updated {}%'.format(nb_updated, nb_elem_to_update, progress))
r_serv_db.set('ail:current_background_script_stat', progress) r_serv_db.set('ail:current_background_script_stat', progress)
def update_domain_language(domain_obj, item_id):
domain_name = domain_obj.get_domain_name()
Domain.add_domain_languages_by_item_id(domain_name, item_id)
if __name__ == '__main__': if __name__ == '__main__':
start_deb = time.time() start_deb = time.time()
config_loader = ConfigLoader.ConfigLoader() config_loader = ConfigLoader.ConfigLoader()
r_serv_db = config_loader.get_redis_conn("ARDB_DB") r_serv_db = config_loader.get_redis_conn("ARDB_DB")
r_serv_onion = config_loader.get_redis_conn("ARDB_Onion")
config_loader = None config_loader = None
r_serv_db.set('ail:current_background_script', 'trackers update') r_serv_db.set('ail:current_background_script', 'trackers update')
@ -39,28 +34,15 @@ if __name__ == '__main__':
else: else:
nb_elem_to_update = int(nb_elem_to_update) nb_elem_to_update = int(nb_elem_to_update)
while True: while True:
tracker_uuid = r_serv_onion.spop('trackers_update_v3.7') tracker_uuid = r_serv_onion.spop('trackers_update_v3.7')
if tracker_uuid is not None: if tracker_uuid is not None:
date_from =
date_to =
# FIX STATS
print(tracker_uuid) print(tracker_uuid)
# get all dates # FIX STATS
Tracker.fix_tracker_stats_per_day(tracker_uuid)
# MAP TRACKER - ITEM_ID
Tracker.fix_tracker_item_link(tracker_uuid)
# get items id
# convert
domain = Domain.Domain(domain)
for domain_history in domain.get_domain_history():
domain_item = domain.get_domain_items_crawled(epoch=domain_history[1]) # item_tag
if "items" in domain_item:
for item_dict in domain_item['items']:
update_domain_language(domain, item_dict['id'])
r_serv_db.incr('update:nb_elem_converted') r_serv_db.incr('update:nb_elem_converted')
update_update_stats() update_update_stats()