From ce3d98d6207f701d6b08bcfab9ecb7f9cd1d6def Mon Sep 17 00:00:00 2001 From: Terrtia Date: Thu, 9 Jan 2020 15:07:51 +0100 Subject: [PATCH] chg: [Update v2.7] add update v2.7 scripts --- bin/packages/Date.py | 3 + bin/update-background.py | 24 +++++- update/v1.5/Update-ARDB_Tags.py | 3 + update/v2.7/Update.py | 43 +++++++++++ update/v2.7/Update.sh | 39 ++++++++++ update/v2.7/Update_domain_tags.py | 119 ++++++++++++++++++++++++++++++ var/www/modules/Flask_config.py | 4 +- 7 files changed, 232 insertions(+), 3 deletions(-) create mode 100755 update/v2.7/Update.py create mode 100755 update/v2.7/Update.sh create mode 100755 update/v2.7/Update_domain_tags.py diff --git a/bin/packages/Date.py b/bin/packages/Date.py index 6b44c942..74563cb5 100644 --- a/bin/packages/Date.py +++ b/bin/packages/Date.py @@ -79,6 +79,9 @@ class Date(object): comp_day = str(computed_date.day).zfill(2) return comp_year + comp_month + comp_day +def get_today_date_str(): + return datetime.date.today().strftime("%Y%m%d") + def date_add_day(date, num_day=1): new_date = datetime.date(int(date[0:4]), int(date[4:6]), int(date[6:8])) + datetime.timedelta(num_day) new_date = str(new_date).replace('-', '') diff --git a/bin/update-background.py b/bin/update-background.py index a75eb60d..d1ec6eaf 100755 --- a/bin/update-background.py +++ b/bin/update-background.py @@ -86,11 +86,31 @@ if __name__ == "__main__": update_file = os.path.join(os.environ['AIL_HOME'], 'update', new_version, 'Update_screenshots.py') process = subprocess.run(['python' ,update_file]) - update_progress = r_serv_db.get('ail:current_background_script_stat') + update_progress = r_serv.get('ail:current_background_script_stat') if update_progress: if int(update_progress) == 100: r_serv.delete('ail:update_in_progress') r_serv.delete('ail:current_background_script') r_serv.delete('ail:current_background_script_stat') r_serv.delete('ail:current_background_update') - r_serv_db.srem('ail:to_update', new_version) + r_serv.srem('ail:to_update', new_version) + + elif r_serv.sismember('ail:to_update', 'v2.7'): + new_version = 'v2.7' + r_serv.delete('ail:update_error') + r_serv.delete('ail:current_background_script_stat') + r_serv.set('ail:update_in_progress', new_version) + r_serv.set('ail:current_background_update', new_version) + r_serv.set('ail:current_background_script', 'domain tags update') + + update_file = os.path.join(os.environ['AIL_HOME'], 'update', new_version, 'Update_domain_tags.py') + process = subprocess.run(['python' ,update_file]) + + update_progress = r_serv.get('ail:current_background_script_stat') + if update_progress: + if int(update_progress) == 100: + r_serv.delete('ail:update_in_progress') + r_serv.delete('ail:current_background_script') + r_serv.delete('ail:current_background_script_stat') + r_serv.delete('ail:current_background_update') + r_serv.srem('ail:to_update', new_version) diff --git a/update/v1.5/Update-ARDB_Tags.py b/update/v1.5/Update-ARDB_Tags.py index f94fc03c..76baa414 100755 --- a/update/v1.5/Update-ARDB_Tags.py +++ b/update/v1.5/Update-ARDB_Tags.py @@ -65,6 +65,9 @@ if __name__ == '__main__': tag_metadata[tag]['last_seen'] = int(tag_metadata[tag]['last_seen']) nb_tags_to_update += r_serv_tag.scard(tag) + if nb_tags_to_update == 0: + nb_tags_to_update = 1 + for tag in tags_list: all_item = r_serv_tag.smembers(tag) diff --git a/update/v2.7/Update.py b/update/v2.7/Update.py new file mode 100755 index 00000000..1f4ead2c --- /dev/null +++ b/update/v2.7/Update.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import re +import sys +import time +import redis +import datetime + +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) +import ConfigLoader + +new_version = 'v2.7' + +if __name__ == '__main__': + + start_deb = time.time() + + config_loader = ConfigLoader.ConfigLoader() + r_serv = config_loader.get_redis_conn("ARDB_DB") + r_serv_tags = config_loader.get_redis_conn("ARDB_Tags") + r_serv_onion = config_loader.get_redis_conn("ARDB_Onion") + config_loader = None + + #Set current update_in_progress + r_serv.set('ail:update_in_progress', new_version) + r_serv.set('ail:current_background_update', new_version) + + r_serv.sadd('ail:to_update', new_version) + + #### Update tags #### + r_serv_tags.sunionstore('list_tags:item', 'list_tags', []) + r_serv_onion.sunionstore('domain_update_v2.7', 'full_onion_up', []) + r_serv_onion.delete('incorrect_domain') + r_serv.set('ail:update_v2.7:deletetagrange', 1) + #### #### + + #Set current ail version + r_serv.set('ail:version', new_version) + + #Set current ail version + r_serv.hset('ail:update_date', new_version, datetime.datetime.now().strftime("%Y%m%d")) diff --git a/update/v2.7/Update.sh b/update/v2.7/Update.sh new file mode 100755 index 00000000..8f9a4efd --- /dev/null +++ b/update/v2.7/Update.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1; + +export PATH=$AIL_HOME:$PATH +export PATH=$AIL_REDIS:$PATH +export PATH=$AIL_ARDB:$PATH +export PATH=$AIL_BIN:$PATH +export PATH=$AIL_FLASK:$PATH + +GREEN="\\033[1;32m" +DEFAULT="\\033[0;39m" + +echo -e $GREEN"Shutting down AIL ..."$DEFAULT +bash ${AIL_BIN}/LAUNCH.sh -ks +wait + +bash ${AIL_BIN}/LAUNCH.sh -lav & +wait +echo "" + +echo "" +echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT +echo "" +python ${AIL_HOME}/update/v2.7/Update.py +wait +echo "" +echo "" + +echo "" +echo -e $GREEN"Shutting down ARDB ..."$DEFAULT +bash ${AIL_BIN}/LAUNCH.sh -ks +wait + +exit 0 diff --git a/update/v2.7/Update_domain_tags.py b/update/v2.7/Update_domain_tags.py new file mode 100755 index 00000000..cde5fea2 --- /dev/null +++ b/update/v2.7/Update_domain_tags.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import sys +import time +import redis +import datetime + +from hashlib import sha256 + +from pyfaup.faup import Faup + +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/')) +import Date + +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) +import ConfigLoader +import Tag + +def sanitize_domain(domain): + faup.decode(domain) + domain_sanitized = faup.get() + domain_sanitized = domain_sanitized['domain'] + try: + domain_sanitized = domain_sanitized.decode() + except: + pass + return domain_sanitized.lower() + +def delete_domain_tag_daterange(): + all_domains_tags = Tag.get_all_obj_tags('domain') + nb_updated = 0 + nb_to_update = len(all_domains_tags) + if nb_to_update == 0: + nb_to_update = 1 + refresh_time = time.time() + l_dates = Date.substract_date('20191008', Date.get_today_date_str()) + for tag in all_domains_tags: + for date_day in l_dates: + r_serv_tags.delete('domain:{}:{}'.format(tag, date_day)) + nb_updated += 1 + refresh_time = update_progress(refresh_time, nb_updated, nb_to_update) + r_serv_db.delete('ail:update_v2.7:deletetagrange') + +def update_domain_tags(domain): + domain_sanitized = sanitize_domain(domain) + if domain != domain_sanitized: + r_serv_onion.sadd('incorrect_domain', domain) + domain = domain_sanitized + + domain_tags = Tag.get_obj_tag(domain) + for tag in domain_tags: + # delete incorrect tags + if tag == 'infoleak:submission="crawler"' and tag == 'infoleak:submission="manual"': + r_serv_metadata.srem('tag:{}'.format(domain), tag) + else: + Tag.add_global_tag(tag, object_type='domain') + r_serv_tags.sadd('{}:{}'.format('domain', tag), domain) + +def update_progress(refresh_time, nb_updated, nb_elem_to_update): + if time.time() - refresh_time > 10: + progress = int((nb_updated * 100) / nb_elem_to_update) + print('{}/{} updated {}%'.format(nb_updated, nb_elem_to_update, progress)) + r_serv_db.set('ail:current_background_script_stat', progress) + refresh_time = time.time() + + return refresh_time + +def update_db(): + nb_updated = 0 + nb_to_update = r_serv_onion.scard('domain_update_v2.7') + refresh_time = time.time() + r_serv_db.set('ail:current_background_script_stat', 0) + r_serv_db.set('ail:current_background_script', 'domain tags update') + domain = r_serv_onion.spop('domain_update_v2.7') + while domain is not None: + update_domain_tags(domain) + nb_updated += 1 + refresh_time = update_progress(refresh_time, nb_updated, nb_to_update) + domain = r_serv_onion.spop('domain_update_v2.7') + if r_serv_db.exists('ail:update_v2.7:deletetagrange'): + r_serv_db.set('ail:current_background_script_stat', 0) + r_serv_db.set('ail:current_background_script', 'tags: remove deprecated keys') + delete_domain_tag_daterange() + +if __name__ == '__main__': + + start_deb = time.time() + faup = Faup() + + config_loader = ConfigLoader.ConfigLoader() + + r_serv_db = config_loader.get_redis_conn("ARDB_DB") + r_serv_tags = config_loader.get_redis_conn("ARDB_Tags") + r_serv_onion = config_loader.get_redis_conn("ARDB_Onion") + config_loader = None + + update_version = 'v2.7' + + r_serv_db.set('ail:update_in_progress', update_version) + r_serv_db.set('ail:current_background_update', update_version) + + r_serv_db.set('ail:current_background_script_stat', 0) + r_serv_db.set('ail:current_background_script', 'tags update') + + update_db() + + r_serv_db.set('ail:current_background_script_stat', 100) + + + end = time.time() + print('ALL domains tags updated in {} s'.format(end - start_deb)) + + r_serv_db.delete('ail:update_in_progress') + r_serv_db.delete('ail:current_background_script') + r_serv_db.delete('ail:current_background_script_stat') + r_serv_db.delete('ail:current_background_update') + r_serv_db.srem('ail:to_update', update_version) diff --git a/var/www/modules/Flask_config.py b/var/www/modules/Flask_config.py index fcedd7e7..6e4e17cc 100644 --- a/var/www/modules/Flask_config.py +++ b/var/www/modules/Flask_config.py @@ -90,7 +90,9 @@ dict_update_description = {'v1.5':{'nb_background_update': 5, 'update_warning_me 'v2.4':{'nb_background_update': 1, 'update_warning_message': 'An Update is running on the background. Some informations like Domain Tags/Correlation can be', 'update_warning_message_notice_me': 'missing from the UI.'}, 'v2.6':{'nb_background_update': 1, 'update_warning_message': 'An Update is running on the background. Some informations like Domain Tags/Correlation can be', - 'update_warning_message_notice_me': 'missing from the UI.'} + 'update_warning_message_notice_me': 'missing from the UI.'}, + 'v2.7':{'nb_background_update': 1, 'update_warning_message': 'An Update is running on the background. Some informations like Domain Tags can be', + 'update_warning_message_notice_me': 'missing from the UI.'} } UPLOAD_FOLDER = os.path.join(os.environ['AIL_FLASK'], 'submitted')