diff --git a/update/bin/old_ail_updater.py b/update/bin/old_ail_updater.py index 9be78e81..14833679 100755 --- a/update/bin/old_ail_updater.py +++ b/update/bin/old_ail_updater.py @@ -2,14 +2,15 @@ # -*-coding:UTF-8 -* import os -import re import sys import time -import redis import datetime -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) -import ConfigLoader +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib import ConfigLoader class AIL_Updater(object): """docstring for AIL_Updater.""" @@ -38,9 +39,9 @@ class AIL_Updater(object): """ Update DB version """ - #Set current ail version + # Set current ail version self.r_serv.hset('ail:update_date', self.version, datetime.datetime.now().strftime("%Y%m%d")) - #Set current ail version + # Set current ail version if self.f_version > self.current_f_version: self.r_serv.set('ail:version', self.version) diff --git a/update/default_update/Update.py b/update/default_update/Update.py index dc074860..3e1de5d8 100755 --- a/update/default_update/Update.py +++ b/update/default_update/Update.py @@ -4,10 +4,7 @@ import os import sys import time -import redis import argparse -import datetime -import configparser sys.path.append(os.environ['AIL_BIN']) ################################## @@ -17,7 +14,7 @@ from lib import ail_updates if __name__ == '__main__': parser = argparse.ArgumentParser(description='AIL default update') - parser.add_argument('-t', help='version tag' , type=str, dest='tag', required=True) + parser.add_argument('-t', help='version tag', type=str, dest='tag', required=True) args = parser.parse_args() if not args.tag: @@ -28,7 +25,7 @@ if __name__ == '__main__': update_tag = args.tag.replace(' ', '') if not ail_updates.check_version(update_tag): parser.print_help() - print(f'Error: Invalid update tag {update_tag}) + print(f'Error: Invalid update tag {update_tag}') sys.exit(0) start_deb = time.time() diff --git a/update/v1.5/Update-ARDB_Metadata.py b/update/v1.5/Update-ARDB_Metadata.py index e8fcb1bb..f678f74e 100755 --- a/update/v1.5/Update-ARDB_Metadata.py +++ b/update/v1.5/Update-ARDB_Metadata.py @@ -4,10 +4,9 @@ import os import sys import time -import redis sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) -import ConfigLoader +from lib import ConfigLoader def update_tracked_terms(main_key, tracked_container_key): for tracked_item in r_serv_term.smembers(main_key): diff --git a/update/v1.5/Update-ARDB_Onions.py b/update/v1.5/Update-ARDB_Onions.py index 7b4aee99..7f36b99a 100755 --- a/update/v1.5/Update-ARDB_Onions.py +++ b/update/v1.5/Update-ARDB_Onions.py @@ -4,11 +4,10 @@ import os import sys import time -import redis import datetime -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) -import ConfigLoader +sys.path.append(os.environ['AIL_BIN']) +from lib import ConfigLoader def substract_date(date_from, date_to): date_from = datetime.date(int(date_from[0:4]), int(date_from[4:6]), int(date_from[6:8])) diff --git a/update/v1.5/Update-ARDB_Onions_screenshots.py b/update/v1.5/Update-ARDB_Onions_screenshots.py index 442f979c..6ad7a025 100755 --- a/update/v1.5/Update-ARDB_Onions_screenshots.py +++ b/update/v1.5/Update-ARDB_Onions_screenshots.py @@ -4,13 +4,12 @@ import os import sys import time -import redis import datetime from hashlib import sha256 -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) -import ConfigLoader +sys.path.append(os.environ['AIL_BIN']) +from lib import ConfigLoader def rreplace(s, old, new, occurrence): li = s.rsplit(old, occurrence) @@ -26,6 +25,7 @@ def substract_date(date_from, date_to): l_date.append( date.strftime('%Y%m%d') ) return l_date + if __name__ == '__main__': start_deb = time.time() diff --git a/update/v1.5/Update-ARDB_Tags.py b/update/v1.5/Update-ARDB_Tags.py index e2e8852e..38f3c09c 100755 --- a/update/v1.5/Update-ARDB_Tags.py +++ b/update/v1.5/Update-ARDB_Tags.py @@ -7,7 +7,7 @@ import time import redis sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) -import ConfigLoader +from lib import ConfigLoader if __name__ == '__main__': diff --git a/update/v1.5/Update-ARDB_Tags_background.py b/update/v1.5/Update-ARDB_Tags_background.py index 0e8eef0d..0df3d75d 100755 --- a/update/v1.5/Update-ARDB_Tags_background.py +++ b/update/v1.5/Update-ARDB_Tags_background.py @@ -4,10 +4,9 @@ import os import sys import time -import redis -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) -import ConfigLoader +sys.path.append(os.environ['AIL_BIN']) +from lib import ConfigLoader def tags_key_fusion(old_item_path_key, new_item_path_key): print('fusion:') @@ -17,10 +16,10 @@ def tags_key_fusion(old_item_path_key, new_item_path_key): r_serv_metadata.sadd(new_item_path_key, tag) r_serv_metadata.srem(old_item_path_key, tag) + if __name__ == '__main__': start_deb = time.time() - config_loader = ConfigLoader.ConfigLoader() PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "pastes")) + '/' @@ -38,7 +37,7 @@ if __name__ == '__main__': print('Updating ARDB_Tags ...') start = time.time() - #update item metadata tags + # update item metadata tags tag_not_updated = True total_to_update = r_serv_tag.scard('maj:v1.5:absolute_path_to_rename') nb_updated = 0 @@ -57,7 +56,7 @@ if __name__ == '__main__': if r_serv_tag.scard('maj:v1.5:absolute_path_to_rename') == 0: tag_not_updated = False else: - progress = int((nb_updated * 100) /total_to_update) + progress = int((nb_updated * 100) / total_to_update) print('{}/{} Tags updated {}%'.format(nb_updated, total_to_update, progress)) # update progress stats if progress != last_progress: @@ -66,7 +65,6 @@ if __name__ == '__main__': end = time.time() - print('Updating ARDB_Tags Done: {} s'.format(end - start)) r_serv.sadd('ail:update_v1.5', 'tags_background') diff --git a/update/v1.5/Update.py b/update/v1.5/Update.py index be80c76d..fd1a2dd5 100755 --- a/update/v1.5/Update.py +++ b/update/v1.5/Update.py @@ -4,11 +4,10 @@ import os import sys import time -import redis import datetime -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) -import ConfigLoader +sys.path.append(os.environ['AIL_BIN']) +from lib import ConfigLoader if __name__ == '__main__': diff --git a/update/v1.7/Update.py b/update/v1.7/Update.py index b3ebd591..f3777d0e 100755 --- a/update/v1.7/Update.py +++ b/update/v1.7/Update.py @@ -4,11 +4,10 @@ import os import sys import time -import redis import datetime -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) -import ConfigLoader +sys.path.append(os.environ['AIL_BIN']) +from lib import ConfigLoader if __name__ == '__main__': @@ -19,8 +18,8 @@ if __name__ == '__main__': r_serv = config_loader.get_redis_conn("ARDB_DB") config_loader = None - #Set current ail version + # Set current ail version r_serv.set('ail:version', 'v1.7') - #Set current ail version + # Set current ail version r_serv.set('ail:update_date_v1.7', datetime.datetime.now().strftime("%Y%m%d")) diff --git a/update/v2.0/Update.py b/update/v2.0/Update.py index d5533a74..4d3504e4 100755 --- a/update/v2.0/Update.py +++ b/update/v2.0/Update.py @@ -4,11 +4,10 @@ import os import sys import time -import redis import datetime -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) -import ConfigLoader +sys.path.append(os.environ['AIL_BIN']) +from lib import ConfigLoader if __name__ == '__main__': @@ -19,7 +18,7 @@ if __name__ == '__main__': r_serv = config_loader.get_redis_conn("ARDB_DB") config_loader = None - #Set current ail version + # Set current ail version r_serv.set('ail:version', 'v2.0') # use new update_date format @@ -30,5 +29,5 @@ if __name__ == '__main__': r_serv.hset('ail:update_date', tag, date_tag) r_serv.delete('ail:update_date_{}'.format(tag)) - #Set current ail version + # Set current ail version r_serv.hset('ail:update_date', 'v2.0', datetime.datetime.now().strftime("%Y%m%d")) diff --git a/update/v2.2/Update.py b/update/v2.2/Update.py index aa2bd3f1..8d7ec233 100755 --- a/update/v2.2/Update.py +++ b/update/v2.2/Update.py @@ -2,51 +2,58 @@ # -*-coding:UTF-8 -* import os -import re import sys import time -import redis import datetime -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages')) -import Item -import Term +sys.path.append(os.environ['AIL_BIN']) +from packages import Term -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) -import ConfigLoader +from lib import ConfigLoader def rreplace(s, old, new, occurrence): li = s.rsplit(old, occurrence) return new.join(li) +def get_item_id(full_path): + return full_path.replace(PASTES_FOLDER, '', 1) + +def get_item_date(id_item): + l_dir = id_item.split('/') + return f'{l_dir[-4]}{l_dir[-3]}{l_dir[-2]}' + if __name__ == '__main__': start_deb = time.time() config_loader = ConfigLoader.ConfigLoader() + PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "pastes")) + '/' + PASTES_FOLDER = os.path.join(os.path.realpath(PASTES_FOLDER), '') + r_serv = config_loader.get_redis_conn("ARDB_DB") r_serv_term_stats = config_loader.get_redis_conn("ARDB_Trending") r_serv_termfreq = config_loader.get_redis_conn("ARDB_TermFreq") config_loader = None r_serv_term_stats.flushdb() - #convert all regex: + # convert all regex: all_regex = r_serv_termfreq.smembers('TrackedRegexSet') for regex in all_regex: - tags = list( r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(regex)) ) - mails = list( r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(regex)) ) + tags = list(r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(regex))) + mails = list(r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(regex))) new_term = regex[1:-1] - res = Term.parse_json_term_to_add({"term": new_term, "type": 'regex', "tags": tags, "mails": mails, "level": 1}, 'admin@admin.test') + res = Term.parse_json_term_to_add({"term": new_term, "type": 'regex', "tags": tags, "mails": mails, "level": 1}, + 'admin@admin.test') if res[1] == 200: term_uuid = res[0]['uuid'] list_items = r_serv_termfreq.smembers('regex_{}'.format(regex)) for paste_item in list_items: - item_id = Item.get_item_id(paste_item) - item_date = Item.get_item_date(item_id) + item_id = get_item_id(paste_item) + item_date = get_item_date(item_id) Term.add_tracked_item(term_uuid, item_id, item_date) # Invalid Tracker => remove it @@ -58,16 +65,16 @@ if __name__ == '__main__': all_tokens = r_serv_termfreq.smembers('TrackedSetTermSet') for token in all_tokens: - tags = list( r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(token)) ) - mails = list( r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(token)) ) + tags = list(r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(token))) + mails = list(r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(token))) res = Term.parse_json_term_to_add({"term": token, "type": 'word', "tags": tags, "mails": mails, "level": 1}, 'admin@admin.test') if res[1] == 200: term_uuid = res[0]['uuid'] list_items = r_serv_termfreq.smembers('tracked_{}'.format(token)) for paste_item in list_items: - item_id = Item.get_item_id(paste_item) - item_date = Item.get_item_date(item_id) + item_id = get_item_id(paste_item) + item_date = get_item_date(item_id) Term.add_tracked_item(term_uuid, item_id, item_date) # Invalid Tracker => remove it else: @@ -78,8 +85,8 @@ if __name__ == '__main__': all_set = r_serv_termfreq.smembers('TrackedSetSet') for curr_set in all_set: - tags = list( r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(curr_set)) ) - mails = list( r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(curr_set)) ) + tags = list(r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(curr_set))) + mails = list(r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(curr_set))) to_remove = ',{}'.format(curr_set.split(',')[-1]) new_set = rreplace(curr_set, to_remove, '', 1) @@ -91,8 +98,8 @@ if __name__ == '__main__': term_uuid = res[0]['uuid'] list_items = r_serv_termfreq.smembers('tracked_{}'.format(curr_set)) for paste_item in list_items: - item_id = Item.get_item_id(paste_item) - item_date = Item.get_item_date(item_id) + item_id = get_item_id(paste_item) + item_date = get_item_date(item_id) Term.add_tracked_item(term_uuid, item_id, item_date) # Invalid Tracker => remove it else: @@ -103,8 +110,8 @@ if __name__ == '__main__': r_serv_termfreq.flushdb() - #Set current ail version + # Set current ail version r_serv.set('ail:version', 'v2.2') - #Set current ail version + # Set current ail version r_serv.hset('ail:update_date', 'v2.2', datetime.datetime.now().strftime("%Y%m%d")) diff --git a/update/v2.4/Update.py b/update/v2.4/Update.py deleted file mode 100755 index 7728fab7..00000000 --- a/update/v2.4/Update.py +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* - -import os -import re -import sys -import time -import redis -import datetime - -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) -import ConfigLoader - -new_version = 'v2.4' - -if __name__ == '__main__': - - start_deb = time.time() - - config_loader = ConfigLoader.ConfigLoader() - r_serv = config_loader.get_redis_conn("ARDB_DB") - r_serv_onion = config_loader.get_redis_conn("ARDB_Onion") - config_loader = None - - r_serv_onion.sunionstore('domain_update_v2.4', 'full_onion_up', 'full_regular_up') - r_serv.set('update:nb_elem_to_convert', r_serv_onion.scard('domain_update_v2.4')) - r_serv.set('update:nb_elem_converted',0) - - # Add background update - r_serv.sadd('ail:to_update', new_version) - - #Set current ail version - r_serv.set('ail:version', new_version) - - #Set current ail version - r_serv.hset('ail:update_date', new_version, datetime.datetime.now().strftime("%Y%m%d")) diff --git a/update/v2.4/Update.sh b/update/v2.4/Update.sh deleted file mode 100755 index 308e2a3c..00000000 --- a/update/v2.4/Update.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash - -[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1; - -export PATH=$AIL_HOME:$PATH -export PATH=$AIL_REDIS:$PATH -export PATH=$AIL_ARDB:$PATH -export PATH=$AIL_BIN:$PATH -export PATH=$AIL_FLASK:$PATH - -GREEN="\\033[1;32m" -DEFAULT="\\033[0;39m" - -echo -e $GREEN"Shutting down AIL ..."$DEFAULT -bash ${AIL_BIN}/LAUNCH.sh -ks -wait - -bash ${AIL_BIN}/LAUNCH.sh -lav & -wait -echo "" - -cp ${AIL_BIN}/packages/config.cfg ${AIL_HOME}/configs/core.cfg -rm ${AIL_BIN}/packages/config.cfg - -echo "" -echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT -echo "" -python ${AIL_HOME}/update/v2.4/Update.py -wait -echo "" -echo "" - -echo "" -echo -e $GREEN"Shutting down ARDB ..."$DEFAULT -bash ${AIL_BIN}/LAUNCH.sh -ks -wait - -exit 0 diff --git a/update/v2.4/Update_domain.py b/update/v2.4/Update_domain.py deleted file mode 100755 index da4ba01d..00000000 --- a/update/v2.4/Update_domain.py +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* - -import os -import re -import sys -import time -import redis -import datetime - -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/')) -import Item -import Tag -from Cryptocurrency import cryptocurrency -from Pgp import pgp - -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) -import ConfigLoader -import Decoded -import Domain - -def update_update_stats(): - nb_updated = int(r_serv_db.get('update:nb_elem_converted')) - progress = int((nb_updated * 100) / nb_elem_to_update) - print('{}/{} updated {}%'.format(nb_updated, nb_elem_to_update, progress)) - r_serv_db.set('ail:current_background_script_stat', progress) - -def update_domain_by_item(domain_obj, item_id): - domain_name = domain_obj.get_domain_name() - # update domain tags - for tag in Tag.get_obj_tag(item_id): - if tag != 'infoleak:submission="crawler"' and tag != 'infoleak:submission="manual"': - Tag.add_tag("domain", tag, domain_name, obj_date=Item.get_item_date(item_id)) - - # update domain correlation - item_correlation = Item.get_item_all_correlation(item_id) - - for correlation_name in item_correlation: - for correlation_type in item_correlation[correlation_name]: - if correlation_name in ('pgp', 'cryptocurrency'): - for correl_value in item_correlation[correlation_name][correlation_type]: - if correlation_name=='pgp': - pgp.save_domain_correlation(domain_name, correlation_type, correl_value) - if correlation_name=='cryptocurrency': - cryptocurrency.save_domain_correlation(domain_name, correlation_type, correl_value) - if correlation_name=='decoded': - for decoded_item in item_correlation['decoded']: - Decoded.save_domain_decoded(domain_name, decoded_item) - -if __name__ == '__main__': - - start_deb = time.time() - - config_loader = ConfigLoader.ConfigLoader() - r_serv_db = config_loader.get_redis_conn("ARDB_DB") - r_serv_onion = config_loader.get_redis_conn("ARDB_Onion") - config_loader = None - - r_serv.set('ail:current_background_script', 'domain update') - - nb_elem_to_update = r_serv_db.get('update:nb_elem_to_convert') - if not nb_elem_to_update: - nb_elem_to_update = 0 - else: - nb_elem_to_update = int(nb_elem_to_update) - - while True: - domain = r_serv_onion.spop('domain_update_v2.4') - if domain is not None: - print(domain) - domain = Domain.Domain(domain) - for domain_history in domain.get_domain_history(): - - domain_item = domain.get_domain_items_crawled(epoch=domain_history[1]) # item_tag - if "items" in domain_item: - for item_dict in domain_item['items']: - update_domain_by_item(domain, item_dict['id']) - - r_serv_db.incr('update:nb_elem_converted') - update_update_stats() - - else: - r_serv.delete('update:nb_elem_to_convert') - r_serv.delete('update:nb_elem_converted') - r_serv_db.set('ail:current_background_script_stat', 100) - sys.exit(0) diff --git a/update/v2.5/Update.py b/update/v2.5/Update.py index 5b8d9726..18966d47 100755 --- a/update/v2.5/Update.py +++ b/update/v2.5/Update.py @@ -2,14 +2,12 @@ # -*-coding:UTF-8 -* import os -import re import sys import time -import redis import datetime -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) -import ConfigLoader +sys.path.append(os.environ['AIL_BIN']) +from lib import ConfigLoader new_version = 'v2.5' @@ -30,8 +28,8 @@ if __name__ == '__main__': r_serv.sadd('user_role:user_no_api', user) r_serv.sadd('user_role:read_only', user) - #Set current ail version + # Set current ail version r_serv.set('ail:version', new_version) - #Set current ail version + # Set current ail version r_serv.hset('ail:update_date', new_version, datetime.datetime.now().strftime("%Y%m%d")) diff --git a/update/v2.6/Update.py b/update/v2.6/Update.py index 4245eade..ded00cf1 100755 --- a/update/v2.6/Update.py +++ b/update/v2.6/Update.py @@ -2,19 +2,16 @@ # -*-coding:UTF-8 -* import os -import re import sys import time -import redis import datetime -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) -import ConfigLoader +sys.path.append(os.environ['AIL_BIN']) +from lib import ConfigLoader new_version = 'v2.6' if __name__ == '__main__': - start_deb = time.time() config_loader = ConfigLoader.ConfigLoader() @@ -23,8 +20,8 @@ if __name__ == '__main__': r_serv.sadd('ail:to_update', new_version) - #Set current ail version + # Set current ail version r_serv.set('ail:version', new_version) - #Set current ail version + # Set current ail version r_serv.hset('ail:update_date', new_version, datetime.datetime.now().strftime("%Y%m%d")) diff --git a/update/v2.6/Update_screenshots.py b/update/v2.6/Update_screenshots.py index 9716dc67..735d64bc 100755 --- a/update/v2.6/Update_screenshots.py +++ b/update/v2.6/Update_screenshots.py @@ -4,21 +4,19 @@ import os import sys import time -import redis -import datetime - -from hashlib import sha256 from pyfaup.faup import Faup -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/')) -import Item +sys.path.append(os.environ['AIL_BIN']) +from lib import ConfigLoader -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) -import ConfigLoader +def get_domain(item_id): + item_id = item_id.split('/') + item_id = item_id[-1] + return item_id[:-36] -def get_all_item(screenshot_sha256): - return r_serv_onion.smembers('screenshot:{}'.format(screenshot_sha256)) +def get_all_item(s_sha256): + return r_serv_onion.smembers(f'screenshot:{s_sha256}') def sanitize_domain(domain): faup.decode(domain) @@ -30,27 +28,24 @@ def sanitize_domain(domain): pass return domain_sanitized.lower() -def update_db(screenshot_sha256): - screenshot_items = get_all_item(screenshot_sha256) +def update_db(s_sha256): + screenshot_items = get_all_item(s_sha256) if screenshot_items: for item_id in screenshot_items: - item_id = item_id.replace(PASTES_FOLDER+'/', '', 1) # remove root path - domain = Item.get_domain(item_id) + item_id = item_id.replace(PASTES_FOLDER+'/', '', 1) # remove root path + domain = get_domain(item_id) domain_sanitized = sanitize_domain(domain) if domain != domain_sanitized: r_serv_onion.sadd('incorrect_domain', domain) domain = domain_sanitized - #print(item_id) - #print(domain) - - r_serv_onion.sadd('domain_screenshot:{}'.format(domain), screenshot_sha256) - r_serv_onion.sadd('screenshot_domain:{}'.format(screenshot_sha256), domain) + r_serv_onion.sadd('domain_screenshot:{}'.format(domain), s_sha256) + r_serv_onion.sadd('screenshot_domain:{}'.format(s_sha256), domain) else: pass # broken screenshot - r_serv_onion.sadd('broken_screenshot', screenshot_sha256) + r_serv_onion.sadd('broken_screenshot', s_sha256) if __name__ == '__main__': @@ -77,15 +72,15 @@ if __name__ == '__main__': if os.path.isdir(SCREENSHOT_FOLDER): for root, dirs, files in os.walk(SCREENSHOT_FOLDER, topdown=False): - #print(dirs) + # print(dirs) for name in files: nb = nb + 1 screenshot_sha256 = os.path.join(root, name) - screenshot_sha256 = screenshot_sha256[:-4] # remove .png + screenshot_sha256 = screenshot_sha256[:-4] # remove .png screenshot_sha256 = screenshot_sha256.replace(SCREENSHOT_FOLDER, '', 1) screenshot_sha256 = screenshot_sha256.replace('/', '') update_db(screenshot_sha256) - #print('Screenshot updated: {}'.format(nb)) + # print('Screenshot updated: {}'.format(nb)) if nb % 1000 == 0: r_serv_db.set('ail:current_background_script', 'screenshot updated: {}'.format(nb)) diff --git a/update/v2.7/Update.py b/update/v2.7/Update.py index fa6f539c..eed7c219 100755 --- a/update/v2.7/Update.py +++ b/update/v2.7/Update.py @@ -2,14 +2,12 @@ # -*-coding:UTF-8 -* import os -import re import sys import time -import redis import datetime -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) -import ConfigLoader +sys.path.append(os.environ['AIL_BIN']) +from lib import ConfigLoader new_version = 'v2.7' @@ -32,8 +30,8 @@ if __name__ == '__main__': r_serv.set('ail:update_v2.7:deletetagrange', 1) #### #### - #Set current ail version + # Set current ail version r_serv.set('ail:version', new_version) - #Set current ail version + # Set current ail version r_serv.hset('ail:update_date', new_version, datetime.datetime.now().strftime("%Y%m%d")) diff --git a/update/v2.7/Update_domain_tags.py b/update/v2.7/Update_domain_tags.py index 1f9f5eca..cbe1e4b6 100755 --- a/update/v2.7/Update_domain_tags.py +++ b/update/v2.7/Update_domain_tags.py @@ -4,19 +4,13 @@ import os import sys import time -import redis -import datetime - -from hashlib import sha256 from pyfaup.faup import Faup -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/')) -import Date +sys.path.append(os.environ['AIL_BIN']) +from packages import Date -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) -import ConfigLoader -import Tag +from lib import ConfigLoader def sanitize_domain(domain): faup.decode(domain) @@ -28,8 +22,23 @@ def sanitize_domain(domain): pass return domain_sanitized.lower() +def get_all_obj_tags(obj_type): + return list(r_serv_tags.smembers(f'list_tags:{obj_type}')) + +def add_global_tag(tag, object_type=None): + r_serv_tags.sadd('list_tags', tag) + if object_type: + r_serv_tags.sadd('list_tags:{}'.format(object_type), tag) + +def get_obj_tag(object_id): + res = r_serv_metadata.smembers('tag:{}'.format(object_id)) + if res: + return list(res) + else: + return [] + def delete_domain_tag_daterange(): - all_domains_tags = Tag.get_all_obj_tags('domain') + all_domains_tags = get_all_obj_tags('domain') nb_updated = 0 nb_to_update = len(all_domains_tags) if nb_to_update == 0: @@ -49,13 +58,13 @@ def update_domain_tags(domain): r_serv_onion.sadd('incorrect_domain', domain) domain = domain_sanitized - domain_tags = Tag.get_obj_tag(domain) + domain_tags = get_obj_tag(domain) for tag in domain_tags: # delete incorrect tags if tag == 'infoleak:submission="crawler"' or tag == 'infoleak:submission="manual"': r_serv_metadata.srem('tag:{}'.format(domain), tag) else: - Tag.add_global_tag(tag, object_type='domain') + add_global_tag(tag, object_type='domain') r_serv_tags.sadd('{}:{}'.format('domain', tag), domain) def update_progress(refresh_time, nb_updated, nb_elem_to_update): @@ -88,6 +97,7 @@ def update_db(): r_serv_onion.sort('full_onion_up', alpha=True) r_serv_onion.sort('full_regular_up', alpha=True) + if __name__ == '__main__': start_deb = time.time() @@ -113,6 +123,5 @@ if __name__ == '__main__': r_serv_db.set('ail:current_background_script_stat', 100) - end = time.time() print('ALL domains tags updated in {} s'.format(end - start_deb)) diff --git a/update/v3.0/Update.py b/update/v3.0/Update.py index 2e86849b..9d393f6f 100755 --- a/update/v3.0/Update.py +++ b/update/v3.0/Update.py @@ -3,27 +3,20 @@ import os import sys -import time -import redis -import argparse -import datetime -import configparser -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) -import ConfigLoader +sys.path.append(os.environ['AIL_HOME']) +################################## +# Import Project packages +################################## +from update.bin.old_ail_updater import AIL_Updater + +class Updater(AIL_Updater): + """default Updater.""" + + def __init__(self, version): + super(Updater, self).__init__(version) -new_version = 'v3.0' if __name__ == '__main__': - - start_deb = time.time() - - config_loader = ConfigLoader.ConfigLoader() - r_serv = config_loader.get_redis_conn("ARDB_DB") - config_loader = None - - #Set current ail version - r_serv.set('ail:version', new_version) - - #Set current ail version - r_serv.hset('ail:update_date', new_version, datetime.datetime.now().strftime("%Y%m%d")) + updater = Updater('v3.0') + updater.run_update() diff --git a/update/v3.1.1/Update.py b/update/v3.1.1/Update.py index 6b56e3da..a5a16751 100755 --- a/update/v3.1.1/Update.py +++ b/update/v3.1.1/Update.py @@ -3,37 +3,20 @@ import os import sys -import time -import redis -import argparse -import datetime -import configparser -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) -import ConfigLoader +sys.path.append(os.environ['AIL_HOME']) +################################## +# Import Project packages +################################## +from update.bin.old_ail_updater import AIL_Updater + +class Updater(AIL_Updater): + """default Updater.""" + + def __init__(self, version): + super(Updater, self).__init__(version) -new_version = 'v3.1.1' if __name__ == '__main__': - - start_deb = time.time() - - config_loader = ConfigLoader.ConfigLoader() - r_serv_db = config_loader.get_redis_conn("ARDB_DB") - config_loader = None - - #### NEW EXPORTER - # remove old tags errors - #r_serv_db.delete('mess_not_saved_export') - - # move solo tags to export in tags_db - #all_misp_tags = r_serv_db.smembers('whitelist_misp') - #all_hive_tags = r_serv_db.smembers('whitelist_hive') - # # TODO: save them in tags db - #### NEW EXPORTER - - #Set current ail version - r_serv_db.set('ail:version', new_version) - - #Set current ail version - r_serv_db.hset('ail:update_date', new_version, datetime.datetime.now().strftime("%Y%m%d")) + updater = Updater('v3.1.1') + updater.run_update() diff --git a/update/v3.1/Update.py b/update/v3.1/Update.py index e54c1d19..6c4c4546 100755 --- a/update/v3.1/Update.py +++ b/update/v3.1/Update.py @@ -3,27 +3,21 @@ import os import sys -import time -import redis -import argparse -import datetime -import configparser -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) -import ConfigLoader +sys.path.append(os.environ['AIL_HOME']) +################################## +# Import Project packages +################################## +from update.bin.old_ail_updater import AIL_Updater + +class Updater(AIL_Updater): + """default Updater.""" + + def __init__(self, version): + super(Updater, self).__init__(version) -new_version = 'v3.1' if __name__ == '__main__': + updater = Updater('v3.1') + updater.run_update() - start_deb = time.time() - - config_loader = ConfigLoader.ConfigLoader() - r_serv = config_loader.get_redis_conn("ARDB_DB") - config_loader = None - - #Set current ail version - r_serv.set('ail:version', new_version) - - #Set current ail version - r_serv.hset('ail:update_date', new_version, datetime.datetime.now().strftime("%Y%m%d")) diff --git a/update/v3.2/Update.py b/update/v3.2/Update.py index fa06c8fe..086ed098 100755 --- a/update/v3.2/Update.py +++ b/update/v3.2/Update.py @@ -3,37 +3,20 @@ import os import sys -import time -import redis -import argparse -import datetime -import configparser -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) -import ConfigLoader +sys.path.append(os.environ['AIL_HOME']) +################################## +# Import Project packages +################################## +from update.bin.old_ail_updater import AIL_Updater + +class Updater(AIL_Updater): + """default Updater.""" + + def __init__(self, version): + super(Updater, self).__init__(version) -new_version = 'v3.2' if __name__ == '__main__': - - start_deb = time.time() - - config_loader = ConfigLoader.ConfigLoader() - r_serv_db = config_loader.get_redis_conn("ARDB_DB") - config_loader = None - - #### NEW EXPORTER - # remove old tags errors - #r_serv_db.delete('mess_not_saved_export') - - # move solo tags to export in tags_db - #all_misp_tags = r_serv_db.smembers('whitelist_misp') - #all_hive_tags = r_serv_db.smembers('whitelist_hive') - # # TODO: save them in tags db - #### NEW EXPORTER - - #Set current ail version - r_serv_db.set('ail:version', new_version) - - #Set current ail version - r_serv_db.hset('ail:update_date', new_version, datetime.datetime.now().strftime("%Y%m%d")) + updater = Updater('v3.2') + updater.run_update() diff --git a/update/v3.3/Update.py b/update/v3.3/Update.py index 39d1371b..5f0efc78 100755 --- a/update/v3.3/Update.py +++ b/update/v3.3/Update.py @@ -3,27 +3,20 @@ import os import sys -import time -import redis -import argparse -import datetime -import configparser -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) -import ConfigLoader +sys.path.append(os.environ['AIL_HOME']) +################################## +# Import Project packages +################################## +from update.bin.old_ail_updater import AIL_Updater + +class Updater(AIL_Updater): + """default Updater.""" + + def __init__(self, version): + super(Updater, self).__init__(version) -new_version = 'v3.3' if __name__ == '__main__': - - start_deb = time.time() - - config_loader = ConfigLoader.ConfigLoader() - r_serv_db = config_loader.get_redis_conn("ARDB_DB") - config_loader = None - - #Set current ail version - r_serv_db.set('ail:version', new_version) - - #Set current ail version - r_serv_db.hset('ail:update_date', new_version, datetime.datetime.now().strftime("%Y%m%d")) + updater = Updater('v3.3') + updater.run_update() diff --git a/update/v3.4/Update.py b/update/v3.4/Update.py index ab88cdbc..d4308551 100755 --- a/update/v3.4/Update.py +++ b/update/v3.4/Update.py @@ -2,17 +2,13 @@ # -*-coding:UTF-8 -* import os -import re import sys -import time -import redis -import datetime -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) -import ConfigLoader - -sys.path.append(os.path.join(os.environ['AIL_HOME'], 'update', 'bin')) -from old_ail_updater import AIL_Updater +sys.path.append(os.environ['AIL_HOME']) +################################## +# Import Project packages +################################## +from update.bin.old_ail_updater import AIL_Updater class Updater(AIL_Updater): """default Updater.""" @@ -27,12 +23,12 @@ class Updater(AIL_Updater): """ self.r_serv_onion.sunionstore('domain_update_v3.4', 'full_onion_up', 'full_regular_up') self.r_serv.set('update:nb_elem_to_convert', self.r_serv_onion.scard('domain_update_v3.4')) - self.r_serv.set('update:nb_elem_converted',0) + self.r_serv.set('update:nb_elem_converted', 0) # Add background update self.r_serv.sadd('ail:to_update', self.version) -if __name__ == '__main__': +if __name__ == '__main__': updater = Updater('v3.4') updater.run_update() diff --git a/update/v3.4/Update_domain.py b/update/v3.4/Update_domain.py index 9e096c48..99cd362a 100755 --- a/update/v3.4/Update_domain.py +++ b/update/v3.4/Update_domain.py @@ -2,33 +2,95 @@ # -*-coding:UTF-8 -* import os -import re import sys import time -import redis -import datetime -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) -import ConfigLoader -import Domain +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib import ConfigLoader +from lib.objects.Items import Item +from lib import Domain + +def get_domain_type(domain_name): + if str(domain_name).endswith('.onion'): + return 'onion' + else: + return 'regular' + +def add_domain_language(domain_name, language): + language = language.split('-')[0] + domain_type = get_domain_type(domain_name) + r_serv_onion.sadd('all_domains_languages', language) + r_serv_onion.sadd(f'all_domains_languages:{domain_type}', language) + r_serv_onion.sadd(f'language:domains:{domain_type}:{language}', domain_name) + r_serv_onion.sadd(f'domain:language:{domain_name}', language) + +def add_domain_languages_by_item_id(domain_name, item_id): + item = Item(item_id) + for lang in item.get_languages(): + add_domain_language(domain_name, lang.language) def update_update_stats(): nb_updated = int(r_serv_db.get('update:nb_elem_converted')) progress = int((nb_updated * 100) / nb_elem_to_update) - print('{}/{} updated {}%'.format(nb_updated, nb_elem_to_update, progress)) + print(f'{nb_updated}/{nb_elem_to_update} updated {progress}%') r_serv_db.set('ail:current_background_script_stat', progress) def update_domain_language(domain_obj, item_id): domain_name = domain_obj.get_domain_name() - Domain.add_domain_languages_by_item_id(domain_name, item_id) + add_domain_languages_by_item_id(domain_name, item_id) + +def get_domain_history(domain_type, domain_name): + return r_serv_onion.zrange(f'crawler_history_{domain_type}:{domain_name}:80', 0, -1, withscores=True) + + +def get_item_children(item_id): + return r_serv_metadata.smembers(f'paste_children:{item_id}') + +def get_domain_items(domain_name, root_item_id): + dom_item = get_domain_item_children(domain_name, root_item_id) + dom_item.append(root_item_id) + return dom_item + +def is_item_in_domain(domain_name, item_id): + is_in_domain = False + domain_length = len(domain_name) + if len(item_id) > (domain_length+48): + if item_id[-36-domain_length:-36] == domain_name: + is_in_domain = True + return is_in_domain + +def get_domain_item_children(domain_name, root_item_id): + all_items = [] + for item_id in get_item_children(root_item_id): + if is_item_in_domain(domain_name, item_id): + all_items.append(item_id) + all_items.extend(get_domain_item_children(domain_name, item_id)) + return all_items + +def get_domain_crawled_item_root(domain_name, domain_type, epoch): + res = r_serv_onion.zrevrangebyscore(f'crawler_history_{domain_type}:{domain_name}:80', int(epoch), int(epoch)) + return {"root_item": res[0], "epoch": int(epoch)} + +def get_domain_items_crawled(domain_name, domain_type, epoch): + item_crawled = [] + item_root = get_domain_crawled_item_root(domain_name, domain_type, epoch) + if item_root: + if item_root['root_item'] != str(item_root['epoch']) and item_root['root_item']: + for item_id in get_domain_items(domain_name, item_root['root_item']): + item_crawled.append(item_id) + return item_crawled + if __name__ == '__main__': start_deb = time.time() - config_loader = ConfigLoader.ConfigLoader() r_serv_db = config_loader.get_redis_conn("ARDB_DB") r_serv_onion = config_loader.get_redis_conn("ARDB_Onion") + r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") config_loader = None r_serv_db.set('ail:current_background_script', 'domain languages update') @@ -39,18 +101,19 @@ if __name__ == '__main__': else: nb_elem_to_update = int(nb_elem_to_update) - #Domain._delete_all_domains_languages() + # _delete_all_domains_languages() while True: domain = r_serv_onion.spop('domain_update_v3.4') if domain is not None: print(domain) + domain = str(domain) + domain_t = get_domain_type(domain) domain = Domain.Domain(domain) - for domain_history in domain.get_domain_history(): - domain_item = domain.get_domain_items_crawled(epoch=domain_history[1]) # item_tag - if "items" in domain_item: - for item_dict in domain_item['items']: - update_domain_language(domain, item_dict['id']) + for domain_history in get_domain_history(domain_t, domain): + domain_items = get_domain_items_crawled(domain, domain_t, domain_history[1]) + for id_item in domain_items: + update_domain_language(domain, id_item) r_serv_db.incr('update:nb_elem_converted') update_update_stats() diff --git a/update/v3.5/Update.py b/update/v3.5/Update.py index 7be17cbb..651fe8ea 100755 --- a/update/v3.5/Update.py +++ b/update/v3.5/Update.py @@ -2,17 +2,13 @@ # -*-coding:UTF-8 -* import os -import re import sys -import time -import redis -import datetime -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) -import ConfigLoader - -sys.path.append(os.path.join(os.environ['AIL_HOME'], 'update', 'bin')) -from old_ail_updater import AIL_Updater +sys.path.append(os.environ['AIL_HOME']) +################################## +# Import Project packages +################################## +from update.bin.old_ail_updater import AIL_Updater class Updater(AIL_Updater): """default Updater.""" @@ -20,7 +16,7 @@ class Updater(AIL_Updater): def __init__(self, version): super(Updater, self).__init__(version) -if __name__ == '__main__': +if __name__ == '__main__': updater = Updater('v3.5') updater.run_update() diff --git a/update/v3.6/Update.py b/update/v3.6/Update.py index 505d90ef..8d7715de 100755 --- a/update/v3.6/Update.py +++ b/update/v3.6/Update.py @@ -2,17 +2,13 @@ # -*-coding:UTF-8 -* import os -import re import sys -import time -import redis -import datetime -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) -import ConfigLoader - -sys.path.append(os.path.join(os.environ['AIL_HOME'], 'update', 'bin')) -from old_ail_updater import AIL_Updater +sys.path.append(os.environ['AIL_HOME']) +################################## +# Import Project packages +################################## +from update.bin.old_ail_updater import AIL_Updater class Updater(AIL_Updater): """default Updater.""" @@ -20,7 +16,7 @@ class Updater(AIL_Updater): def __init__(self, version): super(Updater, self).__init__(version) -if __name__ == '__main__': +if __name__ == '__main__': updater = Updater('v3.6') updater.run_update() diff --git a/update/v3.7/Update.py b/update/v3.7/Update.py index ff75e790..f1f988e8 100755 --- a/update/v3.7/Update.py +++ b/update/v3.7/Update.py @@ -2,18 +2,14 @@ # -*-coding:UTF-8 -* import os -import re import sys -import time -import redis -import datetime -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) -import ConfigLoader -import Tracker - -sys.path.append(os.path.join(os.environ['AIL_HOME'], 'update', 'bin')) -from old_ail_updater import AIL_Updater +sys.path.append(os.environ['AIL_HOME']) +################################## +# Import Project packages +################################## +from update.bin.old_ail_updater import AIL_Updater +from lib import Tracker class Updater(AIL_Updater): """default Updater.""" @@ -38,7 +34,7 @@ class Updater(AIL_Updater): # Add background update self.r_serv.sadd('ail:to_update', self.version) -if __name__ == '__main__': +if __name__ == '__main__': updater = Updater('v3.7') updater.run_update() diff --git a/update/v3.7/Update_trackers.py b/update/v3.7/Update_trackers.py index 1474e49b..b84f06e8 100644 --- a/update/v3.7/Update_trackers.py +++ b/update/v3.7/Update_trackers.py @@ -2,29 +2,31 @@ # -*-coding:UTF-8 -* import os -import re import sys import time -import redis -import datetime -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) -import ConfigLoader -import Tracker +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib import ConfigLoader +from lib import Tracker def update_update_stats(): nb_updated = int(r_serv_db.get('update:nb_elem_converted')) progress = int((nb_updated * 100) / nb_elem_to_update) - print('{}/{} updated {}%'.format(nb_updated, nb_elem_to_update, progress)) + print(f'{nb_updated}/{nb_elem_to_update} updated {progress}%') r_serv_db.set('ail:current_background_script_stat', progress) -if __name__ == '__main__': +if __name__ == '__main__': start_deb = time.time() config_loader = ConfigLoader.ConfigLoader() r_serv_db = config_loader.get_redis_conn("ARDB_DB") + r_serv_tracker = config_loader.get_db_conn("Kvrocks_DB") config_loader = None + Tracker.r_serv_tracker = r_serv_tracker r_serv_db.set('ail:current_background_script', 'trackers update') @@ -43,7 +45,6 @@ if __name__ == '__main__': # MAP TRACKER - ITEM_ID Tracker.fix_tracker_item_link(tracker_uuid) - r_serv_db.incr('update:nb_elem_converted') update_update_stats() diff --git a/update/v4.0/Update.py b/update/v4.0/Update.py index 0c9fa88f..b609ebb6 100755 --- a/update/v4.0/Update.py +++ b/update/v4.0/Update.py @@ -2,17 +2,13 @@ # -*-coding:UTF-8 -* import os -import re import sys -import time -import redis -import datetime -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) -import ConfigLoader - -sys.path.append(os.path.join(os.environ['AIL_HOME'], 'update', 'bin')) -from old_ail_updater import AIL_Updater +sys.path.append(os.environ['AIL_HOME']) +################################## +# Import Project packages +################################## +from update.bin.old_ail_updater import AIL_Updater class Updater(AIL_Updater): """default Updater.""" @@ -20,7 +16,7 @@ class Updater(AIL_Updater): def __init__(self, version): super(Updater, self).__init__(version) -if __name__ == '__main__': +if __name__ == '__main__': updater = Updater('v4.0') updater.run_update() diff --git a/update/v4.1/Update.py b/update/v4.1/Update.py index 696fd719..f7a7a72a 100755 --- a/update/v4.1/Update.py +++ b/update/v4.1/Update.py @@ -2,17 +2,14 @@ # -*-coding:UTF-8 -* import os -import re import sys -import time import redis -import datetime -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) -import ConfigLoader - -sys.path.append(os.path.join(os.environ['AIL_HOME'], 'update', 'bin')) -from old_ail_updater import AIL_Updater +sys.path.append(os.environ['AIL_HOME']) +################################## +# Import Project packages +################################## +from update.bin.old_ail_updater import AIL_Updater class Updater(AIL_Updater): """default Updater.""" @@ -21,17 +18,14 @@ class Updater(AIL_Updater): super(Updater, self).__init__(version) def update(self): - r_tracking = redis.StrictRedis( host='localhost', - port=6382, - db=2, - decode_responses=True ) - + r_tracking = redis.StrictRedis(host='localhost', + port=6382, + db=2, + decode_responses=True) # FLUSH OLD DB r_tracking.flushdb() + if __name__ == '__main__': - updater = Updater('v4.1') - - updater.run_update() diff --git a/update/v4.2/Update.py b/update/v4.2/Update.py index 47c66170..708009b2 100755 --- a/update/v4.2/Update.py +++ b/update/v4.2/Update.py @@ -2,17 +2,13 @@ # -*-coding:UTF-8 -* import os -import re import sys -import time -import redis -import datetime -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) -import ConfigLoader - -sys.path.append(os.path.join(os.environ['AIL_HOME'], 'update', 'bin')) -from old_ail_updater import AIL_Updater +sys.path.append(os.environ['AIL_HOME']) +################################## +# Import Project packages +################################## +from update.bin.old_ail_updater import AIL_Updater class Updater(AIL_Updater): """default Updater.""" @@ -20,7 +16,7 @@ class Updater(AIL_Updater): def __init__(self, version): super(Updater, self).__init__(version) -if __name__ == '__main__': +if __name__ == '__main__': updater = Updater('v4.2') updater.run_update()