From ff2b620647f4b60f0c148555d257131dd4eadf59 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Wed, 12 Jul 2023 11:45:49 +0200 Subject: [PATCH] chg: [update] remove old updates < 5.0 --- update/bin/Update_ARDB.sh | 18 -- update/bin/old_ail_updater.py | 50 ------ update/v1.0/Update.sh | 15 -- update/v1.5/Update-ARDB_Metadata.py | 165 ------------------ update/v1.5/Update-ARDB_Onions.py | 129 -------------- update/v1.5/Update-ARDB_Onions_screenshots.py | 117 ------------- update/v1.5/Update-ARDB_Tags.py | 135 -------------- update/v1.5/Update-ARDB_Tags_background.py | 70 -------- update/v1.5/Update.py | 54 ------ update/v1.5/Update.sh | 60 ------- update/v1.7/Update.py | 25 --- update/v1.7/Update.sh | 65 ------- update/v2.0/Update.py | 33 ---- update/v2.0/Update.sh | 75 -------- update/v2.2/Update.py | 118 ------------- update/v2.2/Update.sh | 39 ----- update/v2.5/Update.py | 35 ---- update/v2.5/Update.sh | 39 ----- update/v2.6/Update.py | 27 --- update/v2.6/Update.sh | 39 ----- update/v2.6/Update_screenshots.py | 90 ---------- update/v2.7/Update.py | 37 ---- update/v2.7/Update.sh | 39 ----- update/v2.7/Update_domain_tags.py | 127 -------------- update/v3.0/Update.py | 22 --- update/v3.0/Update.sh | 43 ----- update/v3.1.1/Update.py | 22 --- update/v3.1.1/Update.sh | 52 ------ update/v3.1/Update.py | 23 --- update/v3.1/Update.sh | 46 ----- update/v3.2/Update.py | 22 --- update/v3.2/Update.sh | 52 ------ update/v3.3/Update.py | 22 --- update/v3.3/Update.sh | 54 ------ update/v3.4/Update.py | 34 ---- update/v3.4/Update.sh | 54 ------ update/v3.4/Update_domain.py | 121 ------------- update/v3.5/Update.py | 22 --- update/v3.5/Update.sh | 35 ---- update/v3.6/Update.py | 22 --- update/v3.6/Update.sh | 39 ----- update/v3.7/Update.py | 40 ----- update/v3.7/Update.sh | 44 ----- update/v3.7/Update_trackers.py | 53 ------ update/v4.0/Update.py | 22 --- update/v4.0/Update.sh | 29 --- update/v4.1/Update.py | 31 ---- update/v4.1/Update.sh | 25 --- update/v4.2.1/Update.py | 22 --- update/v4.2.1/Update.sh | 29 --- update/v4.2/Update.py | 22 --- update/v4.2/Update.sh | 33 ---- 52 files changed, 2636 deletions(-) delete mode 100755 update/bin/Update_ARDB.sh delete mode 100755 update/bin/old_ail_updater.py delete mode 100755 update/v1.0/Update.sh delete mode 100755 update/v1.5/Update-ARDB_Metadata.py delete mode 100755 update/v1.5/Update-ARDB_Onions.py delete mode 100755 update/v1.5/Update-ARDB_Onions_screenshots.py delete mode 100755 update/v1.5/Update-ARDB_Tags.py delete mode 100755 update/v1.5/Update-ARDB_Tags_background.py delete mode 100755 update/v1.5/Update.py delete mode 100755 update/v1.5/Update.sh delete mode 100755 update/v1.7/Update.py delete mode 100755 update/v1.7/Update.sh delete mode 100755 update/v2.0/Update.py delete mode 100755 update/v2.0/Update.sh delete mode 100755 update/v2.2/Update.py delete mode 100755 update/v2.2/Update.sh delete mode 100755 update/v2.5/Update.py delete mode 100755 update/v2.5/Update.sh delete mode 100755 update/v2.6/Update.py delete mode 100755 update/v2.6/Update.sh delete mode 100755 update/v2.6/Update_screenshots.py delete mode 100755 update/v2.7/Update.py delete mode 100755 update/v2.7/Update.sh delete mode 100755 update/v2.7/Update_domain_tags.py delete mode 100755 update/v3.0/Update.py delete mode 100755 update/v3.0/Update.sh delete mode 100755 update/v3.1.1/Update.py delete mode 100755 update/v3.1.1/Update.sh delete mode 100755 update/v3.1/Update.py delete mode 100755 update/v3.1/Update.sh delete mode 100755 update/v3.2/Update.py delete mode 100755 update/v3.2/Update.sh delete mode 100755 update/v3.3/Update.py delete mode 100755 update/v3.3/Update.sh delete mode 100755 update/v3.4/Update.py delete mode 100755 update/v3.4/Update.sh delete mode 100755 update/v3.4/Update_domain.py delete mode 100755 update/v3.5/Update.py delete mode 100755 update/v3.5/Update.sh delete mode 100755 update/v3.6/Update.py delete mode 100755 update/v3.6/Update.sh delete mode 100755 update/v3.7/Update.py delete mode 100755 update/v3.7/Update.sh delete mode 100644 update/v3.7/Update_trackers.py delete mode 100755 update/v4.0/Update.py delete mode 100755 update/v4.0/Update.sh delete mode 100755 update/v4.1/Update.py delete mode 100755 update/v4.1/Update.sh delete mode 100755 update/v4.2.1/Update.py delete mode 100755 update/v4.2.1/Update.sh delete mode 100755 update/v4.2/Update.py delete mode 100755 update/v4.2/Update.sh diff --git a/update/bin/Update_ARDB.sh b/update/bin/Update_ARDB.sh deleted file mode 100755 index 2544973e..00000000 --- a/update/bin/Update_ARDB.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -echo "Killing all screens ..." -bash -c "bash ../../bin/LAUNCH.sh -k" -echo "" -echo "Updating ARDB ..." -pushd ../../ -rm -r ardb -pushd ardb/ -git clone https://github.com/yinqiwen/ardb.git -git checkout 0.10 || exit 1 -make || exit 1 -popd -popd -echo "ARDB Updated" -echo "" - -exit 0 diff --git a/update/bin/old_ail_updater.py b/update/bin/old_ail_updater.py deleted file mode 100755 index 14833679..00000000 --- a/update/bin/old_ail_updater.py +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* - -import os -import sys -import time -import datetime - -sys.path.append(os.environ['AIL_BIN']) -################################## -# Import Project packages -################################## -from lib import ConfigLoader - -class AIL_Updater(object): - """docstring for AIL_Updater.""" - - def __init__(self, new_version): - self.version = new_version - self.start_time = time.time() - - self.config = ConfigLoader.ConfigLoader() - self.r_serv = self.config.get_redis_conn("ARDB_DB") - - self.f_version = float(self.version[1:]) - self.current_f_version = self.r_serv.get('ail:version') - if self.current_f_version: - self.current_f_version = float(self.current_f_version[1:]) - else: - self.current_f_version = 0 - - def update(self): - """ - AIL DB update - """ - pass - - def end_update(self): - """ - Update DB version - """ - # Set current ail version - self.r_serv.hset('ail:update_date', self.version, datetime.datetime.now().strftime("%Y%m%d")) - # Set current ail version - if self.f_version > self.current_f_version: - self.r_serv.set('ail:version', self.version) - - def run_update(self): - self.update() - self.end_update() diff --git a/update/v1.0/Update.sh b/update/v1.0/Update.sh deleted file mode 100755 index 1d3691d0..00000000 --- a/update/v1.0/Update.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -YELLOW="\\033[1;33m" -DEFAULT="\\033[0;39m" - -echo -e $YELLOW"\t" -echo -e "* ------------------------------------------------------------------" -echo -e "\t" -echo -e " - - - - - - - - PLEASE RELAUNCH AIL - - - - - - - - " -echo -e "\t" -echo -e "* ------------------------------------------------------------------" -echo -e "\t" -echo -e "\t"$DEFAULT - -# fix invalid Updater version (kill parent): -kill -SIGUSR1 `ps --pid $$ -oppid=`; exit diff --git a/update/v1.5/Update-ARDB_Metadata.py b/update/v1.5/Update-ARDB_Metadata.py deleted file mode 100755 index f678f74e..00000000 --- a/update/v1.5/Update-ARDB_Metadata.py +++ /dev/null @@ -1,165 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* - -import os -import sys -import time - -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) -from lib import ConfigLoader - -def update_tracked_terms(main_key, tracked_container_key): - for tracked_item in r_serv_term.smembers(main_key): - all_items = r_serv_term.smembers(tracked_container_key.format(tracked_item)) - for item_path in all_items: - if PASTES_FOLDER in item_path: - new_item_path = item_path.replace(PASTES_FOLDER, '', 1) - r_serv_term.sadd(tracked_container_key.format(tracked_item), new_item_path) - r_serv_term.srem(tracked_container_key.format(tracked_item), item_path) - -def update_hash_item(has_type): - #get all hash items: - all_hash_items = r_serv_tag.smembers('infoleak:automatic-detection=\"{}\"'.format(has_type)) - for item_path in all_hash_items: - if PASTES_FOLDER in item_path: - base64_key = '{}_paste:{}'.format(has_type, item_path) - hash_key = 'hash_paste:{}'.format(item_path) - - if r_serv_metadata.exists(base64_key): - new_base64_key = base64_key.replace(PASTES_FOLDER, '', 1) - res = r_serv_metadata.renamenx(base64_key, new_base64_key) - if res == 0: - print('same key, double name: {}'.format(item_path)) - # fusion - all_key = r_serv_metadata.smembers(base64_key) - for elem in all_key: - r_serv_metadata.sadd(new_base64_key, elem) - r_serv_metadata.srem(base64_key, elem) - - if r_serv_metadata.exists(hash_key): - new_hash_key = hash_key.replace(PASTES_FOLDER, '', 1) - res = r_serv_metadata.renamenx(hash_key, new_hash_key) - if res == 0: - print('same key, double name: {}'.format(item_path)) - # fusion - all_key = r_serv_metadata.smembers(hash_key) - for elem in all_key: - r_serv_metadata.sadd(new_hash_key, elem) - r_serv_metadata.srem(hash_key, elem) - -if __name__ == '__main__': - - start_deb = time.time() - - config_loader = ConfigLoader.ConfigLoader() - - PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "pastes")) + '/' - - r_serv = config_loader.get_redis_conn("ARDB_DB") - r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") - r_serv_tag = config_loader.get_redis_conn("ARDB_Tags") - r_serv_term = config_loader.get_redis_conn("ARDB_TermFreq") - r_serv_onion = config_loader.get_redis_conn("ARDB_Onion") - config_loader = None - - r_serv.set('ail:current_background_script', 'metadata') - - ## Update metadata ## - print('Updating ARDB_Metadata ...') - index = 0 - start = time.time() - - #update stats - r_serv.set('ail:current_background_script_stat', 0) - - # Update base64 - update_hash_item('base64') - - #update stats - r_serv.set('ail:current_background_script_stat', 20) - # Update binary - update_hash_item('binary') - - #update stats - r_serv.set('ail:current_background_script_stat', 40) - # Update binary - update_hash_item('hexadecimal') - - #update stats - r_serv.set('ail:current_background_script_stat', 60) - - total_onion = r_serv_tag.scard('infoleak:submission=\"crawler\"') - nb_updated = 0 - last_progress = 0 - - # Update onion metadata - all_crawled_items = r_serv_tag.smembers('infoleak:submission=\"crawler\"') - for item_path in all_crawled_items: - domain = None - if PASTES_FOLDER in item_path: - old_item_metadata = 'paste_metadata:{}'.format(item_path) - item_path = item_path.replace(PASTES_FOLDER, '', 1) - new_item_metadata = 'paste_metadata:{}'.format(item_path) - res = r_serv_metadata.renamenx(old_item_metadata, new_item_metadata) - #key already exist - if res == 0: - r_serv_metadata.delete(old_item_metadata) - - # update domain port - domain = r_serv_metadata.hget(new_item_metadata, 'domain') - if domain: - if domain[-3:] != ':80': - r_serv_metadata.hset(new_item_metadata, 'domain', '{}:80'.format(domain)) - super_father = r_serv_metadata.hget(new_item_metadata, 'super_father') - if super_father: - if PASTES_FOLDER in super_father: - r_serv_metadata.hset(new_item_metadata, 'super_father', super_father.replace(PASTES_FOLDER, '', 1)) - father = r_serv_metadata.hget(new_item_metadata, 'father') - if father: - if PASTES_FOLDER in father: - r_serv_metadata.hset(new_item_metadata, 'father', father.replace(PASTES_FOLDER, '', 1)) - - nb_updated += 1 - progress = int((nb_updated * 30) /total_onion) - print('{}/{} updated {}%'.format(nb_updated, total_onion, progress + 60)) - # update progress stats - if progress != last_progress: - r_serv.set('ail:current_background_script_stat', progress + 60) - last_progress = progress - - #update stats - r_serv.set('ail:current_background_script_stat', 90) - - ## update tracked term/set/regex - # update tracked term - update_tracked_terms('TrackedSetTermSet', 'tracked_{}') - - #update stats - r_serv.set('ail:current_background_script_stat', 93) - # update tracked set - update_tracked_terms('TrackedSetSet', 'set_{}') - - #update stats - r_serv.set('ail:current_background_script_stat', 96) - # update tracked regex - update_tracked_terms('TrackedRegexSet', 'regex_{}') - - #update stats - r_serv.set('ail:current_background_script_stat', 100) - ## - - end = time.time() - - print('Updating ARDB_Metadata Done => {} paths: {} s'.format(index, end - start)) - print() - - r_serv.sadd('ail:update_v1.5', 'metadata') - - ## - #Key, Dynamic Update - ## - #paste_children - #nb_seen_hash, base64_hash, binary_hash - #paste_onion_external_links - #misp_events, hive_cases - ## diff --git a/update/v1.5/Update-ARDB_Onions.py b/update/v1.5/Update-ARDB_Onions.py deleted file mode 100755 index 199ee194..00000000 --- a/update/v1.5/Update-ARDB_Onions.py +++ /dev/null @@ -1,129 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* - -import os -import sys -import time -import datetime - -sys.path.append(os.environ['AIL_BIN']) -from lib import ConfigLoader - -def substract_date(date_from, date_to): - date_from = datetime.date(int(date_from[0:4]), int(date_from[4:6]), int(date_from[6:8])) - date_to = datetime.date(int(date_to[0:4]), int(date_to[4:6]), int(date_to[6:8])) - delta = date_to - date_from # timedelta - l_date = [] - for i in range(delta.days + 1): - date = date_from + datetime.timedelta(i) - l_date.append( date.strftime('%Y%m%d') ) - return l_date - -def get_date_epoch(date): - return int(datetime.datetime(int(date[0:4]), int(date[4:6]), int(date[6:8])).timestamp()) - -def get_domain_root_from_paste_childrens(item_father, domain): - item_children = r_serv_metadata.smembers('paste_children:{}'.format(item_father)) - domain_root = '' - for item_path in item_children: - # remove absolute_path - if PASTES_FOLDER in item_path: - r_serv_metadata.srem('paste_children:{}'.format(item_father), item_path) - item_path = item_path.replace(PASTES_FOLDER, '', 1) - r_serv_metadata.sadd('paste_children:{}'.format(item_father), item_path) - if domain in item_path: - domain_root = item_path - return domain_root - - -if __name__ == '__main__': - - start_deb = time.time() - - config_loader = ConfigLoader.ConfigLoader() - - PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "pastes")) + '/' - - r_serv = config_loader.get_redis_conn("ARDB_DB") - r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") - r_serv_tag = config_loader.get_redis_conn("ARDB_Tags") - r_serv_onion = config_loader.get_redis_conn("ARDB_Onion") - config_loader = None - - r_serv.set('ail:current_background_script', 'onions') - r_serv.set('ail:current_background_script_stat', 0) - - ## Update Onion ## - print('Updating ARDB_Onion ...') - index = 0 - start = time.time() - - # clean down domain from db - date_from = '20180929' - date_today = datetime.date.today().strftime("%Y%m%d") - for date in substract_date(date_from, date_today): - - onion_down = r_serv_onion.smembers('onion_down:{}'.format(date)) - #print(onion_down) - for onion_domain in onion_down: - if not r_serv_onion.sismember('full_onion_up', onion_domain): - # delete history - all_onion_history = r_serv_onion.lrange('onion_history:{}'.format(onion_domain), 0 ,-1) - if all_onion_history: - for date_history in all_onion_history: - #print('onion_history:{}:{}'.format(onion_domain, date_history)) - r_serv_onion.delete('onion_history:{}:{}'.format(onion_domain, date_history)) - r_serv_onion.delete('onion_history:{}'.format(onion_domain)) - - #stats - total_domain = r_serv_onion.scard('full_onion_up') - nb_updated = 0 - last_progress = 0 - - # clean up domain - all_domain_up = r_serv_onion.smembers('full_onion_up') - for onion_domain in all_domain_up: - # delete history - all_onion_history = r_serv_onion.lrange('onion_history:{}'.format(onion_domain), 0 ,-1) - if all_onion_history: - for date_history in all_onion_history: - print('--------') - print('onion_history:{}:{}'.format(onion_domain, date_history)) - item_father = r_serv_onion.lrange('onion_history:{}:{}'.format(onion_domain, date_history), 0, 0) - print('item_father: {}'.format(item_father)) - try: - item_father = item_father[0] - except IndexError: - r_serv_onion.delete('onion_history:{}:{}'.format(onion_domain, date_history)) - continue - #print(item_father) - # delete old history - r_serv_onion.delete('onion_history:{}:{}'.format(onion_domain, date_history)) - # create new history - root_key = get_domain_root_from_paste_childrens(item_father, onion_domain) - if root_key: - r_serv_onion.zadd(f'crawler_history_onion:{onion_domain}:80', {root_key: get_date_epoch(date_history)}) - print('crawler_history_onion:{}:80 {} {}'.format(onion_domain, get_date_epoch(date_history), root_key)) - #update service metadata: paste_parent - r_serv_onion.hset('onion_metadata:{}'.format(onion_domain), 'paste_parent', root_key) - - r_serv_onion.delete('onion_history:{}'.format(onion_domain)) - - r_serv_onion.hset('onion_metadata:{}'.format(onion_domain), 'ports', '80') - r_serv_onion.hdel('onion_metadata:{}'.format(onion_domain), 'last_seen') - - nb_updated += 1 - progress = int((nb_updated * 100) /total_domain) - print('{}/{} updated {}%'.format(nb_updated, total_domain, progress)) - # update progress stats - if progress != last_progress: - r_serv.set('ail:current_background_script_stat', progress) - last_progress = progress - - - end = time.time() - print('Updating ARDB_Onion Done => {} paths: {} s'.format(index, end - start)) - print() - print('Done in {} s'.format(end - start_deb)) - - r_serv.sadd('ail:update_v1.5', 'onions') diff --git a/update/v1.5/Update-ARDB_Onions_screenshots.py b/update/v1.5/Update-ARDB_Onions_screenshots.py deleted file mode 100755 index 6ad7a025..00000000 --- a/update/v1.5/Update-ARDB_Onions_screenshots.py +++ /dev/null @@ -1,117 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* - -import os -import sys -import time -import datetime - -from hashlib import sha256 - -sys.path.append(os.environ['AIL_BIN']) -from lib import ConfigLoader - -def rreplace(s, old, new, occurrence): - li = s.rsplit(old, occurrence) - return new.join(li) - -def substract_date(date_from, date_to): - date_from = datetime.date(int(date_from[0:4]), int(date_from[4:6]), int(date_from[6:8])) - date_to = datetime.date(int(date_to[0:4]), int(date_to[4:6]), int(date_to[6:8])) - delta = date_to - date_from # timedelta - l_date = [] - for i in range(delta.days + 1): - date = date_from + datetime.timedelta(i) - l_date.append( date.strftime('%Y%m%d') ) - return l_date - - -if __name__ == '__main__': - - start_deb = time.time() - - config_loader = ConfigLoader.ConfigLoader() - - SCREENSHOT_FOLDER = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "crawled_screenshot")) - NEW_SCREENSHOT_FOLDER = config_loader.get_files_directory('screenshot') - - PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "pastes")) + '/' - - r_serv = config_loader.get_redis_conn("ARDB_DB") - r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") - r_serv_tag = config_loader.get_redis_conn("ARDB_Tags") - r_serv_onion = config_loader.get_redis_conn("ARDB_Onion") - config_loader = None - - r_serv.set('ail:current_background_script', 'crawled_screenshot') - r_serv.set('ail:current_background_script_stat', 0) - - ## Update Onion ## - print('Updating ARDB_Onion ...') - index = 0 - start = time.time() - - # clean down domain from db - date_from = '20180801' - date_today = datetime.date.today().strftime("%Y%m%d") - list_date = substract_date(date_from, date_today) - nb_done = 0 - last_progress = 0 - total_to_update = len(list_date) - for date in list_date: - screenshot_dir = os.path.join(SCREENSHOT_FOLDER, date[0:4], date[4:6], date[6:8]) - if os.path.isdir(screenshot_dir): - print(screenshot_dir) - for file in os.listdir(screenshot_dir): - if file.endswith(".png"): - index += 1 - #print(file) - - img_path = os.path.join(screenshot_dir, file) - with open(img_path, 'br') as f: - image_content = f.read() - - hash = sha256(image_content).hexdigest() - img_dir_path = os.path.join(hash[0:2], hash[2:4], hash[4:6], hash[6:8], hash[8:10], hash[10:12]) - filename_img = os.path.join(NEW_SCREENSHOT_FOLDER, img_dir_path, hash[12:] +'.png') - dirname = os.path.dirname(filename_img) - if not os.path.exists(dirname): - os.makedirs(dirname) - if not os.path.exists(filename_img): - os.rename(img_path, filename_img) - else: - os.remove(img_path) - - item = os.path.join('crawled', date[0:4], date[4:6], date[6:8], file[:-4]) - # add item metadata - r_serv_metadata.hset('paste_metadata:{}'.format(item), 'screenshot', hash) - # add sha256 metadata - r_serv_onion.sadd('screenshot:{}'.format(hash), item) - - if file.endswith('.pnghar.txt'): - har_path = os.path.join(screenshot_dir, file) - new_file = rreplace(file, '.pnghar.txt', '.json', 1) - new_har_path = os.path.join(screenshot_dir, new_file) - os.rename(har_path, new_har_path) - - progress = int((nb_done * 100) /total_to_update) - # update progress stats - if progress != last_progress: - r_serv.set('ail:current_background_script_stat', progress) - print('{}/{} screenshot updated {}%'.format(nb_done, total_to_update, progress)) - last_progress = progress - - nb_done += 1 - - r_serv.set('ail:current_background_script_stat', 100) - - - end = time.time() - print('Updating ARDB_Onion Done => {} paths: {} s'.format(index, end - start)) - print() - print('Done in {} s'.format(end - start_deb)) - - r_serv.set('ail:current_background_script_stat', 100) - r_serv.sadd('ail:update_v1.5', 'crawled_screenshot') - if r_serv.scard('ail:update_v1.5') != 5: - r_serv.set('ail:update_error', 'Update v1.5 Failed, please relaunch the bin/update-background.py script') diff --git a/update/v1.5/Update-ARDB_Tags.py b/update/v1.5/Update-ARDB_Tags.py deleted file mode 100755 index 38f3c09c..00000000 --- a/update/v1.5/Update-ARDB_Tags.py +++ /dev/null @@ -1,135 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* - -import os -import sys -import time -import redis - -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) -from lib import ConfigLoader - -if __name__ == '__main__': - - start_deb = time.time() - - config_loader = ConfigLoader.ConfigLoader() - - PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "pastes")) + '/' - - r_serv = config_loader.get_redis_conn("ARDB_DB") - r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") - r_serv_tag = config_loader.get_redis_conn("ARDB_Tags") - r_serv_onion = config_loader.get_redis_conn("ARDB_Onion") - r_important_paste_2018 = redis.StrictRedis( - host=config_loader.get_config_str("ARDB_Metadata", "host"), - port=config_loader.get_config_int("ARDB_Metadata", "port"), - db=2018, - decode_responses=True) - - r_important_paste_2019 = redis.StrictRedis( - host=config_loader.get_config_str("ARDB_Metadata", "host"), - port=config_loader.get_config_int("ARDB_Metadata", "port"), - db=2019, - decode_responses=True) - - config_loader = None - - r_serv.set('ail:current_background_script', 'tags') - r_serv.set('ail:current_background_script_stat', 0) - - if r_serv.sismember('ail:update_v1.5', 'onions') and r_serv.sismember('ail:update_v1.5', 'metadata'): - - print('Updating ARDB_Tags ...') - index = 0 - nb_tags_to_update = 0 - nb_updated = 0 - last_progress = 0 - start = time.time() - - tags_list = r_serv_tag.smembers('list_tags') - # create temp tags metadata - tag_metadata = {} - for tag in tags_list: - tag_metadata[tag] = {} - tag_metadata[tag]['first_seen'] = r_serv_tag.hget('tag_metadata:{}'.format(tag), 'first_seen') - if tag_metadata[tag]['first_seen'] is None: - tag_metadata[tag]['first_seen'] = 99999999 - else: - tag_metadata[tag]['first_seen'] = int(tag_metadata[tag]['first_seen']) - - tag_metadata[tag]['last_seen'] = r_serv_tag.hget('tag_metadata:{}'.format(tag), 'last_seen') - if tag_metadata[tag]['last_seen'] is None: - tag_metadata[tag]['last_seen'] = 0 - else: - tag_metadata[tag]['last_seen'] = int(tag_metadata[tag]['last_seen']) - nb_tags_to_update += r_serv_tag.scard(tag) - - if nb_tags_to_update == 0: - nb_tags_to_update = 1 - - for tag in tags_list: - - all_item = r_serv_tag.smembers(tag) - for item_path in all_item: - splitted_item_path = item_path.split('/') - #print(tag) - #print(item_path) - try: - item_date = int( ''.join([splitted_item_path[-4], splitted_item_path[-3], splitted_item_path[-2]]) ) - except IndexError: - r_serv_tag.srem(tag, item_path) - continue - - # remove absolute path - new_path = item_path.replace(PASTES_FOLDER, '', 1) - if new_path != item_path: - # save in queue absolute path to remove - r_serv_tag.sadd('maj:v1.5:absolute_path_to_rename', item_path) - - # update metadata first_seen - if item_date < tag_metadata[tag]['first_seen']: - tag_metadata[tag]['first_seen'] = item_date - r_serv_tag.hset('tag_metadata:{}'.format(tag), 'first_seen', item_date) - - # update metadata last_seen - if item_date > tag_metadata[tag]['last_seen']: - tag_metadata[tag]['last_seen'] = item_date - last_seen_db = r_serv_tag.hget('tag_metadata:{}'.format(tag), 'last_seen') - if last_seen_db: - if item_date > int(last_seen_db): - r_serv_tag.hset('tag_metadata:{}'.format(tag), 'last_seen', item_date) - else: - tag_metadata[tag]['last_seen'] = last_seen_db - - r_serv_tag.sadd('{}:{}'.format(tag, item_date), new_path) - r_serv_tag.hincrby('daily_tags:{}'.format(item_date), tag, 1) - - # clean db - r_serv_tag.srem(tag, item_path) - index = index + 1 - - nb_updated += 1 - progress = int((nb_updated * 100) /nb_tags_to_update) - print('{}/{} updated {}%'.format(nb_updated, nb_tags_to_update, progress)) - # update progress stats - if progress != last_progress: - r_serv.set('ail:current_background_script_stat', progress) - last_progress = progress - - #flush browse importante pastes db - try: - r_important_paste_2018.flushdb() - except Exception: - pass - - try: - r_important_paste_2019.flushdb() - except Exception: - pass - - end = time.time() - - print('Updating ARDB_Tags Done => {} paths: {} s'.format(index, end - start)) - - r_serv.sadd('ail:update_v1.5', 'tags') diff --git a/update/v1.5/Update-ARDB_Tags_background.py b/update/v1.5/Update-ARDB_Tags_background.py deleted file mode 100755 index 0df3d75d..00000000 --- a/update/v1.5/Update-ARDB_Tags_background.py +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* - -import os -import sys -import time - -sys.path.append(os.environ['AIL_BIN']) -from lib import ConfigLoader - -def tags_key_fusion(old_item_path_key, new_item_path_key): - print('fusion:') - print(old_item_path_key) - print(new_item_path_key) - for tag in r_serv_metadata.smembers(old_item_path_key): - r_serv_metadata.sadd(new_item_path_key, tag) - r_serv_metadata.srem(old_item_path_key, tag) - - -if __name__ == '__main__': - - start_deb = time.time() - config_loader = ConfigLoader.ConfigLoader() - - PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "pastes")) + '/' - - r_serv = config_loader.get_redis_conn("ARDB_DB") - r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") - r_serv_tag = config_loader.get_redis_conn("ARDB_Tags") - config_loader = None - - if r_serv.sismember('ail:update_v1.5', 'tags'): - - r_serv.set('ail:current_background_script', 'tags_background') - r_serv.set('ail:current_background_script_stat', 0) - - print('Updating ARDB_Tags ...') - start = time.time() - - # update item metadata tags - tag_not_updated = True - total_to_update = r_serv_tag.scard('maj:v1.5:absolute_path_to_rename') - nb_updated = 0 - last_progress = 0 - if total_to_update > 0: - while tag_not_updated: - item_path = r_serv_tag.srandmember('maj:v1.5:absolute_path_to_rename') - old_tag_item_key = 'tag:{}'.format(item_path) - new_item_path = item_path.replace(PASTES_FOLDER, '', 1) - new_tag_item_key = 'tag:{}'.format(new_item_path) - res = r_serv_metadata.renamenx(old_tag_item_key, new_tag_item_key) - if res == 0: - tags_key_fusion(old_tag_item_key, new_tag_item_key) - nb_updated += 1 - r_serv_tag.srem('maj:v1.5:absolute_path_to_rename', item_path) - if r_serv_tag.scard('maj:v1.5:absolute_path_to_rename') == 0: - tag_not_updated = False - else: - progress = int((nb_updated * 100) / total_to_update) - print('{}/{} Tags updated {}%'.format(nb_updated, total_to_update, progress)) - # update progress stats - if progress != last_progress: - r_serv.set('ail:current_background_script_stat', progress) - last_progress = progress - - end = time.time() - - print('Updating ARDB_Tags Done: {} s'.format(end - start)) - - r_serv.sadd('ail:update_v1.5', 'tags_background') diff --git a/update/v1.5/Update.py b/update/v1.5/Update.py deleted file mode 100755 index fd1a2dd5..00000000 --- a/update/v1.5/Update.py +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* - -import os -import sys -import time -import datetime - -sys.path.append(os.environ['AIL_BIN']) -from lib import ConfigLoader - -if __name__ == '__main__': - - start_deb = time.time() - - config_loader = ConfigLoader.ConfigLoader() - - PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "pastes")) + '/' - - r_serv = config_loader.get_redis_conn("ARDB_DB") - r_serv_onion = config_loader.get_redis_conn("ARDB_Onion") - config_loader = None - - print() - print('Updating ARDB_Onion ...') - index = 0 - start = time.time() - - # update crawler queue - for elem in r_serv_onion.smembers('onion_crawler_queue'): - if PASTES_FOLDER in elem: - r_serv_onion.srem('onion_crawler_queue', elem) - r_serv_onion.sadd('onion_crawler_queue', elem.replace(PASTES_FOLDER, '', 1)) - index = index +1 - for elem in r_serv_onion.smembers('onion_crawler_priority_queue'): - if PASTES_FOLDER in elem: - r_serv_onion.srem('onion_crawler_queue', elem) - r_serv_onion.sadd('onion_crawler_queue', elem.replace(PASTES_FOLDER, '', 1)) - index = index +1 - - end = time.time() - print('Updating ARDB_Onion Done => {} paths: {} s'.format(index, end - start)) - print() - - # Add background update - r_serv.sadd('ail:to_update', 'v1.5') - - #Set current ail version - r_serv.set('ail:version', 'v1.5') - - #Set current ail version - r_serv.set('ail:update_date_v1.5', datetime.datetime.now().strftime("%Y%m%d")) - - print('Done in {} s'.format(end - start_deb)) diff --git a/update/v1.5/Update.sh b/update/v1.5/Update.sh deleted file mode 100755 index cf70a444..00000000 --- a/update/v1.5/Update.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash - -[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1; - -export PATH=$AIL_HOME:$PATH -export PATH=$AIL_REDIS:$PATH -export PATH=$AIL_ARDB:$PATH -export PATH=$AIL_BIN:$PATH -export PATH=$AIL_FLASK:$PATH - -GREEN="\\033[1;32m" -DEFAULT="\\033[0;39m" - -echo -e $GREEN"Shutting down AIL ..."$DEFAULT -bash ${AIL_BIN}/LAUNCH.sh -k & -wait - -echo "" -bash -c "bash ${AIL_HOME}/update/bin/Update_Redis.sh" -#bash -c "bash ${AIL_HOME}/update/bin/Update_ARDB.sh" - -echo "" -echo -e $GREEN"Update DomainClassifier"$DEFAULT -echo "" -pip3 install --upgrade --force-reinstall git+https://github.com/D4-project/BGP-Ranking.git/@28013297efb039d2ebbce96ee2d89493f6ae56b0#subdirectory=client&egg=pybgpranking -pip3 install --upgrade --force-reinstall git+https://github.com/adulau/DomainClassifier.git -wait -echo "" - -echo "" -echo -e $GREEN"Update Web thirdparty"$DEFAULT -echo "" -bash -c "(cd ${AIL_FLASK}; ./update_thirdparty.sh &)" -wait -echo "" - -bash ${AIL_BIN}LAUNCH.sh -lav & -wait -echo "" - -echo "" -echo -e $GREEN"Fixing ARDB ..."$DEFAULT -echo "" -python ${AIL_HOME}/update/v1.5/Update.py & -wait -echo "" -echo "" - -echo "" -echo -e $GREEN"Shutting down ARDB ..."$DEFAULT -bash ${AIL_BIN}/LAUNCH.sh -ks & -wait - -echo "" - -exit 0 diff --git a/update/v1.7/Update.py b/update/v1.7/Update.py deleted file mode 100755 index f3777d0e..00000000 --- a/update/v1.7/Update.py +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* - -import os -import sys -import time -import datetime - -sys.path.append(os.environ['AIL_BIN']) -from lib import ConfigLoader - -if __name__ == '__main__': - - start_deb = time.time() - - config_loader = ConfigLoader.ConfigLoader() - - r_serv = config_loader.get_redis_conn("ARDB_DB") - config_loader = None - - # Set current ail version - r_serv.set('ail:version', 'v1.7') - - # Set current ail version - r_serv.set('ail:update_date_v1.7', datetime.datetime.now().strftime("%Y%m%d")) diff --git a/update/v1.7/Update.sh b/update/v1.7/Update.sh deleted file mode 100755 index 603e9517..00000000 --- a/update/v1.7/Update.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/bin/bash - -[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1; - -export PATH=$AIL_HOME:$PATH -export PATH=$AIL_REDIS:$PATH -export PATH=$AIL_ARDB:$PATH -export PATH=$AIL_BIN:$PATH -export PATH=$AIL_FLASK:$PATH - -GREEN="\\033[1;32m" -DEFAULT="\\033[0;39m" - -echo -e $GREEN"Shutting down AIL ..."$DEFAULT -bash ${AIL_BIN}/LAUNCH.sh -ks & -wait - -echo "" -echo -e $GREEN"Update DomainClassifier"$DEFAULT -echo "" - -cd $AIL_HOME -git clone https://github.com/kazu-yamamoto/pgpdump.git -cd pgpdump -./configure -make -sudo make install -wait -echo "" - -echo "" -echo -e $GREEN"Update requirement"$DEFAULT -echo "" -pip3 install beautifulsoup4 - -bash ${AIL_BIN}LAUNCH.sh -lav & -wait -echo "" - -echo "" -echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT -echo "" -python ${AIL_HOME}/update/v1.7/Update.py & -wait -echo "" -echo "" - -echo "" -echo -e $GREEN"Shutting down ARDB ..."$DEFAULT -bash ${AIL_BIN}/LAUNCH.sh -ks & -wait - -echo "" -echo -e $GREEN"Update thirdparty ..."$DEFAULT -bash ${AIL_BIN}/LAUNCH.sh -t & -wait - - -echo "" - -exit 0 diff --git a/update/v2.0/Update.py b/update/v2.0/Update.py deleted file mode 100755 index 4d3504e4..00000000 --- a/update/v2.0/Update.py +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* - -import os -import sys -import time -import datetime - -sys.path.append(os.environ['AIL_BIN']) -from lib import ConfigLoader - -if __name__ == '__main__': - - start_deb = time.time() - - config_loader = ConfigLoader.ConfigLoader() - - r_serv = config_loader.get_redis_conn("ARDB_DB") - config_loader = None - - # Set current ail version - r_serv.set('ail:version', 'v2.0') - - # use new update_date format - date_tag_to_replace = ['v1.5', 'v1.7'] - for tag in date_tag_to_replace: - if r_serv.exists('ail:update_date_{}'.format(tag)): - date_tag = r_serv.get('ail:update_date_{}'.format(tag)) - r_serv.hset('ail:update_date', tag, date_tag) - r_serv.delete('ail:update_date_{}'.format(tag)) - - # Set current ail version - r_serv.hset('ail:update_date', 'v2.0', datetime.datetime.now().strftime("%Y%m%d")) diff --git a/update/v2.0/Update.sh b/update/v2.0/Update.sh deleted file mode 100755 index 0132ec8e..00000000 --- a/update/v2.0/Update.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/bin/bash - -[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1; - -export PATH=$AIL_HOME:$PATH -export PATH=$AIL_REDIS:$PATH -export PATH=$AIL_ARDB:$PATH -export PATH=$AIL_BIN:$PATH -export PATH=$AIL_FLASK:$PATH - -GREEN="\\033[1;32m" -DEFAULT="\\033[0;39m" - -echo -e $GREEN"Shutting down AIL ..."$DEFAULT -bash ${AIL_BIN}/LAUNCH.sh -ks -wait - -echo "" -echo -e $GREEN"Create Self-Signed Certificate"$DEFAULT -echo "" -pushd ${AIL_BIN}/helper/gen_cert -bash gen_root.sh -wait -bash gen_cert.sh -wait -popd - -cp ${AIL_BIN}/helper/gen_cert/server.crt ${AIL_FLASK}/server.crt -cp ${AIL_BIN}/helper/gen_cert/server.key ${AIL_FLASK}/server.key - -echo "" -echo -e $GREEN"Update requirement"$DEFAULT -echo "" -pip3 install flask-login -wait -echo "" -pip3 install bcrypt -wait -echo "" -echo "" - -bash ${AIL_BIN}/LAUNCH.sh -lav & -wait -echo "" - -echo "" -echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT -echo "" -python ${AIL_HOME}/update/v2.0/Update.py -wait -echo "" -echo "" - -echo "" -echo -e $GREEN"Update thirdparty ..."$DEFAULT -bash ${AIL_BIN}/LAUNCH.sh -t -wait -echo "" - -echo "" -echo -e $GREEN"Create Default User"$DEFAULT -echo "" -python3 ${AIL_FLASK}create_default_user.py - - -echo "" -echo -e $GREEN"Shutting down ARDB ..."$DEFAULT -bash ${AIL_BIN}/LAUNCH.sh -ks -wait - -exit 0 diff --git a/update/v2.2/Update.py b/update/v2.2/Update.py deleted file mode 100755 index 2bfef8e9..00000000 --- a/update/v2.2/Update.py +++ /dev/null @@ -1,118 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* - -import os -import sys -import time -import datetime - -sys.path.append(os.environ['AIL_BIN']) -from packages import Term - -from lib import ConfigLoader - - -def rreplace(s, old, new, occurrence): - li = s.rsplit(old, occurrence) - return new.join(li) - -def get_item_id(full_path): - return full_path.replace(PASTES_FOLDER, '', 1) - -def get_item_date(id_item): - l_dir = id_item.split('/') - return f'{l_dir[-4]}{l_dir[-3]}{l_dir[-2]}' - - -if __name__ == '__main__': - - start_deb = time.time() - - config_loader = ConfigLoader.ConfigLoader() - PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "pastes")) + '/' - PASTES_FOLDER = os.path.join(os.path.realpath(PASTES_FOLDER), '') - - r_serv = config_loader.get_redis_conn("ARDB_DB") - r_serv_term_stats = config_loader.get_redis_conn("ARDB_Trending") - r_serv_termfreq = config_loader.get_redis_conn("ARDB_TermFreq") - config_loader = None - - r_serv_term_stats.flushdb() - - # Disabled. Checkout the v2.2 branch if you need it - # # convert all regex: - # all_regex = r_serv_termfreq.smembers('TrackedRegexSet') - # for regex in all_regex: - # tags = list(r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(regex))) - # mails = list(r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(regex))) - # - # new_term = regex[1:-1] - # res = Term.parse_json_term_to_add({"term": new_term, "type": 'regex', "tags": tags, "mails": mails, "level": 1}, - # 'admin@admin.test') - # if res[1] == 200: - # term_uuid = res[0]['uuid'] - # list_items = r_serv_termfreq.smembers('regex_{}'.format(regex)) - # for paste_item in list_items: - # item_id = get_item_id(paste_item) - # item_date = get_item_date(item_id) - # Term.add_tracked_item(term_uuid, item_id, item_date) - # - # # Invalid Tracker => remove it - # else: - # print('Invalid Regex Removed: {}'.format(regex)) - # print(res[0]) - # # allow reprocess - # r_serv_termfreq.srem('TrackedRegexSet', regex) - # - # all_tokens = r_serv_termfreq.smembers('TrackedSetTermSet') - # for token in all_tokens: - # tags = list(r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(token))) - # mails = list(r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(token))) - # - # res = Term.parse_json_term_to_add({"term": token, "type": 'word', "tags": tags, "mails": mails, "level": 1}, 'admin@admin.test') - # if res[1] == 200: - # term_uuid = res[0]['uuid'] - # list_items = r_serv_termfreq.smembers('tracked_{}'.format(token)) - # for paste_item in list_items: - # item_id = get_item_id(paste_item) - # item_date = get_item_date(item_id) - # Term.add_tracked_item(term_uuid, item_id, item_date) - # # Invalid Tracker => remove it - # else: - # print('Invalid Token Removed: {}'.format(token)) - # print(res[0]) - # # allow reprocess - # r_serv_termfreq.srem('TrackedSetTermSet', token) - # - # all_set = r_serv_termfreq.smembers('TrackedSetSet') - # for curr_set in all_set: - # tags = list(r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(curr_set))) - # mails = list(r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(curr_set))) - # - # to_remove = ',{}'.format(curr_set.split(',')[-1]) - # new_set = rreplace(curr_set, to_remove, '', 1) - # new_set = new_set[2:] - # new_set = new_set.replace(',', '') - # - # res = Term.parse_json_term_to_add({"term": new_set, "type": 'set', "nb_words": 1, "tags": tags, "mails": mails, "level": 1}, 'admin@admin.test') - # if res[1] == 200: - # term_uuid = res[0]['uuid'] - # list_items = r_serv_termfreq.smembers('tracked_{}'.format(curr_set)) - # for paste_item in list_items: - # item_id = get_item_id(paste_item) - # item_date = get_item_date(item_id) - # Term.add_tracked_item(term_uuid, item_id, item_date) - # # Invalid Tracker => remove it - # else: - # print('Invalid Set Removed: {}'.format(curr_set)) - # print(res[0]) - # # allow reprocess - # r_serv_termfreq.srem('TrackedSetSet', curr_set) - - r_serv_termfreq.flushdb() - - # Set current ail version - r_serv.set('ail:version', 'v2.2') - - # Set current ail version - r_serv.hset('ail:update_date', 'v2.2', datetime.datetime.now().strftime("%Y%m%d")) diff --git a/update/v2.2/Update.sh b/update/v2.2/Update.sh deleted file mode 100755 index 37704f3b..00000000 --- a/update/v2.2/Update.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash - -[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1; - -export PATH=$AIL_HOME:$PATH -export PATH=$AIL_REDIS:$PATH -export PATH=$AIL_ARDB:$PATH -export PATH=$AIL_BIN:$PATH -export PATH=$AIL_FLASK:$PATH - -GREEN="\\033[1;32m" -DEFAULT="\\033[0;39m" - -echo -e $GREEN"Shutting down AIL ..."$DEFAULT -bash ${AIL_BIN}/LAUNCH.sh -ks -wait - -bash ${AIL_BIN}/LAUNCH.sh -lav & -wait -echo "" - -echo "" -echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT -echo "" -python ${AIL_HOME}/update/v2.2/Update.py -wait -echo "" -echo "" - -echo "" -echo -e $GREEN"Shutting down ARDB ..."$DEFAULT -bash ${AIL_BIN}/LAUNCH.sh -ks -wait - -exit 0 diff --git a/update/v2.5/Update.py b/update/v2.5/Update.py deleted file mode 100755 index 6264c7f4..00000000 --- a/update/v2.5/Update.py +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* - -import os -import sys -import time -import datetime - -sys.path.append(os.environ['AIL_BIN']) -from lib import ConfigLoader - -new_version = 'v2.5' - -if __name__ == '__main__': - - start_deb = time.time() - - config_loader = ConfigLoader.ConfigLoader() - r_serv = config_loader.get_redis_conn("ARDB_DB") - config_loader = None - - r_serv.zadd('ail:all_role', {'user': 3}) - r_serv.zadd('ail:all_role', {'user_no_api': 4}) - r_serv.zadd('ail:all_role', {'read_only': 5}) - - for user in r_serv.hkeys('user:all'): - r_serv.sadd('user_role:user', user) - r_serv.sadd('user_role:user_no_api', user) - r_serv.sadd('user_role:read_only', user) - - # Set current ail version - r_serv.set('ail:version', new_version) - - # Set current ail version - r_serv.hset('ail:update_date', new_version, datetime.datetime.now().strftime("%Y%m%d")) diff --git a/update/v2.5/Update.sh b/update/v2.5/Update.sh deleted file mode 100755 index 6c75f15f..00000000 --- a/update/v2.5/Update.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash - -[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1; - -export PATH=$AIL_HOME:$PATH -export PATH=$AIL_REDIS:$PATH -export PATH=$AIL_ARDB:$PATH -export PATH=$AIL_BIN:$PATH -export PATH=$AIL_FLASK:$PATH - -GREEN="\\033[1;32m" -DEFAULT="\\033[0;39m" - -echo -e $GREEN"Shutting down AIL ..."$DEFAULT -bash ${AIL_BIN}/LAUNCH.sh -ks -wait - -bash ${AIL_BIN}/LAUNCH.sh -lav & -wait -echo "" - -echo "" -echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT -echo "" -python ${AIL_HOME}/update/v2.5/Update.py -wait -echo "" -echo "" - -echo "" -echo -e $GREEN"Shutting down ARDB ..."$DEFAULT -bash ${AIL_BIN}/LAUNCH.sh -ks -wait - -exit 0 diff --git a/update/v2.6/Update.py b/update/v2.6/Update.py deleted file mode 100755 index ded00cf1..00000000 --- a/update/v2.6/Update.py +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* - -import os -import sys -import time -import datetime - -sys.path.append(os.environ['AIL_BIN']) -from lib import ConfigLoader - -new_version = 'v2.6' - -if __name__ == '__main__': - start_deb = time.time() - - config_loader = ConfigLoader.ConfigLoader() - r_serv = config_loader.get_redis_conn("ARDB_DB") - config_loader = None - - r_serv.sadd('ail:to_update', new_version) - - # Set current ail version - r_serv.set('ail:version', new_version) - - # Set current ail version - r_serv.hset('ail:update_date', new_version, datetime.datetime.now().strftime("%Y%m%d")) diff --git a/update/v2.6/Update.sh b/update/v2.6/Update.sh deleted file mode 100755 index 874bf0ec..00000000 --- a/update/v2.6/Update.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash - -[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1; - -export PATH=$AIL_HOME:$PATH -export PATH=$AIL_REDIS:$PATH -export PATH=$AIL_ARDB:$PATH -export PATH=$AIL_BIN:$PATH -export PATH=$AIL_FLASK:$PATH - -GREEN="\\033[1;32m" -DEFAULT="\\033[0;39m" - -echo -e $GREEN"Shutting down AIL ..."$DEFAULT -bash ${AIL_BIN}/LAUNCH.sh -ks -wait - -bash ${AIL_BIN}/LAUNCH.sh -lav & -wait -echo "" - -echo "" -echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT -echo "" -python ${AIL_HOME}/update/v2.6/Update.py -wait -echo "" -echo "" - -echo "" -echo -e $GREEN"Shutting down ARDB ..."$DEFAULT -bash ${AIL_BIN}/LAUNCH.sh -ks -wait - -exit 0 diff --git a/update/v2.6/Update_screenshots.py b/update/v2.6/Update_screenshots.py deleted file mode 100755 index 735d64bc..00000000 --- a/update/v2.6/Update_screenshots.py +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* - -import os -import sys -import time - -from pyfaup.faup import Faup - -sys.path.append(os.environ['AIL_BIN']) -from lib import ConfigLoader - -def get_domain(item_id): - item_id = item_id.split('/') - item_id = item_id[-1] - return item_id[:-36] - -def get_all_item(s_sha256): - return r_serv_onion.smembers(f'screenshot:{s_sha256}') - -def sanitize_domain(domain): - faup.decode(domain) - domain_sanitized = faup.get() - domain_sanitized = domain_sanitized['domain'] - try: - domain_sanitized = domain_sanitized.decode() - except: - pass - return domain_sanitized.lower() - -def update_db(s_sha256): - screenshot_items = get_all_item(s_sha256) - if screenshot_items: - for item_id in screenshot_items: - item_id = item_id.replace(PASTES_FOLDER+'/', '', 1) # remove root path - domain = get_domain(item_id) - - domain_sanitized = sanitize_domain(domain) - if domain != domain_sanitized: - r_serv_onion.sadd('incorrect_domain', domain) - domain = domain_sanitized - - r_serv_onion.sadd('domain_screenshot:{}'.format(domain), s_sha256) - r_serv_onion.sadd('screenshot_domain:{}'.format(s_sha256), domain) - else: - pass - # broken screenshot - r_serv_onion.sadd('broken_screenshot', s_sha256) - - -if __name__ == '__main__': - - start_deb = time.time() - faup = Faup() - - config_loader = ConfigLoader.ConfigLoader() - - PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "pastes")) - SCREENSHOT_FOLDER = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "crawled_screenshot"), 'screenshot') - - r_serv_db = config_loader.get_redis_conn("ARDB_DB") - r_serv_onion = config_loader.get_redis_conn("ARDB_Onion") - config_loader = None - - r_serv_db.set('ail:update_in_progress', 'v2.6') - r_serv_db.set('ail:current_background_update', 'v2.6') - - r_serv_db.set('ail:current_background_script_stat', 20) - r_serv_db.set('ail:current_background_script', 'screenshot update') - - nb = 0 - - if os.path.isdir(SCREENSHOT_FOLDER): - for root, dirs, files in os.walk(SCREENSHOT_FOLDER, topdown=False): - # print(dirs) - for name in files: - nb = nb + 1 - screenshot_sha256 = os.path.join(root, name) - screenshot_sha256 = screenshot_sha256[:-4] # remove .png - screenshot_sha256 = screenshot_sha256.replace(SCREENSHOT_FOLDER, '', 1) - screenshot_sha256 = screenshot_sha256.replace('/', '') - update_db(screenshot_sha256) - # print('Screenshot updated: {}'.format(nb)) - if nb % 1000 == 0: - r_serv_db.set('ail:current_background_script', 'screenshot updated: {}'.format(nb)) - - r_serv_db.set('ail:current_background_script_stat', 100) - - end = time.time() - print('ALL screenshot updated: {} in {} s'.format(nb, end - start_deb)) diff --git a/update/v2.7/Update.py b/update/v2.7/Update.py deleted file mode 100755 index eed7c219..00000000 --- a/update/v2.7/Update.py +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* - -import os -import sys -import time -import datetime - -sys.path.append(os.environ['AIL_BIN']) -from lib import ConfigLoader - -new_version = 'v2.7' - -if __name__ == '__main__': - - start_deb = time.time() - - config_loader = ConfigLoader.ConfigLoader() - r_serv = config_loader.get_redis_conn("ARDB_DB") - r_serv_tags = config_loader.get_redis_conn("ARDB_Tags") - r_serv_onion = config_loader.get_redis_conn("ARDB_Onion") - config_loader = None - - r_serv.sadd('ail:to_update', new_version) - - #### Update tags #### - r_serv_tags.sunionstore('list_tags:item', 'list_tags', []) - r_serv_onion.sunionstore('domain_update_v2.7', 'full_onion_up', []) - r_serv_onion.delete('incorrect_domain') - r_serv.set('ail:update_v2.7:deletetagrange', 1) - #### #### - - # Set current ail version - r_serv.set('ail:version', new_version) - - # Set current ail version - r_serv.hset('ail:update_date', new_version, datetime.datetime.now().strftime("%Y%m%d")) diff --git a/update/v2.7/Update.sh b/update/v2.7/Update.sh deleted file mode 100755 index 8f9a4efd..00000000 --- a/update/v2.7/Update.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash - -[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1; - -export PATH=$AIL_HOME:$PATH -export PATH=$AIL_REDIS:$PATH -export PATH=$AIL_ARDB:$PATH -export PATH=$AIL_BIN:$PATH -export PATH=$AIL_FLASK:$PATH - -GREEN="\\033[1;32m" -DEFAULT="\\033[0;39m" - -echo -e $GREEN"Shutting down AIL ..."$DEFAULT -bash ${AIL_BIN}/LAUNCH.sh -ks -wait - -bash ${AIL_BIN}/LAUNCH.sh -lav & -wait -echo "" - -echo "" -echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT -echo "" -python ${AIL_HOME}/update/v2.7/Update.py -wait -echo "" -echo "" - -echo "" -echo -e $GREEN"Shutting down ARDB ..."$DEFAULT -bash ${AIL_BIN}/LAUNCH.sh -ks -wait - -exit 0 diff --git a/update/v2.7/Update_domain_tags.py b/update/v2.7/Update_domain_tags.py deleted file mode 100755 index cbe1e4b6..00000000 --- a/update/v2.7/Update_domain_tags.py +++ /dev/null @@ -1,127 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* - -import os -import sys -import time - -from pyfaup.faup import Faup - -sys.path.append(os.environ['AIL_BIN']) -from packages import Date - -from lib import ConfigLoader - -def sanitize_domain(domain): - faup.decode(domain) - domain_sanitized = faup.get() - domain_sanitized = domain_sanitized['domain'] - try: - domain_sanitized = domain_sanitized.decode() - except: - pass - return domain_sanitized.lower() - -def get_all_obj_tags(obj_type): - return list(r_serv_tags.smembers(f'list_tags:{obj_type}')) - -def add_global_tag(tag, object_type=None): - r_serv_tags.sadd('list_tags', tag) - if object_type: - r_serv_tags.sadd('list_tags:{}'.format(object_type), tag) - -def get_obj_tag(object_id): - res = r_serv_metadata.smembers('tag:{}'.format(object_id)) - if res: - return list(res) - else: - return [] - -def delete_domain_tag_daterange(): - all_domains_tags = get_all_obj_tags('domain') - nb_updated = 0 - nb_to_update = len(all_domains_tags) - if nb_to_update == 0: - nb_to_update = 1 - refresh_time = time.time() - l_dates = Date.substract_date('20191008', Date.get_today_date_str()) - for tag in all_domains_tags: - for date_day in l_dates: - r_serv_tags.delete('domain:{}:{}'.format(tag, date_day)) - nb_updated += 1 - refresh_time = update_progress(refresh_time, nb_updated, nb_to_update) - r_serv_db.delete('ail:update_v2.7:deletetagrange') - -def update_domain_tags(domain): - domain_sanitized = sanitize_domain(domain) - if domain != domain_sanitized: - r_serv_onion.sadd('incorrect_domain', domain) - domain = domain_sanitized - - domain_tags = get_obj_tag(domain) - for tag in domain_tags: - # delete incorrect tags - if tag == 'infoleak:submission="crawler"' or tag == 'infoleak:submission="manual"': - r_serv_metadata.srem('tag:{}'.format(domain), tag) - else: - add_global_tag(tag, object_type='domain') - r_serv_tags.sadd('{}:{}'.format('domain', tag), domain) - -def update_progress(refresh_time, nb_updated, nb_elem_to_update): - if time.time() - refresh_time > 10: - progress = int((nb_updated * 100) / nb_elem_to_update) - print('{}/{} updated {}%'.format(nb_updated, nb_elem_to_update, progress)) - r_serv_db.set('ail:current_background_script_stat', progress) - refresh_time = time.time() - - return refresh_time - -def update_db(): - nb_updated = 0 - nb_to_update = r_serv_onion.scard('domain_update_v2.7') - refresh_time = time.time() - r_serv_db.set('ail:current_background_script_stat', 0) - r_serv_db.set('ail:current_background_script', 'domain tags update') - domain = r_serv_onion.spop('domain_update_v2.7') - while domain is not None: - update_domain_tags(domain) - nb_updated += 1 - refresh_time = update_progress(refresh_time, nb_updated, nb_to_update) - domain = r_serv_onion.spop('domain_update_v2.7') - if r_serv_db.exists('ail:update_v2.7:deletetagrange'): - r_serv_db.set('ail:current_background_script_stat', 0) - r_serv_db.set('ail:current_background_script', 'tags: remove deprecated keys') - delete_domain_tag_daterange() - - # sort all crawled domain - r_serv_onion.sort('full_onion_up', alpha=True) - r_serv_onion.sort('full_regular_up', alpha=True) - - -if __name__ == '__main__': - - start_deb = time.time() - faup = Faup() - - config_loader = ConfigLoader.ConfigLoader() - - r_serv_db = config_loader.get_redis_conn("ARDB_DB") - r_serv_tags = config_loader.get_redis_conn("ARDB_Tags") - r_serv_onion = config_loader.get_redis_conn("ARDB_Onion") - r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") - config_loader = None - - update_version = 'v2.7' - - r_serv_db.set('ail:update_in_progress', update_version) - r_serv_db.set('ail:current_background_update', update_version) - - r_serv_db.set('ail:current_background_script_stat', 0) - r_serv_db.set('ail:current_background_script', 'tags update') - - update_db() - - r_serv_db.set('ail:current_background_script_stat', 100) - - end = time.time() - print('ALL domains tags updated in {} s'.format(end - start_deb)) diff --git a/update/v3.0/Update.py b/update/v3.0/Update.py deleted file mode 100755 index 9d393f6f..00000000 --- a/update/v3.0/Update.py +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* - -import os -import sys - -sys.path.append(os.environ['AIL_HOME']) -################################## -# Import Project packages -################################## -from update.bin.old_ail_updater import AIL_Updater - -class Updater(AIL_Updater): - """default Updater.""" - - def __init__(self, version): - super(Updater, self).__init__(version) - - -if __name__ == '__main__': - updater = Updater('v3.0') - updater.run_update() diff --git a/update/v3.0/Update.sh b/update/v3.0/Update.sh deleted file mode 100755 index 9c7a7d9d..00000000 --- a/update/v3.0/Update.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash - -[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1; - -export PATH=$AIL_HOME:$PATH -export PATH=$AIL_REDIS:$PATH -export PATH=$AIL_ARDB:$PATH -export PATH=$AIL_BIN:$PATH -export PATH=$AIL_FLASK:$PATH - -GREEN="\\033[1;32m" -DEFAULT="\\033[0;39m" - -echo -e $GREEN"Shutting down AIL ..."$DEFAULT -bash ${AIL_BIN}/LAUNCH.sh -ks -wait - -bash ${AIL_BIN}/LAUNCH.sh -lav & -wait -echo "" - -echo "" -echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT -echo "" -python ${AIL_HOME}/update/v3.0/Update.py -wait -echo "" -echo "" - -echo "" -echo -e $GREEN"Update thirdparty ..."$DEFAULT -bash ${AIL_BIN}/LAUNCH.sh -t - -echo "" -echo -e $GREEN"Shutting down ARDB ..."$DEFAULT -bash ${AIL_BIN}/LAUNCH.sh -ks -wait - -exit 0 diff --git a/update/v3.1.1/Update.py b/update/v3.1.1/Update.py deleted file mode 100755 index a5a16751..00000000 --- a/update/v3.1.1/Update.py +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* - -import os -import sys - -sys.path.append(os.environ['AIL_HOME']) -################################## -# Import Project packages -################################## -from update.bin.old_ail_updater import AIL_Updater - -class Updater(AIL_Updater): - """default Updater.""" - - def __init__(self, version): - super(Updater, self).__init__(version) - - -if __name__ == '__main__': - updater = Updater('v3.1.1') - updater.run_update() diff --git a/update/v3.1.1/Update.sh b/update/v3.1.1/Update.sh deleted file mode 100755 index f43591c0..00000000 --- a/update/v3.1.1/Update.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash - -[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1; - -export PATH=$AIL_HOME:$PATH -export PATH=$AIL_REDIS:$PATH -export PATH=$AIL_ARDB:$PATH -export PATH=$AIL_BIN:$PATH -export PATH=$AIL_FLASK:$PATH - -GREEN="\\033[1;32m" -DEFAULT="\\033[0;39m" - -echo -e $GREEN"Shutting down AIL ..."$DEFAULT -bash ${AIL_BIN}/LAUNCH.sh -ks -wait - -bash ${AIL_BIN}/LAUNCH.sh -ldbv & -wait -echo "" - -# SUBMODULES # -git submodule init -git submodule update - -echo -e $GREEN"Installing YARA ..."$DEFAULT -pip3 install yara-python -bash ${AIL_BIN}/LAUNCH.sh -t - -# SUBMODULES # -git submodule init -git submodule update - -echo "" -echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT -echo "" -python ${AIL_HOME}/update/v3.1.1/Update.py -wait -echo "" -echo "" - - -echo "" -echo -e $GREEN"Shutting down ARDB ..."$DEFAULT -bash ${AIL_BIN}/LAUNCH.sh -ks -wait - -exit 0 diff --git a/update/v3.1/Update.py b/update/v3.1/Update.py deleted file mode 100755 index 6c4c4546..00000000 --- a/update/v3.1/Update.py +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* - -import os -import sys - -sys.path.append(os.environ['AIL_HOME']) -################################## -# Import Project packages -################################## -from update.bin.old_ail_updater import AIL_Updater - -class Updater(AIL_Updater): - """default Updater.""" - - def __init__(self, version): - super(Updater, self).__init__(version) - - -if __name__ == '__main__': - updater = Updater('v3.1') - updater.run_update() - diff --git a/update/v3.1/Update.sh b/update/v3.1/Update.sh deleted file mode 100755 index 53b27cc6..00000000 --- a/update/v3.1/Update.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash - -[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1; - -export PATH=$AIL_HOME:$PATH -export PATH=$AIL_REDIS:$PATH -export PATH=$AIL_ARDB:$PATH -export PATH=$AIL_BIN:$PATH -export PATH=$AIL_FLASK:$PATH - -GREEN="\\033[1;32m" -DEFAULT="\\033[0;39m" - -echo -e $GREEN"Shutting down AIL ..."$DEFAULT -bash ${AIL_BIN}/LAUNCH.sh -ks -wait - -bash ${AIL_BIN}/LAUNCH.sh -lav & -wait -echo "" - -pip3 install scrapy -pip3 install scrapy-splash - -echo "" -echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT -echo "" -python ${AIL_HOME}/update/v3.1/Update.py -wait -echo "" -echo "" - -echo "" -echo -e $GREEN"Update thirdparty ..."$DEFAULT -bash ${AIL_BIN}/LAUNCH.sh -t - -echo "" -echo -e $GREEN"Shutting down ARDB ..."$DEFAULT -bash ${AIL_BIN}/LAUNCH.sh -ks -wait - -exit 0 diff --git a/update/v3.2/Update.py b/update/v3.2/Update.py deleted file mode 100755 index 086ed098..00000000 --- a/update/v3.2/Update.py +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* - -import os -import sys - -sys.path.append(os.environ['AIL_HOME']) -################################## -# Import Project packages -################################## -from update.bin.old_ail_updater import AIL_Updater - -class Updater(AIL_Updater): - """default Updater.""" - - def __init__(self, version): - super(Updater, self).__init__(version) - - -if __name__ == '__main__': - updater = Updater('v3.2') - updater.run_update() diff --git a/update/v3.2/Update.sh b/update/v3.2/Update.sh deleted file mode 100755 index a588e55d..00000000 --- a/update/v3.2/Update.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash - -[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1; - -export PATH=$AIL_HOME:$PATH -export PATH=$AIL_REDIS:$PATH -export PATH=$AIL_ARDB:$PATH -export PATH=$AIL_BIN:$PATH -export PATH=$AIL_FLASK:$PATH - -GREEN="\\033[1;32m" -DEFAULT="\\033[0;39m" - -echo -e $GREEN"Shutting down AIL ..."$DEFAULT -bash ${AIL_BIN}/LAUNCH.sh -ks -wait - -bash ${AIL_BIN}/LAUNCH.sh -ldbv & -wait -echo "" - -# SUBMODULES # -git submodule init -git submodule update - -echo -e $GREEN"Installing YARA ..."$DEFAULT -pip3 install yara-python -bash ${AIL_BIN}/LAUNCH.sh -t - -# SUBMODULES # -git submodule init -git submodule update - -echo "" -echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT -echo "" -python ${AIL_HOME}/update/v3.2/Update.py -wait -echo "" -echo "" - - -echo "" -echo -e $GREEN"Shutting down ARDB ..."$DEFAULT -bash ${AIL_BIN}/LAUNCH.sh -ks -wait - -exit 0 diff --git a/update/v3.3/Update.py b/update/v3.3/Update.py deleted file mode 100755 index 5f0efc78..00000000 --- a/update/v3.3/Update.py +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* - -import os -import sys - -sys.path.append(os.environ['AIL_HOME']) -################################## -# Import Project packages -################################## -from update.bin.old_ail_updater import AIL_Updater - -class Updater(AIL_Updater): - """default Updater.""" - - def __init__(self, version): - super(Updater, self).__init__(version) - - -if __name__ == '__main__': - updater = Updater('v3.3') - updater.run_update() diff --git a/update/v3.3/Update.sh b/update/v3.3/Update.sh deleted file mode 100755 index 86289dba..00000000 --- a/update/v3.3/Update.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash - -[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1; - -export PATH=$AIL_HOME:$PATH -export PATH=$AIL_REDIS:$PATH -export PATH=$AIL_ARDB:$PATH -export PATH=$AIL_BIN:$PATH -export PATH=$AIL_FLASK:$PATH - -GREEN="\\033[1;32m" -DEFAULT="\\033[0;39m" - -echo -e $GREEN"Shutting down AIL ..."$DEFAULT -bash ${AIL_BIN}/LAUNCH.sh -ks -wait - -bash ${AIL_BIN}/LAUNCH.sh -ldbv & -wait -echo "" - -# SUBMODULES # -git submodule update - -# echo "" -# echo -e $GREEN"installing KVORCKS ..."$DEFAULT -# cd ${AIL_HOME} -# test ! -d kvrocks/ && git clone https://github.com/bitleak/kvrocks.git -# pushd kvrocks/ -# make -j4 -# popd - -echo -e $GREEN"Installing html2text ..."$DEFAULT -pip3 install html2text - -echo "" -echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT -echo "" -python ${AIL_HOME}/update/v3.3/Update.py -wait -echo "" -echo "" - - -echo "" -echo -e $GREEN"Shutting down ARDB ..."$DEFAULT -bash ${AIL_BIN}/LAUNCH.sh -ks -wait - -exit 0 diff --git a/update/v3.4/Update.py b/update/v3.4/Update.py deleted file mode 100755 index d4308551..00000000 --- a/update/v3.4/Update.py +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* - -import os -import sys - -sys.path.append(os.environ['AIL_HOME']) -################################## -# Import Project packages -################################## -from update.bin.old_ail_updater import AIL_Updater - -class Updater(AIL_Updater): - """default Updater.""" - - def __init__(self, version): - super(Updater, self).__init__(version) - self.r_serv_onion = self.config.get_redis_conn("ARDB_Onion") - - def update(self): - """ - Update Domain Languages - """ - self.r_serv_onion.sunionstore('domain_update_v3.4', 'full_onion_up', 'full_regular_up') - self.r_serv.set('update:nb_elem_to_convert', self.r_serv_onion.scard('domain_update_v3.4')) - self.r_serv.set('update:nb_elem_converted', 0) - - # Add background update - self.r_serv.sadd('ail:to_update', self.version) - - -if __name__ == '__main__': - updater = Updater('v3.4') - updater.run_update() diff --git a/update/v3.4/Update.sh b/update/v3.4/Update.sh deleted file mode 100755 index 16a9ccb7..00000000 --- a/update/v3.4/Update.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash - -[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1; - -export PATH=$AIL_HOME:$PATH -export PATH=$AIL_REDIS:$PATH -export PATH=$AIL_ARDB:$PATH -export PATH=$AIL_BIN:$PATH -export PATH=$AIL_FLASK:$PATH - -GREEN="\\033[1;32m" -DEFAULT="\\033[0;39m" - -echo -e $GREEN"Shutting down AIL ..."$DEFAULT -bash ${AIL_BIN}/LAUNCH.sh -ks -wait - -# bash ${AIL_BIN}/LAUNCH.sh -ldbv & -# wait -# echo "" - -# SUBMODULES # -git submodule update - -# echo "" -# echo -e $GREEN"installing KVORCKS ..."$DEFAULT -# cd ${AIL_HOME} -# test ! -d kvrocks/ && git clone https://github.com/bitleak/kvrocks.git -# pushd kvrocks/ -# make -j4 -# popd - -echo -e $GREEN"Installing html2text ..."$DEFAULT -pip3 install pycld3 - -echo "" -echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT -echo "" -python ${AIL_HOME}/update/v3.4/Update.py -wait -echo "" -echo "" - - -echo "" -echo -e $GREEN"Shutting down ARDB ..."$DEFAULT -bash ${AIL_BIN}/LAUNCH.sh -ks -wait - -exit 0 diff --git a/update/v3.4/Update_domain.py b/update/v3.4/Update_domain.py deleted file mode 100755 index c6183725..00000000 --- a/update/v3.4/Update_domain.py +++ /dev/null @@ -1,121 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* - -import os -import sys -import time - -sys.path.append(os.environ['AIL_BIN']) -################################## -# Import Project packages -################################## -from lib import ConfigLoader -from lib.objects.Items import Item - -def get_domain_type(domain_name): - if str(domain_name).endswith('.onion'): - return 'onion' - else: - return 'regular' - -def add_domain_language(domain_name, language): - language = language.split('-')[0] - domain_type = get_domain_type(domain_name) - r_serv_onion.sadd('all_domains_languages', language) - r_serv_onion.sadd(f'all_domains_languages:{domain_type}', language) - r_serv_onion.sadd(f'language:domains:{domain_type}:{language}', domain_name) - r_serv_onion.sadd(f'domain:language:{domain_name}', language) - -def add_domain_languages_by_item_id(domain_name, item_id): - item = Item(item_id) - for lang in item.get_languages(): - add_domain_language(domain_name, lang.language) - -def update_update_stats(): - nb_updated = int(r_serv_db.get('update:nb_elem_converted')) - progress = int((nb_updated * 100) / nb_elem_to_update) - print(f'{nb_updated}/{nb_elem_to_update} updated {progress}%') - r_serv_db.set('ail:current_background_script_stat', progress) - -def update_domain_language(domain_obj, item_id): - domain_name = domain_obj.get_domain_name() - add_domain_languages_by_item_id(domain_name, item_id) - -def get_domain_history(domain_type, domain_name): - return r_serv_onion.zrange(f'crawler_history_{domain_type}:{domain_name}:80', 0, -1, withscores=True) - - -def get_item_children(item_id): - return r_serv_metadata.smembers(f'paste_children:{item_id}') - -def get_domain_items(domain_name, root_item_id): - dom_item = get_domain_item_children(domain_name, root_item_id) - dom_item.append(root_item_id) - return dom_item - -def is_item_in_domain(domain_name, item_id): - is_in_domain = False - domain_length = len(domain_name) - if len(item_id) > (domain_length+48): - if item_id[-36-domain_length:-36] == domain_name: - is_in_domain = True - return is_in_domain - -def get_domain_item_children(domain_name, root_item_id): - all_items = [] - for item_id in get_item_children(root_item_id): - if is_item_in_domain(domain_name, item_id): - all_items.append(item_id) - all_items.extend(get_domain_item_children(domain_name, item_id)) - return all_items - -def get_domain_crawled_item_root(domain_name, domain_type, epoch): - res = r_serv_onion.zrevrangebyscore(f'crawler_history_{domain_type}:{domain_name}:80', int(epoch), int(epoch)) - return {"root_item": res[0], "epoch": int(epoch)} - -def get_domain_items_crawled(domain_name, domain_type, epoch): - item_crawled = [] - item_root = get_domain_crawled_item_root(domain_name, domain_type, epoch) - if item_root: - if item_root['root_item'] != str(item_root['epoch']) and item_root['root_item']: - for item_id in get_domain_items(domain_name, item_root['root_item']): - item_crawled.append(item_id) - return item_crawled - - -if __name__ == '__main__': - - start_deb = time.time() - config_loader = ConfigLoader.ConfigLoader() - r_serv_db = config_loader.get_redis_conn("ARDB_DB") - r_serv_onion = config_loader.get_redis_conn("ARDB_Onion") - r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") - config_loader = None - - r_serv_db.set('ail:current_background_script', 'domain languages update') - - nb_elem_to_update = r_serv_db.get('update:nb_elem_to_convert') - if not nb_elem_to_update: - nb_elem_to_update = 1 - else: - nb_elem_to_update = int(nb_elem_to_update) - - # _delete_all_domains_languages() - - while True: - domain = r_serv_onion.spop('domain_update_v3.4') - if domain is not None: - print(domain) - domain = str(domain) - domain_t = get_domain_type(domain) - for domain_history in get_domain_history(domain_t, domain): - domain_items = get_domain_items_crawled(domain, domain_t, domain_history[1]) - for id_item in domain_items: - update_domain_language(domain, id_item) - - r_serv_db.incr('update:nb_elem_converted') - update_update_stats() - - else: - r_serv_db.set('ail:current_background_script_stat', 100) - sys.exit(0) diff --git a/update/v3.5/Update.py b/update/v3.5/Update.py deleted file mode 100755 index 651fe8ea..00000000 --- a/update/v3.5/Update.py +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* - -import os -import sys - -sys.path.append(os.environ['AIL_HOME']) -################################## -# Import Project packages -################################## -from update.bin.old_ail_updater import AIL_Updater - -class Updater(AIL_Updater): - """default Updater.""" - - def __init__(self, version): - super(Updater, self).__init__(version) - - -if __name__ == '__main__': - updater = Updater('v3.5') - updater.run_update() diff --git a/update/v3.5/Update.sh b/update/v3.5/Update.sh deleted file mode 100755 index ea0d39c3..00000000 --- a/update/v3.5/Update.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash - -[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1; - -export PATH=$AIL_HOME:$PATH -export PATH=$AIL_REDIS:$PATH -export PATH=$AIL_ARDB:$PATH -export PATH=$AIL_BIN:$PATH -export PATH=$AIL_FLASK:$PATH - -GREEN="\\033[1;32m" -DEFAULT="\\033[0;39m" - -echo -e $GREEN"Shutting down AIL ..."$DEFAULT -bash ${AIL_BIN}/LAUNCH.sh -ks -wait - -# SUBMODULES # -git submodule update - -echo -e $GREEN"Installing PyAIL ..."$DEFAULT -pip3 install -U pyail - -echo -e $GREEN"Installing D4 CLIENT ..."$DEFAULT -pip3 install -U d4-pyclient - -echo "" -echo -e $GREEN"Updating DomainClassifier ..."$DEFAULT -pip3 install -U DomainClassifier - -exit 0 diff --git a/update/v3.6/Update.py b/update/v3.6/Update.py deleted file mode 100755 index 8d7715de..00000000 --- a/update/v3.6/Update.py +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* - -import os -import sys - -sys.path.append(os.environ['AIL_HOME']) -################################## -# Import Project packages -################################## -from update.bin.old_ail_updater import AIL_Updater - -class Updater(AIL_Updater): - """default Updater.""" - - def __init__(self, version): - super(Updater, self).__init__(version) - - -if __name__ == '__main__': - updater = Updater('v3.6') - updater.run_update() diff --git a/update/v3.6/Update.sh b/update/v3.6/Update.sh deleted file mode 100755 index e82dba98..00000000 --- a/update/v3.6/Update.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash - -[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1; - -export PATH=$AIL_HOME:$PATH -export PATH=$AIL_REDIS:$PATH -export PATH=$AIL_ARDB:$PATH -export PATH=$AIL_BIN:$PATH -export PATH=$AIL_FLASK:$PATH - -GREEN="\\033[1;32m" -DEFAULT="\\033[0;39m" - -echo -e $GREEN"Shutting down AIL ..."$DEFAULT -bash ${AIL_BIN}/LAUNCH.sh -ks -wait - -# SUBMODULES # -git submodule update - -echo -e $GREEN"Updating D4 CLIENT ..."$DEFAULT -pip3 install -U d4-pyclient - -echo "" -echo -e $GREEN"Installing nose ..."$DEFAULT -pip3 install -U nose - -echo -e $GREEN"Installing coverage ..."$DEFAULT -pip3 install -U coverage - -echo "" -echo -e $GREEN"Updating DomainClassifier ..."$DEFAULT -pip3 install -U DomainClassifier - -exit 0 diff --git a/update/v3.7/Update.py b/update/v3.7/Update.py deleted file mode 100755 index 8b238f9c..00000000 --- a/update/v3.7/Update.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* - -import os -import sys - -sys.path.append(os.environ['AIL_HOME']) -################################## -# Import Project packages -################################## -from update.bin.old_ail_updater import AIL_Updater -from lib import Tracker - -class Updater(AIL_Updater): - """default Updater.""" - - def __init__(self, version): - super(Updater, self).__init__(version) - - def update(self): - """ - Update Domain Languages - """ - print('Fixing Tracker_uuid list ...') - Tracker.fix_all_tracker_uuid_list() - nb = 0 - for tracker_uuid in Tracker.get_trackers(): - self.r_serv.sadd('trackers_update_v3.7', tracker_uuid) - nb += 1 - - self.r_serv.set('update:nb_elem_to_convert', nb) - self.r_serv.set('update:nb_elem_converted',0) - - # Add background update - self.r_serv.sadd('ail:to_update', self.version) - - -if __name__ == '__main__': - updater = Updater('v3.7') - updater.run_update() diff --git a/update/v3.7/Update.sh b/update/v3.7/Update.sh deleted file mode 100755 index 0196d872..00000000 --- a/update/v3.7/Update.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash - -[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1; - -export PATH=$AIL_HOME:$PATH -export PATH=$AIL_REDIS:$PATH -export PATH=$AIL_ARDB:$PATH -export PATH=$AIL_BIN:$PATH -export PATH=$AIL_FLASK:$PATH - -GREEN="\\033[1;32m" -DEFAULT="\\033[0;39m" - -echo -e $GREEN"Shutting down AIL ..."$DEFAULT -bash ${AIL_BIN}/LAUNCH.sh -ks -wait - -# SUBMODULES # -git submodule update - - -echo -e $GREEN"Updating thirdparty ..."$DEFAULT -bash ${AIL_BIN}/LAUNCH.sh -ut -wait - -echo "" -echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT -echo "" -python ${AIL_HOME}/update/v3.7/Update.py -wait -echo "" -echo "" - - -echo "" -echo -e $GREEN"Shutting down ARDB ..."$DEFAULT -bash ${AIL_BIN}/LAUNCH.sh -ks -wait - -exit 0 diff --git a/update/v3.7/Update_trackers.py b/update/v3.7/Update_trackers.py deleted file mode 100644 index b84f06e8..00000000 --- a/update/v3.7/Update_trackers.py +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* - -import os -import sys -import time - -sys.path.append(os.environ['AIL_BIN']) -################################## -# Import Project packages -################################## -from lib import ConfigLoader -from lib import Tracker - -def update_update_stats(): - nb_updated = int(r_serv_db.get('update:nb_elem_converted')) - progress = int((nb_updated * 100) / nb_elem_to_update) - print(f'{nb_updated}/{nb_elem_to_update} updated {progress}%') - r_serv_db.set('ail:current_background_script_stat', progress) - - -if __name__ == '__main__': - start_deb = time.time() - - config_loader = ConfigLoader.ConfigLoader() - r_serv_db = config_loader.get_redis_conn("ARDB_DB") - r_serv_tracker = config_loader.get_db_conn("Kvrocks_DB") - config_loader = None - Tracker.r_serv_tracker = r_serv_tracker - - r_serv_db.set('ail:current_background_script', 'trackers update') - - nb_elem_to_update = r_serv_db.get('update:nb_elem_to_convert') - if not nb_elem_to_update: - nb_elem_to_update = 1 - else: - nb_elem_to_update = int(nb_elem_to_update) - - while True: - tracker_uuid = r_serv_db.spop('trackers_update_v3.7') - if tracker_uuid is not None: - print(tracker_uuid) - # FIX STATS - Tracker.fix_tracker_stats_per_day(tracker_uuid) - # MAP TRACKER - ITEM_ID - Tracker.fix_tracker_item_link(tracker_uuid) - - r_serv_db.incr('update:nb_elem_converted') - update_update_stats() - - else: - r_serv_db.set('ail:current_background_script_stat', 100) - sys.exit(0) diff --git a/update/v4.0/Update.py b/update/v4.0/Update.py deleted file mode 100755 index b609ebb6..00000000 --- a/update/v4.0/Update.py +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* - -import os -import sys - -sys.path.append(os.environ['AIL_HOME']) -################################## -# Import Project packages -################################## -from update.bin.old_ail_updater import AIL_Updater - -class Updater(AIL_Updater): - """default Updater.""" - - def __init__(self, version): - super(Updater, self).__init__(version) - - -if __name__ == '__main__': - updater = Updater('v4.0') - updater.run_update() diff --git a/update/v4.0/Update.sh b/update/v4.0/Update.sh deleted file mode 100755 index 09bc3f4f..00000000 --- a/update/v4.0/Update.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1; - -export PATH=$AIL_HOME:$PATH -export PATH=$AIL_REDIS:$PATH -export PATH=$AIL_ARDB:$PATH -export PATH=$AIL_BIN:$PATH -export PATH=$AIL_FLASK:$PATH - -GREEN="\\033[1;32m" -DEFAULT="\\033[0;39m" - -echo -e $GREEN"Shutting down AIL ..."$DEFAULT -bash ${AIL_BIN}/LAUNCH.sh -ks -wait - -# SUBMODULES # -git submodule update - -echo "" -echo -e $GREEN"Installing nose ..."$DEFAULT -pip3 install -U websockets - -exit 0 diff --git a/update/v4.1/Update.py b/update/v4.1/Update.py deleted file mode 100755 index f7a7a72a..00000000 --- a/update/v4.1/Update.py +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* - -import os -import sys -import redis - -sys.path.append(os.environ['AIL_HOME']) -################################## -# Import Project packages -################################## -from update.bin.old_ail_updater import AIL_Updater - -class Updater(AIL_Updater): - """default Updater.""" - - def __init__(self, version): - super(Updater, self).__init__(version) - - def update(self): - r_tracking = redis.StrictRedis(host='localhost', - port=6382, - db=2, - decode_responses=True) - # FLUSH OLD DB - r_tracking.flushdb() - - -if __name__ == '__main__': - updater = Updater('v4.1') - updater.run_update() diff --git a/update/v4.1/Update.sh b/update/v4.1/Update.sh deleted file mode 100755 index 2be5376a..00000000 --- a/update/v4.1/Update.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1; - -export PATH=$AIL_HOME:$PATH -export PATH=$AIL_REDIS:$PATH -export PATH=$AIL_ARDB:$PATH -export PATH=$AIL_BIN:$PATH -export PATH=$AIL_FLASK:$PATH - -GREEN="\\033[1;32m" -DEFAULT="\\033[0;39m" - -echo -e $GREEN"Shutting down AIL ..."$DEFAULT -bash ${AIL_BIN}/LAUNCH.sh -ks -wait - -# SUBMODULES # -git submodule update - -exit 0 diff --git a/update/v4.2.1/Update.py b/update/v4.2.1/Update.py deleted file mode 100755 index f20a9184..00000000 --- a/update/v4.2.1/Update.py +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* - -import os -import sys - -sys.path.append(os.environ['AIL_HOME']) -################################## -# Import Project packages -################################## -from update.bin.old_ail_updater import AIL_Updater - -class Updater(AIL_Updater): - """default Updater.""" - - def __init__(self, version): - super(Updater, self).__init__(version) - - -if __name__ == '__main__': - updater = Updater('v4.2.1') - updater.run_update() diff --git a/update/v4.2.1/Update.sh b/update/v4.2.1/Update.sh deleted file mode 100755 index bf04638b..00000000 --- a/update/v4.2.1/Update.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1; - -export PATH=$AIL_HOME:$PATH -export PATH=$AIL_REDIS:$PATH -export PATH=$AIL_ARDB:$PATH -export PATH=$AIL_BIN:$PATH -export PATH=$AIL_FLASK:$PATH - -GREEN="\\033[1;32m" -DEFAULT="\\033[0;39m" - -echo -e $GREEN"Shutting down AIL ..."$DEFAULT -bash ${AIL_BIN}/LAUNCH.sh -ks -wait - -# SUBMODULES # -git submodule update - -echo "" -echo -e $GREEN"Updating pusblogger ..."$DEFAULT -pip3 install -U pubsublogger - -exit 0 diff --git a/update/v4.2/Update.py b/update/v4.2/Update.py deleted file mode 100755 index 708009b2..00000000 --- a/update/v4.2/Update.py +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* - -import os -import sys - -sys.path.append(os.environ['AIL_HOME']) -################################## -# Import Project packages -################################## -from update.bin.old_ail_updater import AIL_Updater - -class Updater(AIL_Updater): - """default Updater.""" - - def __init__(self, version): - super(Updater, self).__init__(version) - - -if __name__ == '__main__': - updater = Updater('v4.2') - updater.run_update() diff --git a/update/v4.2/Update.sh b/update/v4.2/Update.sh deleted file mode 100755 index a18aae61..00000000 --- a/update/v4.2/Update.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash - -[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1; - -export PATH=$AIL_HOME:$PATH -export PATH=$AIL_REDIS:$PATH -export PATH=$AIL_ARDB:$PATH -export PATH=$AIL_BIN:$PATH -export PATH=$AIL_FLASK:$PATH - -GREEN="\\033[1;32m" -DEFAULT="\\033[0;39m" - -echo -e $GREEN"Shutting down AIL ..."$DEFAULT -bash ${AIL_BIN}/LAUNCH.sh -ks -wait - -# SUBMODULES # -git submodule update - -echo "" -echo -e $GREEN"Installing typo-squatting ..."$DEFAULT -pip3 install -U ail_typo_squatting - -echo "" -echo -e $GREEN"Updating d4-client ..."$DEFAULT -pip3 install -U d4-pyclient - -exit 0