From 5c4db5f2f1d4aa3a6d872fe28a8e5369ec9f1ae1 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Thu, 11 Apr 2019 17:49:20 +0200 Subject: [PATCH] chg: [update] add update script Redis ARDB + add background update --- bin/LAUNCH.sh | 8 ++- bin/Update.py | 2 +- bin/update-background.py | 58 +++++++++++++++++++++ installing_deps.sh | 2 +- update/bin/Update_ARDB.sh | 18 +++++++ update/bin/Update_Redis.sh | 27 ++++++++++ update/v1.4/Update-ARDB_Metadata.py | 43 +++++++++++----- update/v1.4/Update-ARDB_Onions.py | 29 ++++------- update/v1.4/Update-ARDB_Tags.py | 8 +++ update/v1.4/Update-ARDB_Tags_background.py | 8 +++ update/v1.4/Update.py | 60 ++++++++++++++++++++++ update/v1.4/Update.sh | 29 +++++++++++ 12 files changed, 257 insertions(+), 35 deletions(-) create mode 100755 bin/update-background.py create mode 100755 update/bin/Update_ARDB.sh create mode 100755 update/bin/Update_Redis.sh create mode 100755 update/v1.4/Update.py create mode 100755 update/v1.4/Update.sh diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index 74a54196..aa658f4e 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -465,7 +465,7 @@ function launch_all { if [[ "${choices[i]}" ]]; then case ${options[i]} in Redis) - launch_redis + launch_redis; ;; Ardb) launch_ardb; @@ -507,12 +507,16 @@ function launch_all { exit } -echo "$@" +#echo "$@" while [ "$1" != "" ]; do case $1 in -l | --launchAuto ) launch_all "automatic"; ;; + -lr | --launchRedis ) launch_redis; + ;; + -la | --launchARDB ) launch_ardb; + ;; -k | --killAll ) killall; ;; -u | --update ) update; diff --git a/bin/Update.py b/bin/Update.py index bfe60e01..8a7556dd 100755 --- a/bin/Update.py +++ b/bin/Update.py @@ -259,7 +259,7 @@ def launch_update_version(version, roll_back_commit, current_version_path, is_fo print('{}------------------------------------------------------------------'.format(TERMINAL_YELLOW)) print('- Launching Update: {}{}{} -'.format(TERMINAL_BLUE, version, TERMINAL_YELLOW)) print('-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --{}'.format(TERMINAL_DEFAULT)) - process = subprocess.Popen(['bash', update_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + process = subprocess.Popen(['unbuffer', 'bash', update_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE) while True: output = process.stdout.readline().decode() if output == '' and process.poll() is not None: diff --git a/bin/update-background.py b/bin/update-background.py new file mode 100755 index 00000000..ede08c95 --- /dev/null +++ b/bin/update-background.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +""" +Update AIL +============================ + +Update AIL clone and fork + +""" + +import os +import sys +import redis +import subprocess +import configparser + +if __name__ == "__main__": + + configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') + if not os.path.exists(configfile): + raise Exception('Unable to find the configuration file. \ + Did you set environment variables? \ + Or activate the virtualenv.') + cfg = configparser.ConfigParser() + cfg.read(configfile) + + r_serv = redis.StrictRedis( + host=cfg.get("ARDB_DB", "host"), + port=cfg.getint("ARDB_DB", "port"), + db=cfg.getint("ARDB_DB", "db"), + decode_responses=True) + + ail_version = r_serv.get('ail:version') + if ail_version is None: + sys.exit(0) + else: + if ail_version == 'v1.5': + onions_update_status = r_serv.get('v1.5:onions') + if onions_update_status is None: + if int(onions_update_status) != 1: + update_file = os.path.join(os.environ['AIL_HOME'], 'update', 'v1.4', 'Update-ARDB_Onions.py') + process = subprocess.run(['unbuffer', 'python' ,'update_file'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + metadata_update_status = r_serv.get('v1.5:metadata') + if metadata_update_status is None: + if int(metadata_update_status) != 1: + pass + #launch update + tags_update_status = r_serv.get('v1.5:tags') + if tags_update_status is None: + if int(tags_update_status) != 1: + pass + #launch update + tags_background_update_status = r_serv.get('v1.5:tags_background') + if tags_background_update_status is None: + if int(tags_background_update_status) != 1: + pass + #launch update diff --git a/installing_deps.sh b/installing_deps.sh index 579c2a9d..fdadd9e7 100755 --- a/installing_deps.sh +++ b/installing_deps.sh @@ -39,7 +39,7 @@ sudo apt-get install p7zip-full -y # REDIS # test ! -d redis/ && git clone https://github.com/antirez/redis.git pushd redis/ -git checkout 3.2 +git checkout 5.0 make popd diff --git a/update/bin/Update_ARDB.sh b/update/bin/Update_ARDB.sh new file mode 100755 index 00000000..2544973e --- /dev/null +++ b/update/bin/Update_ARDB.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +echo "Killing all screens ..." +bash -c "bash ../../bin/LAUNCH.sh -k" +echo "" +echo "Updating ARDB ..." +pushd ../../ +rm -r ardb +pushd ardb/ +git clone https://github.com/yinqiwen/ardb.git +git checkout 0.10 || exit 1 +make || exit 1 +popd +popd +echo "ARDB Updated" +echo "" + +exit 0 diff --git a/update/bin/Update_Redis.sh b/update/bin/Update_Redis.sh new file mode 100755 index 00000000..238d53f7 --- /dev/null +++ b/update/bin/Update_Redis.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1; + +export PATH=$AIL_HOME:$PATH +export PATH=$AIL_REDIS:$PATH +export PATH=$AIL_ARDB:$PATH +export PATH=$AIL_BIN:$PATH +export PATH=$AIL_FLASK:$PATH + +echo "Killing all screens ..." +bash -c "bash ${AIL_BIN}/LAUNCH.sh -k" +echo "" +echo "Updating Redis ..." +pushd $AIL_HOME/redis +git pull || exit 1 +git checkout 5.0 || exit 1 +make || exit 1 +popd +echo "Redis Updated" +echo "" + +exit 0 diff --git a/update/v1.4/Update-ARDB_Metadata.py b/update/v1.4/Update-ARDB_Metadata.py index 5573cede..757fdc9a 100755 --- a/update/v1.4/Update-ARDB_Metadata.py +++ b/update/v1.4/Update-ARDB_Metadata.py @@ -10,27 +10,35 @@ import configparser def update_hash_item(has_type): #get all hash items: - #all_base64 = r_serv_tag.smembers('infoleak:automatic-detection=\"{}\"'.format(has_type)) - all_hash_items = r_serv_tag.smembers('infoleak:automatic-detection=\"{}\":20180925'.format(has_type)) + all_hash_items = r_serv_tag.smembers('infoleak:automatic-detection=\"{}\"'.format(has_type)) for item_path in all_hash_items: - item_path = '/home/aurelien/git/python3/AIL-framework/PASTES/archive/pastebin.com_pro/2018/09/25/Fu9akJaz.gz' if PASTES_FOLDER in item_path: base64_key = '{}_paste:{}'.format(has_type, item_path) hash_key = 'hash_paste:{}'.format(item_path) - ## TODO: catch error if r_serv_metadata.exists(base64_key): - res = r_serv_metadata.renamenx(base64_key, base64_key.replace(PASTES_FOLDER, '', 1)) - ## TODO: key merge - if not res: + new_base64_key = base64_key.replace(PASTES_FOLDER, '', 1) + res = r_serv_metadata.renamenx(base64_key, new_base64_key) + print(res) + if res == 0: print('same key, double name: {}'.format(item_path)) + # fusion + all_key = r_serv_metadata.smembers(base64_key) + for elem in all_key: + r_serv_metadata.sadd(new_base64_key, elem) + r_serv_metadata.srem(base64_key, elem) if r_serv_metadata.exists(hash_key): - ## TODO: catch error - res = r_serv_metadata.renamenx(hash_key, hash_key.replace(PASTES_FOLDER, '', 1)) - ## TODO: key merge - if not res: + new_hash_key = hash_key.replace(PASTES_FOLDER, '', 1) + res = r_serv_metadata.renamenx(hash_key, new_hash_key) + print(res) + if res == 0: print('same key, double name: {}'.format(item_path)) + # fusion + all_key = r_serv_metadata.smembers(hash_key) + for elem in all_key: + r_serv_metadata.sadd(new_hash_key, elem) + r_serv_metadata.srem(hash_key, elem) if __name__ == '__main__': @@ -46,6 +54,12 @@ if __name__ == '__main__': PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], cfg.get("Directories", "pastes")) + '/' + r_serv = redis.StrictRedis( + host=cfg.get("ARDB_DB", "host"), + port=cfg.getint("ARDB_DB", "port"), + db=cfg.getint("ARDB_DB", "db"), + decode_responses=True) + r_serv_metadata = redis.StrictRedis( host=cfg.get("ARDB_Metadata", "host"), port=cfg.getint("ARDB_Metadata", "port"), @@ -84,8 +98,11 @@ if __name__ == '__main__': old_item_metadata = 'paste_metadata:{}'.format(item_path) item_path = item_path.replace(PASTES_FOLDER, '', 1) new_item_metadata = 'paste_metadata:{}'.format(item_path) - ## TODO: catch error res = r_serv_metadata.renamenx(old_item_metadata, new_item_metadata) + #key already exist + if res == 0: + r_serv_metadata.delete(old_item_metadata) + # update domain port domain = r_serv_metadata.hget(new_item_metadata, 'domain') if domain: @@ -244,3 +261,5 @@ if __name__ == '__main__': print('Updating ARDB_Metadata Done => {} paths: {} s'.format(index, end - start)) print() + + r_serv.set('v1.5:metadata', 1) diff --git a/update/v1.4/Update-ARDB_Onions.py b/update/v1.4/Update-ARDB_Onions.py index 10709069..7dbd0550 100755 --- a/update/v1.4/Update-ARDB_Onions.py +++ b/update/v1.4/Update-ARDB_Onions.py @@ -27,9 +27,9 @@ def get_domain_root_from_paste_childrens(item_father, domain): for item_path in item_children: # remove absolute_path if PASTES_FOLDER in item_path: - #r_serv_metadata.srem('paste_children:{}'.format(item_father), item_path) + r_serv_metadata.srem('paste_children:{}'.format(item_father), item_path) item_path = item_path.replace(PASTES_FOLDER, '', 1) - #r_serv_metadata.sadd('paste_children:{}'.format(item_father), item_path) + r_serv_metadata.sadd('paste_children:{}'.format(item_father), item_path) if domain in item_path: domain_root = item_path return domain_root @@ -49,6 +49,12 @@ if __name__ == '__main__': PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], cfg.get("Directories", "pastes")) + '/' + r_serv = redis.StrictRedis( + host=cfg.get("ARDB_DB", "host"), + port=cfg.getint("ARDB_DB", "port"), + db=cfg.getint("ARDB_DB", "db"), + decode_responses=True) + r_serv_metadata = redis.StrictRedis( host=cfg.get("ARDB_Metadata", "host"), port=cfg.getint("ARDB_Metadata", "port"), @@ -99,7 +105,6 @@ if __name__ == '__main__': for date_history in all_onion_history: print('--------') print('onion_history:{}:{}'.format(onion_domain, date_history)) - #item_father = r_serv_onion.lpop('onion_history:{}:{}'.format(onion_domain, date_history)) item_father = r_serv_onion.lrange('onion_history:{}:{}'.format(onion_domain, date_history), 0, 0) print('item_father: {}'.format(item_father)) item_father = item_father[0] @@ -120,23 +125,9 @@ if __name__ == '__main__': r_serv_onion.hdel('onion_metadata:{}'.format(onion_domain), 'last_seen') - ''' - # update crawler queue - for elem in r_serv_onion.smembers('onion_crawler_queue'): - if PASTES_FOLDER in elem: - r_serv_onion.srem('onion_crawler_queue', elem) - r_serv_onion.sadd('onion_crawler_queue', elem.replace(PASTES_FOLDER, '', 1)) - index = index +1 - for elem in r_serv_onion.smembers('onion_crawler_priority_queue'): - if PASTES_FOLDER in elem: - r_serv_onion.srem('onion_crawler_queue', elem) - r_serv_onion.sadd('onion_crawler_queue', elem.replace(PASTES_FOLDER, '', 1)) - index = index +1 - - ''' - - end = time.time() print('Updating ARDB_Onion Done => {} paths: {} s'.format(index, end - start)) print() print('Done in {} s'.format(end - start_deb)) + + r_serv.set('v1.5:onions', 1) diff --git a/update/v1.4/Update-ARDB_Tags.py b/update/v1.4/Update-ARDB_Tags.py index fa26e6e0..9928ec77 100755 --- a/update/v1.4/Update-ARDB_Tags.py +++ b/update/v1.4/Update-ARDB_Tags.py @@ -29,6 +29,12 @@ if __name__ == '__main__': PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], cfg.get("Directories", "pastes")) + '/' + r_serv = redis.StrictRedis( + host=cfg.get("ARDB_DB", "host"), + port=cfg.getint("ARDB_DB", "port"), + db=cfg.getint("ARDB_DB", "db"), + decode_responses=True) + r_serv_metadata = redis.StrictRedis( host=cfg.get("ARDB_Metadata", "host"), port=cfg.getint("ARDB_Metadata", "port"), @@ -121,3 +127,5 @@ if __name__ == '__main__': print('Updating ARDB_Tags Done => {} paths: {} s'.format(index, end - start)) + + r_serv.set('v1.5:tags', 1) diff --git a/update/v1.4/Update-ARDB_Tags_background.py b/update/v1.4/Update-ARDB_Tags_background.py index 8333e96c..2dc050b7 100755 --- a/update/v1.4/Update-ARDB_Tags_background.py +++ b/update/v1.4/Update-ARDB_Tags_background.py @@ -29,6 +29,12 @@ if __name__ == '__main__': PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], cfg.get("Directories", "pastes")) + '/' + r_serv = redis.StrictRedis( + host=cfg.get("ARDB_DB", "host"), + port=cfg.getint("ARDB_DB", "port"), + db=cfg.getint("ARDB_DB", "db"), + decode_responses=True) + r_serv_metadata = redis.StrictRedis( host=cfg.get("ARDB_Metadata", "host"), port=cfg.getint("ARDB_Metadata", "port"), @@ -68,3 +74,5 @@ if __name__ == '__main__': print('Updating ARDB_Tags Done: {} s'.format(end - start)) + + r_serv.set('v1.5:tags_background', 1) diff --git a/update/v1.4/Update.py b/update/v1.4/Update.py new file mode 100755 index 00000000..c4b5fdfe --- /dev/null +++ b/update/v1.4/Update.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import sys +import time +import redis +import configparser + +if __name__ == '__main__': + + start_deb = time.time() + + configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') + if not os.path.exists(configfile): + raise Exception('Unable to find the configuration file. \ + Did you set environment variables? \ + Or activate the virtualenv.') + cfg = configparser.ConfigParser() + cfg.read(configfile) + + PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], cfg.get("Directories", "pastes")) + '/' + + r_serv = redis.StrictRedis( + host=cfg.get("ARDB_DB", "host"), + port=cfg.getint("ARDB_DB", "port"), + db=cfg.getint("ARDB_DB", "db"), + decode_responses=True) + + r_serv_onion = redis.StrictRedis( + host=cfg.get("ARDB_Onion", "host"), + port=cfg.getint("ARDB_Onion", "port"), + db=cfg.getint("ARDB_Onion", "db"), + decode_responses=True) + + print() + print('Updating ARDB_Onion ...') + index = 0 + start = time.time() + + # update crawler queue + for elem in r_serv_onion.smembers('onion_crawler_queue'): + if PASTES_FOLDER in elem: + r_serv_onion.srem('onion_crawler_queue', elem) + r_serv_onion.sadd('onion_crawler_queue', elem.replace(PASTES_FOLDER, '', 1)) + index = index +1 + for elem in r_serv_onion.smembers('onion_crawler_priority_queue'): + if PASTES_FOLDER in elem: + r_serv_onion.srem('onion_crawler_queue', elem) + r_serv_onion.sadd('onion_crawler_queue', elem.replace(PASTES_FOLDER, '', 1)) + index = index +1 + + end = time.time() + print('Updating ARDB_Onion Done => {} paths: {} s'.format(index, end - start)) + print() + + #Set current ail version + r_serv.set('ail:version', 'v1.5') + + print('Done in {} s'.format(end - start_deb)) diff --git a/update/v1.4/Update.sh b/update/v1.4/Update.sh new file mode 100755 index 00000000..d2500213 --- /dev/null +++ b/update/v1.4/Update.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1; + +export PATH=$AIL_HOME:$PATH +export PATH=$AIL_REDIS:$PATH +export PATH=$AIL_ARDB:$PATH +export PATH=$AIL_BIN:$PATH +export PATH=$AIL_FLASK:$PATH + +echo "" +bash -c "bash ${AIL_HOME}/update/bin/Update_Redis.sh" +#bash -c "bash ${AIL_HOME}/update/bin/Update_ARDB.sh" + +echo "" +echo "Fixing ARDB ..." +echo "" +bash -c "unbuffer python ${AIL_HOME}/update/v1.5/Update.py" + +echo "Shutting down ARDB ..." +bash -c "bash ${AIL_BIN}/LAUNCH.sh -k" + +echo "" + +exit 0