From 2127b4363fe158663c98c01c32939e2f8a51f580 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Wed, 17 May 2023 15:46:44 +0200 Subject: [PATCH 1/3] chg: [kvrocks] add migration notice --- HOWTO.md | 88 +++++++------------ bin/Update.py | 45 +++++++--- .../modules/dashboard/templates/index.html | 4 +- var/www/templates/dashboard/update_modal.html | 67 ++++++++------ 4 files changed, 104 insertions(+), 100 deletions(-) diff --git a/HOWTO.md b/HOWTO.md index 2bb034f6..fda7f511 100644 --- a/HOWTO.md +++ b/HOWTO.md @@ -78,21 +78,6 @@ If you want to add a new processing or analysis module in AIL, follow these simp 2. Use [./bin/template.py](./bin/template.py) as a sample module and create a new file in bin/ with the module name used in the modules.cfg configuration. -How to create a new webpage ---------------------------- - -If you want to add a new webpage for a module in AIL, follow these simple steps: - -1. Launch [./var/www/create_new_web_module.py](./var/www/create_new_web_module.py) and enter the name to use for your webpage (Usually, your newly created python module). - -2. A template and flask skeleton has been created for your new webpage in [./var/www/modules/](./var/www/modules/) - -3. Edit the created html files under the template folder as well as the Flask_* python script so that they fit your needs. - -4. You can change the order of your module in the top navigation header in the file [./var/www/templates/header_base.html](./var/www/templates/header_base.html) - -5. You can ignore module, and so, not display them in the top navigation header by adding the module name in the file [./var/www/templates/ignored_modules.txt](./var/www/templates/ignored_modules.txt) - How to contribute a module -------------------------- @@ -104,61 +89,54 @@ To contribute your module, feel free to pull your contribution. Additional information ====================== -Manage modules: ModulesInformationV2.py ---------------------------------------- - -You can do a lots of things easily with the [./bin/ModulesInformationV2](./bin/ModulesInformationV2) script: - -- Monitor the health of other modules -- Monitor the ressources comsumption of other modules -- Start one or more modules -- Kill running modules -- Restart automatically stuck modules -- Show the paste currently processed by a module - -### Navigation - -You can navigate into the interface by using arrow keys. In order to perform an action on a selected module, you can either press or to show the dialog box. - -To change list, you can press the key. - -Also, you can quickly stop or start modules by clicking on the ```` or ```` symbol respectively. These are located in the _Action_ column. - -Finally, you can quit this program by pressing either ```` or ````. - - - Crawler --------------------- -In AIL, you can crawl Tor hidden services. Don't forget to review the proxy configuration of your Tor client and especially if you enabled the SOCKS5 proxy and binding on the appropriate IP address reachable via the dockers where Splash runs. +In AIL, you can crawl websites and Tor hidden services. Don't forget to review the proxy configuration of your Tor client and especially if you enabled the SOCKS5 proxy + +[//]: # (and binding on the appropriate IP address reachable via the dockers where Splash runs.) ### Installation -[Install AIL-Splash-Manager](https://github.com/ail-project/ail-splash-manager) +[Install Lacus](https://github.com/ail-project/lacus) ### Configuration -1. Search the Splash-Manager API key. This API key is generated when you launch the manager for the first time. -(located in your Splash Manager directory ``ail-splash-manager/token_admin.txt``) - - -2. Splash Manager URL and API Key: +1. Lacus URL: In the webinterface, go to ``Crawlers>Settings`` and click on the Edit button -![Splash Manager Config](./doc/screenshots/splash_manager_config_edit_1.png?raw=true "AIL framework Splash Manager Config") -![Splash Manager Config](./doc/screenshots/splash_manager_config_edit_2.png?raw=true "AIL framework Splash Manager Config") +![Splash Manager Config](./doc/screenshots/lacus_config.png?raw=true "AIL Lacus Config") -3. Launch AIL Crawlers: +![Splash Manager Config](./doc/screenshots/lacus_config_edit.png?raw=true "AIL Lacus Config") + +2. Launch AIL Crawlers: Choose the number of crawlers you want to launch -![Splash Manager Nb Crawlers Config](./doc/screenshots/splash_manager_nb_crawlers_1.png?raw=true "AIL framework Nb Crawlers Config") -![Splash Manager Nb Crawlers Config](./doc/screenshots/splash_manager_nb_crawlers_2.png?raw=true "AIL framework Nb Crawlers Config") + +![Splash Manager Nb Crawlers Config](./doc/screenshots/crawler_nb_captures.png?raw=true "AIL Lacus Nb Crawlers Config") +![Splash Manager Nb Crawlers Config](./doc/screenshots/crawler_nb_captures_edit.png?raw=true "AIL Lacus Nb Crawlers Config") +Kvrocks Migration +--------------------- +**Important Note: +We are currently working on a [migration script](https://github.com/ail-project/ail-framework/blob/master/bin/DB_KVROCKS_MIGRATION.py) to facilitate the migration to Kvrocks. +Once this script is ready, AIL version 5.0 will be released.** -#### Old updates +Please note that the current version of this migration script only supports migrating the database on the same server. +(If you plan to migrate to another server, we will provide additional instructions in this section once the migration script is completed) -##### Python 3 Upgrade - -To upgrade from an existing AIL installation, you have to launch [python3_upgrade.sh](./python3_upgrade.sh), this script will delete and create a new virtual environment. The script **will upgrade the packages but won't keep your previous data** (neverthless the data is copied into a directory called `old`). If you install from scratch, you don't require to launch the [python3_upgrade.sh](./python3_upgrade.sh). +To migrate your database to Kvrocks: +1. Launch ARDB and Kvrocks +2. Pull from remote + ``` + git checkout master + git pull + ``` +3. Launch the migration script: + ``` + git checkout master + git pull + cd bin/ + ./DB_KVROCKS_MIGRATION.py + ``` diff --git a/bin/Update.py b/bin/Update.py index 5e885301..76308d0f 100755 --- a/bin/Update.py +++ b/bin/Update.py @@ -33,6 +33,7 @@ def auto_update_enabled(cfg): # check if files are modify locally def check_if_files_modified(): + # return True process = subprocess.run(['git', 'ls-files' ,'-m'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) if process.returncode == 0: modified_files = process.stdout @@ -57,6 +58,7 @@ def check_if_files_modified(): sys.exit(1) def repo_is_fork(): + # return False print('Check if this repository is a fork:') process = subprocess.run(['git', 'remote', '-v'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -155,7 +157,7 @@ def get_git_current_tag(current_version_path): with open(current_version_path, 'r') as version_content: version = version_content.read() except FileNotFoundError: - version = 'v1.4' + version = 'v5.0' with open(current_version_path, 'w') as version_content: version_content.write(version) @@ -229,9 +231,14 @@ def get_git_upper_tags_remote(current_tag, is_fork): except ValueError: continue - # add tag with last commit - if float(tag_val) >= float(current_tag_val): - dict_tags_commit[tag] = commit + if float(current_tag) < 5.0: + # add tag with last commit + if float(current_tag_val) <= float(tag_val) < float(5.0): + dict_tags_commit[tag] = commit + else: + # add tag with last commit + if float(tag_val) >= float(current_tag_val): + dict_tags_commit[tag] = commit list_upper_tags = [('v{}'.format(key), dict_tags_commit[key]) for key in dict_tags_commit] # force update order list_upper_tags.sort() @@ -260,13 +267,28 @@ def update_ail(current_tag, list_upper_tags_remote, current_version_path, is_for update_submodules() - print('{}git pull:{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT)) - process = subprocess.run(['git', 'pull'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + temp_current_tag = current_tag.replace('v', '') + if temp_current_tag.count('.') > 1: + temp_current_tag = temp_current_tag.rsplit('.', 1) + temp_current_tag = ''.join(temp_current_tag) - if process.returncode == 0: - output = process.stdout.decode() - print(output) + if float(temp_current_tag) < 5.0: + roll_back_update('01f459109fa36473f66cc9c400c27a4cebf77858') + pulled = True + else: + print('{}git pull:{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT)) + process = subprocess.run(['git', 'pull'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if process.returncode == 0: + output = process.stdout.decode() + print(output) + pulled = True + else: + print('{}{}{}'.format(TERMINAL_RED, process.stderr.decode(), TERMINAL_DEFAULT)) + aborting_update() + pulled = False + sys.exit(1) + if pulled: # CHECK IF UPDATER Update if float(os.stat(UPDATER_FILENAME).st_mtime) > UPDATER_LAST_MODIFICATION: # request updater relauch @@ -310,10 +332,7 @@ def update_ail(current_tag, list_upper_tags_remote, current_version_path, is_for print('{}**************** AIL Sucessfully Updated *****************{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT)) print() sys.exit(0) - else: - print('{}{}{}'.format(TERMINAL_RED, process.stderr.decode(), TERMINAL_DEFAULT)) - aborting_update() - sys.exit(1) + else: print('{}{}{}'.format(TERMINAL_RED, process.stderr.decode(), TERMINAL_DEFAULT)) aborting_update() diff --git a/var/www/modules/dashboard/templates/index.html b/var/www/modules/dashboard/templates/index.html index 35d5188f..8fa42745 100644 --- a/var/www/modules/dashboard/templates/index.html +++ b/var/www/modules/dashboard/templates/index.html @@ -73,9 +73,7 @@ {%endif%} - + {% include 'dashboard/update_modal.html' %}
diff --git a/var/www/templates/dashboard/update_modal.html b/var/www/templates/dashboard/update_modal.html index 54bee18e..a9bef03e 100644 --- a/var/www/templates/dashboard/update_modal.html +++ b/var/www/templates/dashboard/update_modal.html @@ -1,40 +1,49 @@