chg: [UI] add server Management, check if new update/version are available + background update status + link to new realease/commits

bin/update-background.py

@@ -31,23 +31,27 @@ if __name__ == "__main__":
         db=cfg.getint("ARDB_DB", "db"),
         decode_responses=True)
 
-    if r_serv.exists('ail:update_v1.5'):
+    if r_serv.scard('ail:update_v1.5') != 4:
-        onions_update_status = r_serv.get('v1.5:onions')
+        r_serv.delete('ail:update_error')
-        if onions_update_status is None:
+        r_serv.set('ail:update_in_progress', 'v1.5')
+        r_serv.set('ail:current_background_update', 'v1.5')
+        if not r_serv.sismember('ail:update_v1.5', 'onions'):
             update_file = os.path.join(os.environ['AIL_HOME'], 'update', 'v1.4', 'Update-ARDB_Onions.py')
             process = subprocess.run(['python' ,update_file])
 
-        metadata_update_status = r_serv.get('v1.5:metadata')
+        if not r_serv.sismember('ail:update_v1.5', 'metadata'):
-        if metadata_update_status is None:
             update_file = os.path.join(os.environ['AIL_HOME'], 'update', 'v1.4', 'Update-ARDB_Metadata.py')
             process = subprocess.run(['python' ,update_file])
 
-        tags_update_status = r_serv.get('v1.5:tags')
+        if not r_serv.sismember('ail:update_v1.5', 'tags'):
-        if tags_update_status is None:
             update_file = os.path.join(os.environ['AIL_HOME'], 'update', 'v1.4', 'Update-ARDB_Tags.py')
             process = subprocess.run(['python' ,update_file])
 
-        tags_background_update_status = r_serv.get('v1.5:tags_background')
+        if if not r_serv.sismember('ail:update_v1.5', 'tags_background'):
-        if tags_background_update_status is None:
             update_file = os.path.join(os.environ['AIL_HOME'], 'update', 'v1.4', 'Update-ARDB_Tags_background.py')
             process = subprocess.run(['python' ,update_file])
+        if r_serv.scard('ail:update_v1.5') != 4:
+            r_serv.set('ail:update_error', 'Update v1.5 Failed, please relaunch the bin/update-background.py script')
+        else:
+            r_serv.delete('ail:update_in_progress')
+            r_serv.delete('ail:current_background_update', 'v1.5')

update/v1.4/Update-ARDB_Metadata.py

@@ -143,7 +143,7 @@ if __name__ == '__main__':
     print('Updating ARDB_Metadata Done => {} paths: {} s'.format(index, end - start))
     print()
 
-    r_serv.set('v1.5:metadata', 1)
+    r_serv.sadd('ail:update_v1.5', 'metadata')
 
     ##
     #Key, Dynamic Update

update/v1.4/Update-ARDB_Onions.py

@@ -134,4 +134,4 @@ if __name__ == '__main__':
     print()
     print('Done in {} s'.format(end - start_deb))
 
-    r_serv.set('v1.5:onions', 1)
+    r_serv.sadd('ail:update_v1.5', 'onions')

update/v1.4/Update-ARDB_Tags.py

@@ -130,4 +130,4 @@ if __name__ == '__main__':
 
         print('Updating ARDB_Tags Done => {} paths: {} s'.format(index, end - start))
 
-        r_serv.set('v1.5:tags', 1)
+        r_serv.sadd('ail:update_v1.5', 'tags')

update/v1.4/Update-ARDB_Tags_background.py

@@ -76,4 +76,4 @@ if __name__ == '__main__':
 
     print('Updating ARDB_Tags Done: {} s'.format(end - start))
 
-    r_serv.set('v1.5:tags_background', 1)
+    r_serv.sadd('ail:update_v1.5', 'tags_background')

update/v1.4/Update.py

@@ -57,7 +57,10 @@ if __name__ == '__main__':
 
     #Set current ail version
     r_serv.set('ail:version', 'v1.5')
-    r_serv.set('ail:update_v1.5', 1)
+
+    #Set current update_in_progress
+    r_serv.set('ail:update_in_progress', 'v1.5')
+    r_serv.set('ail:current_background_update', 'v1.5')
 
     #Set current ail version
     r_serv.set('ail:update_date_v1.5', datetime.datetime.now().strftime("%Y%m%d"))

var/www/modules/Flask_config.py

@@ -165,6 +165,8 @@ UPLOAD_FOLDER = os.path.join(os.environ['AIL_FLASK'], 'submitted')
 PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], cfg.get("Directories", "pastes")) + '/'
 SCREENSHOT_FOLDER = os.path.join(os.environ['AIL_HOME'], cfg.get("Directories", "crawled_screenshot"))
 
+REPO_ORIGIN = 'https://github.com/CIRCL/AIL-framework.git'
+
 max_dashboard_logs = int(cfg.get("Flask", "max_dashboard_logs"))
 
 # VT

var/www/modules/settings/Flask_settings.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+'''
+    Flask functions and routes for the settings modules page
+'''
+from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for
+
+import json
+import datetime
+
+import git_status
+
+# ============ VARIABLES ============
+import Flask_config
+
+app = Flask_config.app
+cfg = Flask_config.cfg
+baseUrl = Flask_config.baseUrl
+r_serv_db = Flask_config.r_serv_db
+max_preview_char = Flask_config.max_preview_char
+max_preview_modal = Flask_config.max_preview_modal
+REPO_ORIGIN = Flask_config.REPO_ORIGIN
+
+settings = Blueprint('settings', __name__, template_folder='templates')
+
+
+
+# ============ FUNCTIONS ============
+def one():
+    return 1
+
+#def get_v1.5_update_tags_backgroud_status():
+#    return '38%'
+
+def get_git_metadata():
+    dict_git = {}
+    dict_git['current_branch'] = git_status.get_current_branch()
+    dict_git['is_clone'] = git_status.is_not_fork(REPO_ORIGIN)
+    dict_git['is_working_directory_clean'] = git_status.is_working_directory_clean()
+    dict_git['current_commit'] = git_status.get_last_commit_id_from_local()
+    dict_git['last_remote_commit'] = git_status.get_last_commit_id_from_remote()
+    dict_git['last_local_tag'] = git_status.get_last_tag_from_local()
+    dict_git['last_remote_tag'] = git_status.get_last_tag_from_remote()
+
+    # # DEBUG:
+    dict_git['last_local_tag'] = 'v1.3'
+    dict_git['last_remote_commit'] = '234328439828943843839'
+
+    if dict_git['current_commit'] != dict_git['last_remote_commit']:
+        dict_git['new_git_update_available'] = True
+    else:
+        dict_git['new_git_update_available'] = False
+
+    if dict_git['last_local_tag'] != dict_git['last_remote_tag']:
+        dict_git['new_git_version_available'] = True
+    else:
+        dict_git['new_git_version_available'] = False
+
+    return dict_git
+
+def get_update_metadata():
+    dict_update = {}
+    dict_update['current_version'] = r_serv_db.get('ail:version')
+    dict_update['current_background_update'] = r_serv_db.get('ail:current_background_update')
+    dict_update['update_in_progress'] = r_serv_db.get('ail:update_in_progress')
+    dict_update['update_error'] = r_serv_db.get('ail:update_error')
+
+    dict_update['update_in_progress']='v1.5'
+
+    return dict_update
+# ============= ROUTES ==============
+
+@settings.route("/settings/", methods=['GET'])
+def settings_page():
+    git_metadata = get_git_metadata()
+    update_metadata = get_update_metadata()
+
+
+    return render_template("settings_index.html", git_metadata=git_metadata,
+                            update_metadata=update_metadata)
+
+# ========= REGISTRATION =========
+app.register_blueprint(settings, url_prefix=baseUrl)

var/www/modules/settings/templates/settings_index.html

@@ -0,0 +1,172 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+  <title>Server Management - AIL</title>
+	<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
+
+  <!-- Core CSS -->
+	<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/dataTables.bootstrap4.min.css') }}" rel="stylesheet">
+
+  <!-- JS -->
+	<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
+
+</head>
+<body>
+
+  {% include 'nav_bar.html' %}
+
+	<div class="container-fluid">
+		<div class="row">
+
+			{% include 'options/menu_sidebar.html' %}
+
+			<div class="col-12 col-lg-10" id="core_content">
+
+				<div class="card mb-3 mt-1">
+					<div class="card-header text-white bg-dark pb-1">
+						<h5 class="card-title">AIL-framework Status :</h5>
+					</div>
+					<div class="card-body">
+
+            <div class="row">
+              <div class="col-xl-6">
+                <div class="card text-center border-secondary">
+                  <div class="card-body px-1 py-0">
+                    <table class="table table-sm">
+                      <tbody>
+                        <tr>
+                          <td>AIL Version</td>
+                          <td>{{update_metadata['current_version']}}<a target="_blank" href="https://github.com/CIRCL/AIL-framework/releases/tag/{{update_metadata['current_version']}}" class="text-info"><small> (release note)</small></a></td>
+                        </tr>
+                        <tr
+                        {%if git_metadata['current_branch'] != 'master'%}
+                          class="table-danger"
+                        {%endif%}
+                        >
+                          <td>Current Branch</td>
+                          <td>
+                            {%if git_metadata['current_branch'] != 'master'%}
+                              <i class="fas fa-times-circle text-danger" data-toggle="tooltip" data-placement="top" title="Please checkout the master branch"></i>&nbsp;
+                            {%endif%}
+                            {{git_metadata['current_branch']}}
+                          </td>
+                        </tr>
+                        {%if git_metadata['is_clone']%}
+                          {%if update_metadata['update_in_progress']%}
+                            <tr>
+                              <td>Update in progress</td>
+                              <td>{{update_metadata['update_in_progress']}}</td>
+                            </tr>
+                          {%endif%}
+                          <tr
+                          {%if git_metadata['new_git_update_available']%}
+                            class="table-warning"
+                          {%endif%}
+                          >
+                            <td>Current Commit ID</td>
+                            <td>
+                              {%if git_metadata['new_git_update_available']%}
+                                <i class="fas fa-exclamation-triangle text-secondary" data-toggle="tooltip" data-placement="top" title="A New Update Is Available"></i>&nbsp;
+                              {%endif%}
+                              {{git_metadata['current_commit']}}
+                            </td>
+                          </tr>
+                          <tr
+                          {%if git_metadata['new_git_version_available']%}
+                            class="table-danger"
+                          {%endif%}
+                          >
+                            <td>Current Tag</td>
+                            <td>
+                              {%if git_metadata['new_git_version_available']%}
+                                <i class="fas fa-exclamation-circle text-danger" data-toggle="tooltip" data-placement="top" title="A New Version Is Available"></i>&nbsp;&nbsp;
+                              {%endif%}
+                              {{git_metadata['last_local_tag']}}
+                            </td>
+                          </tr>
+                        {%else%}
+                        <tr>
+                          <td>This working directory is a fork</td>
+                        </tr>
+                        {%endif%}
+                      </tbody>
+                    </table>
+                  </div>
+                </div>
+              </div>
+
+              <div class="col-xl-6">
+                {%if update_metadata['update_in_progress']%}
+                <div class="card text-center border-success">
+                  <div class="card-body">
+                    <h5 class="card-title">Backgroud Update: {{update_metadata['current_background_update']}}</h5>
+                    <img id="loading-gif-modal" src="{{url_for('static', filename='image/loading.gif') }}" height="26" width="26">
+                  </div>
+                </div>
+                {%elif update_metadata['update_error']%}
+                  <div class="card text-center border-danger">
+                    <div class="card-body text-danger">
+                      <h5 class="card-title">Update Error: {{update_metadata['current_background_update']}}</h5>
+                      <p>{{update_metadata['update_error']}}</p>
+                    </div>
+                  </div>
+                {%endif%}
+              </div>
+
+            </div>
+
+					</div>
+				</div>
+
+        {%if git_metadata['new_git_version_available']%}
+          <div class="alert alert-danger" role="alert">
+            <h4 class="alert-heading">New Version Available!</h4>
+            <hr class="my-0">
+            <p>A new version is available, new version: <strong>{{git_metadata['last_remote_tag']}}</strong></p>
+            <a target="_blank" href="https://github.com/CIRCL/AIL-framework/releases/tag/{{git_metadata['last_remote_tag']}}"> Check last release note.</a>
+          </div>
+        {%endif%}
+
+        {%if git_metadata['new_git_update_available']%}
+          <div class="alert alert-warning" role="alert">
+            <h4 class="alert-heading">New Update Available!</h4>
+            <hr class="my-0">
+            <p>A new update is available, new commit ID: <strong>{{git_metadata['last_remote_commit']}}</strong></p>
+            <a target="_blank" href="https://github.com/CIRCL/AIL-framework/commit/{{git_metadata['last_remote_commit']}}"> Check last commit content.</a>
+          </div>
+        {%endif%}
+
+    	</div>
+		</div>
+	</div>
+
+</body>
+
+<script>
+$(document).ready(function(){
+	$("#page-options").addClass("active");
+} );
+
+function toggle_sidebar(){
+	if($('#nav_menu').is(':visible')){
+		$('#nav_menu').hide();
+		$('#side_menu').removeClass('border-right')
+		$('#side_menu').removeClass('col-lg-2')
+		$('#core_content').removeClass('col-lg-10')
+	}else{
+		$('#nav_menu').show();
+		$('#side_menu').addClass('border-right')
+		$('#side_menu').addClass('col-lg-2')
+		$('#core_content').addClass('col-lg-10')
+	}
+}
+</script>
+
+</html>

var/www/templates/nav_bar.html

@@ -31,7 +31,7 @@
         <a class="nav-link" href="{{ url_for('trendingmodules.moduletrending') }}" aria-disabled="true"><i class="fas fa-chart-bar"></i> Statistics</a>
       </li>
       <li class="nav-item mr-3">
-        <a class="nav-link" id="page-options" href="{{ url_for('options.options_page') }}" aria-disabled="true"><i class="fas fa-cog"></i> Options</a>
+        <a class="nav-link" id="page-options" href="{{ url_for('settings.settings_page') }}" aria-disabled="true"><i class="fas fa-cog"></i> Server Management</a>
       </li>
     </ul>
 

var/www/templates/options/menu_sidebar.html

@@ -0,0 +1,13 @@
+      <div class="col-12 col-lg-2 p-0 bg-light border-right" id="side_menu">
+
+      	<button type="button" class="btn btn-outline-secondary mt-1 ml-3" onclick="toggle_sidebar()">
+          <i class="fas fa-align-left"></i>
+      	  <span>Toggle Sidebar</span>
+      	</button>
+
+      	<nav class="navbar navbar-expand navbar-light bg-light flex-md-column flex-row align-items-start py-2" id="nav_menu">
+      		<h5 class="d-flex text-muted w-100">
+      		  <span>Diagnostic</span>
+      		</h5>
+      	</nav>
+      </div>