From 10430135d1d92e83913f773d4a8f0a3c7d81909c Mon Sep 17 00:00:00 2001 From: Terrtia Date: Mon, 2 Mar 2020 16:56:20 +0100 Subject: [PATCH] chg; [Analyzer Queue UI] add queue creator template + bug fix --- server/lib/Analyzer_Queue.py | 106 +++++++++--- server/lib/d4_type.py | 42 +++++ server/web/Flask_server.py | 72 ++------ server/web/blueprints/analyzer_queue.py | 81 +++++++++ .../block_add_sensor_to_group_block.html | 7 + .../analyzer_queue/queue_creator.html | 158 ++++++++++++++++++ server/web/templates/server_management.html | 100 ++++++----- 7 files changed, 435 insertions(+), 131 deletions(-) create mode 100755 server/lib/d4_type.py create mode 100644 server/web/blueprints/analyzer_queue.py create mode 100644 server/web/templates/analyzer_queue/block_add_sensor_to_group_block.html create mode 100644 server/web/templates/analyzer_queue/queue_creator.html diff --git a/server/lib/Analyzer_Queue.py b/server/lib/Analyzer_Queue.py index 2785382..062102c 100755 --- a/server/lib/Analyzer_Queue.py +++ b/server/lib/Analyzer_Queue.py @@ -10,6 +10,7 @@ import redis sys.path.append(os.path.join(os.environ['D4_HOME'], 'lib/')) import ConfigLoader +import d4_type ### Config ### config_loader = ConfigLoader.ConfigLoader() @@ -31,29 +32,32 @@ def is_valid_uuid_v4(uuid_v4): except: return False -def sanitize_uuid(uuid_v4): +def sanitize_uuid(uuid_v4, not_exist=False): if not is_valid_uuid_v4(uuid_v4): uuid_v4 = str(uuid.uuid4()) + if not_exist: + if exist_queue(uuid_v4): + uuid_v4 = str(uuid.uuid4()) return uuid_v4 +def sanitize_queue_type(format_type): + try: + format_type = int(format_type) + except: + format_type = 1 + if format_type == 2: + format_type = 254 + return format_type + +def exist_queue(queue_uuid): + return r_serv_metadata.exists('analyzer:{}'.format(queue_uuid)) + def get_all_queues(r_list=None): res = r_serv_metadata.smembers('all_analyzer_queues') if r_list: return list(res) return res -def get_all_queues_standard_format(r_list=None): - res = r_serv_metadata.smembers('all_analyzer_queues') - if r_list: - return list(res) - return res - -def get_all_queues_standard_extended_format(r_list=None): - res = r_serv_metadata.smembers('all_analyzer_queues') - if r_list: - return list(res) - return res - def get_all_queues_by_type(format_type, r_list=None): ''' Get all analyzer Queues by type @@ -72,12 +76,24 @@ def get_all_queues_by_type(format_type, r_list=None): return list(res) return res +def get_all_queues_by_extended_type(extended_type, r_list=None): + res = r_serv_metadata.smembers('analyzer:254:{}'.format(extended_type)) + if r_list: + return list(res) + return res + def get_all_queues_group_by_type(format_type, r_list=None): res = r_serv_metadata.smembers('analyzer_uuid_group:{}'.format(format_type)) if r_list: return list(res) return res +def get_all_queues_group_by_extended_type(extended_type, r_list=None): + res = r_serv_metadata.smembers('analyzer_uuid_group:254:{}'.format(extended_type)) + if r_list: + return list(res) + return res + def get_all_queues_by_sensor_group(format_type, sensor_uuid, r_list=None): print('sensor:queues:{}:{}'.format(format_type, sensor_uuid)) res = r_serv_metadata.smembers('sensor:queues:{}:{}'.format(format_type, sensor_uuid)) @@ -106,7 +122,24 @@ def get_queue_max_size(queue_uuid): max_size = LIST_DEFAULT_SIZE return max_size -def get_queue_metadata(queue_uuid, format_type=None, f_date='str_date'): +def get_queue_size(queue_uuid, format_type, extended_type=None): + if format_type==254: + if not extended_type: + extended_type = get_queue_extended_type(queue_uuid) + length = r_serv_analyzer.llen('analyzer:{}:{}'.format(extended_type, queue_uuid)) + else: + length = r_serv_analyzer.llen('analyzer:{}:{}'.format(format_type, queue_uuid)) + if length is None: + length = 0 + return length + +def get_queue_format_type(queue_uuid): + return r_serv_metadata.hget('analyzer:{}'.format(queue_uuid), 'type') + +def get_queue_extended_type(queue_uuid): + return r_serv_metadata.hget('analyzer:{}'.format(queue_uuid), 'metatype') + +def get_queue_metadata(queue_uuid, format_type=None, extended_type=None, f_date='str_date', force_is_group_queue=False): dict_queue_meta = {} dict_queue_meta['uuid'] = queue_uuid dict_queue_meta['size_limit'] = get_queue_max_size(queue_uuid) @@ -116,10 +149,21 @@ def get_queue_metadata(queue_uuid, format_type=None, f_date='str_date'): if dict_queue_meta['description'] is None: dict_queue_meta['description'] = '' - if format_type: - dict_queue_meta['length'] = r_serv_analyzer.llen('analyzer:{}:{}'.format(format_type, queue_uuid)) - if dict_queue_meta['length'] is None: - dict_queue_meta['length'] = 0 + if not format_type: + format_type = get_queue_format_type(queue_uuid) + dict_queue_meta['format_type'] = format_type + if format_type==254: + if not extended_type: + extended_type = get_queue_extended_type(queue_uuid) + dict_queue_meta['extended_type'] = extended_type + + dict_queue_meta['length'] = get_queue_size(queue_uuid, format_type, extended_type=extended_type) + + if force_is_group_queue: + dict_queue_meta['is_group_queue'] = True + else: + dict_queue_meta['is_group_queue'] = False + return dict_queue_meta def edit_queue_description(queue_uuid, description): @@ -138,7 +182,15 @@ def edit_queue_max_size(queue_uuid, max_size): # create queu by type or by group of uuid # # TODO: add size limit def create_queues(format_type, queue_uuid=None, l_uuid=[], queue_type='list', metatype_name=None, description=None): - queue_uuid = sanitize_uuid(queue_uuid) + if not d4_type.is_accepted_format_type(format_type): + return {'error': 'Invalid type'} + + ormat_type = sanitize_queue_type(format_type) + + if format_type == 254 and not d4_type.is_accepted_extended_type(metatype_name): + return {'error': 'Invalid extended type'} + + queue_uuid = sanitize_uuid(queue_uuid, not_exist=True) r_serv_metadata.hset('analyzer:{}'.format(queue_uuid), 'type', format_type) edit_queue_description(queue_uuid, description) @@ -151,6 +203,7 @@ def create_queues(format_type, queue_uuid=None, l_uuid=[], queue_type='list', me if format_type == 254: # TODO: check metatype_name r_serv_metadata.sadd('{}:{}:{}'.format(analyzer_key_name, format_type, metatype_name), queue_uuid) + r_serv_metadata.hset('analyzer:{}'.format(queue_uuid), 'metatype', metatype_name) else: r_serv_metadata.sadd('{}:{}'.format(analyzer_key_name, format_type), queue_uuid) @@ -193,9 +246,23 @@ def flush_queue(queue_uuid, format_type): r_serv_analyzer.delete('analyzer:{}:{}'.format(format_type, queue_uuid)) def remove_queues(queue_uuid, format_type, metatype_name=None): + try: + format_type = int(format_type) + except: + print('error: Invalid format type') + return {'error': 'Invalid format type'} + if not is_valid_uuid_v4(queue_uuid): + print('error: Invalid uuid') return {'error': 'Invalid uuid'} + if not exist_queue(queue_uuid): + print('error: unknow queue uuid') + return {'error': 'unknow queue uuid'} + + if format_type==254 and not metatype_name: + metatype_name = get_queue_extended_type(queue_uuid) + # delete metadata r_serv_metadata.delete('analyzer:{}'.format(queue_uuid)) @@ -212,7 +279,6 @@ def remove_queues(queue_uuid, format_type, metatype_name=None): analyzer_key_name = 'analyzer' if format_type == 254: - # TODO: check metatype_name r_serv_metadata.srem('{}:{}:{}'.format(analyzer_key_name, format_type, metatype_name), queue_uuid) else: r_serv_metadata.srem('{}:{}'.format(analyzer_key_name, format_type), queue_uuid) diff --git a/server/lib/d4_type.py b/server/lib/d4_type.py new file mode 100755 index 0000000..8a1f7bd --- /dev/null +++ b/server/lib/d4_type.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import sys +import datetime +import time +import uuid +import redis + +sys.path.append(os.path.join(os.environ['D4_HOME'], 'lib/')) +import ConfigLoader + +### Config ### +config_loader = ConfigLoader.ConfigLoader() +r_serv_metadata = config_loader.get_redis_conn("Redis_METADATA") +config_loader = None +### ### + +def get_all_accepted_format_type(r_list=False): + res = r_serv_metadata.smembers('server:accepted_type') + if r_list: + if res: + return list(res) + else: + return [] + return res + +def get_all_accepted_extended_type(r_list=False): + res = r_serv_metadata.smembers('server:accepted_extended_type') + if r_list: + if res: + return list(res) + else: + return [] + return res + +def is_accepted_format_type(format_type): + return r_serv_metadata.sismember('server:accepted_type', format_type) + +def is_accepted_extended_type(extended_type): + return r_serv_metadata.sismember('server:accepted_extended_type', extended_type) diff --git a/server/web/Flask_server.py b/server/web/Flask_server.py index 0273ae0..7752b10 100755 --- a/server/web/Flask_server.py +++ b/server/web/Flask_server.py @@ -34,6 +34,7 @@ import Analyzer_Queue # Import Blueprint from blueprints.restApi import restApi from blueprints.settings import settings +from blueprints.analyzer_queue import analyzer_queue baseUrl = '' if baseUrl != '': @@ -107,6 +108,7 @@ login_manager.init_app(app) # ========= BLUEPRINT =========# app.register_blueprint(restApi) app.register_blueprint(settings) +app.register_blueprint(analyzer_queue) # ========= =========# # ========= LOGIN MANAGER ======== @@ -516,53 +518,29 @@ def server_management(): description = 'Please update your web server' list_analyzer_uuid = [] - for analyzer_uuid in redis_server_metadata.smembers('analyzer:{}'.format(type)): - size_limit = redis_server_metadata.hget('analyzer:{}'.format(analyzer_uuid), 'max_size') - if size_limit is None: - size_limit = analyzer_list_max_default_size - last_updated = redis_server_metadata.hget('analyzer:{}'.format(analyzer_uuid), 'last_updated') - if last_updated is None: - last_updated = 'Never' - else: - last_updated = datetime.datetime.fromtimestamp(float(last_updated)).strftime('%Y-%m-%d %H:%M:%S') - description_analyzer = redis_server_metadata.hget('analyzer:{}'.format(analyzer_uuid), 'description') - if description_analyzer is None: - description_analyzer = '' - len_queue = redis_server_analyzer.llen('analyzer:{}:{}'.format(type, analyzer_uuid)) - if len_queue is None: - len_queue = 0 - list_analyzer_uuid.append({'uuid': analyzer_uuid, 'description': description_analyzer, 'size_limit': size_limit,'last_updated': last_updated, 'length': len_queue}) + for analyzer_uuid in Analyzer_Queue.get_all_queues_by_type(type): + list_analyzer_uuid.append(Analyzer_Queue.get_queue_metadata(analyzer_uuid, format_type=type)) for analyzer_uuid in Analyzer_Queue.get_all_queues_group_by_type(type): - list_analyzer_uuid.append(Analyzer_Queue.get_queue_metadata(analyzer_uuid, type)) + list_analyzer_uuid.append(Analyzer_Queue.get_queue_metadata(analyzer_uuid, format_type=type, force_is_group_queue=True)) list_accepted_types.append({"id": int(type), "description": description, 'list_analyzer_uuid': list_analyzer_uuid}) list_accepted_extended_types = [] + l_queue_extended_type = [] for extended_type in redis_server_metadata.smembers('server:accepted_extended_type'): + list_accepted_extended_types.append({"name": extended_type, 'list_analyzer_uuid': []}) - list_analyzer_uuid = [] - for analyzer_uuid in redis_server_metadata.smembers('analyzer:254:{}'.format(extended_type)): - size_limit = redis_server_metadata.hget('analyzer:{}'.format(analyzer_uuid), 'max_size') - if size_limit is None: - size_limit = analyzer_list_max_default_size - last_updated = redis_server_metadata.hget('analyzer:{}'.format(analyzer_uuid), 'last_updated') - if last_updated is None: - last_updated = 'Never' - else: - last_updated = datetime.datetime.fromtimestamp(float(last_updated)).strftime('%Y-%m-%d %H:%M:%S') - description_analyzer = redis_server_metadata.hget('analyzer:{}'.format(analyzer_uuid), 'description') - if description_analyzer is None: - description_analyzer = '' - len_queue = redis_server_analyzer.llen('analyzer:{}:{}'.format(extended_type, analyzer_uuid)) - if len_queue is None: - len_queue = 0 - list_analyzer_uuid.append({'uuid': analyzer_uuid, 'description': description_analyzer, 'size_limit': size_limit,'last_updated': last_updated, 'length': len_queue}) + for extended_queue_uuid in Analyzer_Queue.get_all_queues_by_extended_type(extended_type): + l_queue_extended_type.append(Analyzer_Queue.get_queue_metadata(extended_queue_uuid, format_type=254, extended_type=extended_type)) + + for extended_queue_uuid in Analyzer_Queue.get_all_queues_group_by_extended_type(extended_type): + l_queue_extended_type.append(Analyzer_Queue.get_queue_metadata(extended_queue_uuid, format_type=254, extended_type=extended_type, force_is_group_queue=True)) - list_accepted_extended_types.append({"name": extended_type, 'list_analyzer_uuid': list_analyzer_uuid}) return render_template("server_management.html", list_accepted_types=list_accepted_types, list_accepted_extended_types=list_accepted_extended_types, server_mode=server_mode, + l_queue_extended_type=l_queue_extended_type, nb_sensors_registered=nb_sensors_registered, nb_sensors_pending=nb_sensors_pending, default_analyzer_max_line_len=default_analyzer_max_line_len, blacklisted_ip=blacklisted_ip, unblacklisted_ip=unblacklisted_ip, @@ -788,30 +766,6 @@ def uuid_change_description(): else: return jsonify({'error':'invalid uuid'}), 400 -# # TODO: check analyser uuid dont exist -@app.route('/add_new_analyzer') -@login_required -@login_user_basic -def add_new_analyzer(): - format_type = request.args.get('type') - user = request.args.get('redirect') - metatype_name = request.args.get('metatype_name') - analyzer_description = request.args.get('analyzer_description') - analyzer_uuid = request.args.get('analyzer_uuid') - if is_valid_uuid_v4(analyzer_uuid): - try: - format_type = int(format_type) - if format_type < 0: - return 'type, Invalid Integer' - except: - return 'type, Invalid Integer' - - Analyzer_Queue.create_queues(format_type, queue_uuid=analyzer_uuid, l_uuid=[], queue_type='list', metatype_name=metatype_name, description=analyzer_description) - if user: - return redirect(url_for('server_management')) - else: - return 'Invalid uuid' - @app.route('/empty_analyzer_queue') @login_required @login_user_basic diff --git a/server/web/blueprints/analyzer_queue.py b/server/web/blueprints/analyzer_queue.py new file mode 100644 index 0000000..6711201 --- /dev/null +++ b/server/web/blueprints/analyzer_queue.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +''' + Flask functions and routes for the rest api +''' + +import os +import re +import sys +import redis + +sys.path.append(os.path.join(os.environ['D4_HOME'], 'lib')) +import ConfigLoader +import Analyzer_Queue + +from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response +from flask_login import login_required, current_user + +from Role_Manager import login_admin, login_user_basic +from Role_Manager import create_user_db, edit_user_db, delete_user_db, check_password_strength, generate_new_token, gen_password, get_all_role + +# ============ BLUEPRINT ============ + +analyzer_queue = Blueprint('analyzer_queue', __name__, template_folder='templates') + +# ============ VARIABLES ============ + +### Config ### +config_loader = ConfigLoader.ConfigLoader() +r_serv_metadata = config_loader.get_redis_conn("Redis_METADATA") +r_serv_db = config_loader.get_redis_conn("Redis_SERV") +config_loader = None +### ### + +# ============ FUNCTIONS ============ + + +# ============= ROUTES ============== + +@analyzer_queue.route("/analyzer_queue/create_queue", methods=['GET']) +@login_required +@login_user_basic +def create_analyzer_queue(): + return render_template("analyzer_queue/queue_creator.html") + +@analyzer_queue.route("/analyzer_queue/create_queue_post", methods=['POST']) +@login_required +@login_user_basic +def create_analyzer_queue_post(): + l_queue_meta = ['analyzer_type', 'analyzer_metatype', 'description', 'analyzer_uuid'] + queue_type = request.form.get("analyzer_type") + queue_metatype = request.form.get("analyzer_metatype") + queue_description = request.form.get("description") + queue_uuid = request.form.get("analyzer_uuid") + + queue_type = Analyzer_Queue.sanitize_queue_type(queue_type) + + # unpack uuid group + l_uuid = set() + l_invalid_uuid = set() + for obj_tuple in list(request.form): + if obj_tuple not in l_queue_meta: + sensor_uuid = request.form.get(obj_tuple) + if Analyzer_Queue.is_valid_uuid_v4(sensor_uuid): + l_uuid.add(sensor_uuid) + else: + if sensor_uuid: + l_invalid_uuid.add(sensor_uuid) + + l_uuid = list(l_uuid) + l_invalid_uuid = list(l_invalid_uuid) + if l_invalid_uuid: + return render_template("analyzer_queue/queue_creator.html", queue_uuid=queue_uuid, queue_type=queue_type, metatype_name=queue_metatype, + description=queue_description, l_uuid=l_uuid, l_invalid_uuid=l_invalid_uuid) + + res = Analyzer_Queue.create_queues(queue_type, queue_uuid=queue_uuid, l_uuid=l_uuid, metatype_name=queue_metatype, description=queue_description) + if isinstance(res,dict): + return jsonify(res) + if res: + return redirect(url_for('server_management', _anchor=res)) diff --git a/server/web/templates/analyzer_queue/block_add_sensor_to_group_block.html b/server/web/templates/analyzer_queue/block_add_sensor_to_group_block.html new file mode 100644 index 0000000..1274da2 --- /dev/null +++ b/server/web/templates/analyzer_queue/block_add_sensor_to_group_block.html @@ -0,0 +1,7 @@ +
+ + +
+ Please provide a valid UUID v4. +
+
diff --git a/server/web/templates/analyzer_queue/queue_creator.html b/server/web/templates/analyzer_queue/queue_creator.html new file mode 100644 index 0000000..76e85e4 --- /dev/null +++ b/server/web/templates/analyzer_queue/queue_creator.html @@ -0,0 +1,158 @@ + + + + + D4-Project + + + + + + + + + + + + + + + + + + + + + {% include 'navbar.html' %} + + + +
+
+
Create Analyzer Queue
+
+
+ +
+ +
+ + + +
+ +
+
+ +
+ +
+ +
+ +
+ +
+ +

Create Queue by Group of UUID

+ +
Sensor UUID
+ +
+
+
+
+ + {% if l_uuid %} + {% for sensor_uuid in l_uuid %} + {% with sensor_uuid=sensor_uuid, error=False%} + {% include 'analyzer_queue/block_add_sensor_to_group_block.html' %} + {% endwith %} + {% endfor %} +
+ {% endif %} + {% if l_invalid_uuid %} + {% for sensor_uuid in l_invalid_uuid %} + {% with sensor_uuid=sensor_uuid, error=True%} + {% include 'analyzer_queue/block_add_sensor_to_group_block.html' %} + {% endwith %} + {% endfor %} +
+ {% endif %} +
+ + +
+ +
+
+
+
+ +
+ +
+ +
+ +
+ +
+
+ + + {% include 'navfooter.html' %} + + + diff --git a/server/web/templates/server_management.html b/server/web/templates/server_management.html index cbb062a..19a1141 100644 --- a/server/web/templates/server_management.html +++ b/server/web/templates/server_management.html @@ -231,11 +231,12 @@
-
+
+ @@ -246,8 +247,13 @@ {% for type in list_accepted_types %} {% if type['list_analyzer_uuid'] %} {% for analyzer in type['list_analyzer_uuid'] %} - + + {%if analyzer['is_group_queue']%} + + {%else%} + + {%endif%} + @@ -291,60 +298,49 @@ - {% for type in list_accepted_extended_types %} - {% if type['list_analyzer_uuid'] %} - {% for analyzer in type['list_analyzer_uuid'] %} - - - - - - - - {% endfor %} - {% endif %} + {% for dict_queue in l_queue_extended_type %} + + + {%if dict_queue['is_group_queue']%} + + {%else%} + + {%endif%} + + + + + {% endfor %}
TypeGroup uuid last updated Change max size limit
{{type['id']}}
{{analyzer['uuid']}} @@ -284,6 +290,7 @@
Type NameGroup uuid last updated Change max size limit
{{type['name']}} -
- {{analyzer['uuid']}} - - - -
- {%if analyzer['description']%} -
{{analyzer['description']}}
- {%endif%} -
{{analyzer['last_updated']}} -
- - -
-
- - - - -
{{dict_queue['extended_type']}} +
+ {{dict_queue['uuid']}} + + + +
+ {%if dict_queue['description']%} +
{{dict_queue['description']}}
+ {%endif%} +
{{dict_queue['last_updated']}} +
+ + +
+
+ + + + +
-
-
-
-
Add New Analyzer Queue
- - -
-
- -
- -
- - -
-
+
@@ -430,7 +426,7 @@ if (tbody.children().length == 0) { } $('#accepted_type').on('input', function() { - if ($('#accepted_type').val() == 254){ + if ($('#analyzer_type').val() == 2 || $('#accepted_type').val() == 254){ $('#extended_type_name').show() } else { $('#extended_type_name').hide() @@ -438,7 +434,7 @@ $('#accepted_type').on('input', function() { }); $('#analyzer_type').on('input', function() { - if ($('#analyzer_type').val() == 254){ + if ($('#analyzer_type').val() == 2 || $('#analyzer_type').val() == 254){ $('#analyzer_metatype_name').show() } else { $('#analyzer_metatype_name').hide()