From 6fee7df9fe6b78e668f3820119c37910e729bac1 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 10 Nov 2020 11:11:23 +0100 Subject: [PATCH] chg: [Sensors API + UI] add sensors monitoring --- server/LAUNCH.sh | 2 + server/lib/Sensor.py | 105 ++++++++++++++++++ server/lib/d4_server.py | 44 ++++++++ server/sensors_manager.py | 53 +++++++++ server/server.py | 9 +- server/web/Flask_server.py | 15 ++- server/web/blueprints/D4_sensors.py | 77 +++++++++++++ server/web/blueprints/analyzer_queue.py | 1 - server/web/blueprints/restApi.py | 10 +- server/web/templates/login.html | 3 +- .../sensors/add_sensor_to_monitor.html | 57 ++++++++++ server/web/templates/uuid_management.html | 12 ++ 12 files changed, 376 insertions(+), 12 deletions(-) create mode 100755 server/lib/d4_server.py create mode 100755 server/sensors_manager.py create mode 100644 server/web/blueprints/D4_sensors.py create mode 100644 server/web/templates/sensors/add_sensor_to_monitor.html diff --git a/server/LAUNCH.sh b/server/LAUNCH.sh index 908d8ca..6b422da 100755 --- a/server/LAUNCH.sh +++ b/server/LAUNCH.sh @@ -69,6 +69,8 @@ function launching_d4_server { screen -S "Server_D4" -X screen -t "Server_D4" bash -c "cd ${D4_HOME}; ./server.py -v 10; read x" sleep 0.1 + screen -S "Server_D4" -X screen -t "sensors_manager" bash -c "cd ${D4_HOME}; ./sensors_manager.py; read x" + sleep 0.1 } function launching_workers { diff --git a/server/lib/Sensor.py b/server/lib/Sensor.py index 7cc22e2..c72e13d 100755 --- a/server/lib/Sensor.py +++ b/server/lib/Sensor.py @@ -11,6 +11,7 @@ from flask import escape sys.path.append(os.path.join(os.environ['D4_HOME'], 'lib/')) import ConfigLoader +import d4_server ### Config ### config_loader = ConfigLoader.ConfigLoader() @@ -26,6 +27,13 @@ def is_valid_uuid_v4(UUID): except: return False +def get_time_sensor_last_seen(sensor_uuid): + res = r_serv_db.hget('metadata_uuid:{}'.format(sensor_uuid), 'last_seen') + if res: + return int(res) + else: + return 0 + def _get_sensor_type(sensor_uuid, first_seen=True, last_seen=True, time_format='default'): uuid_type = [] uuid_all_type = r_serv_db.smembers('all_types_by_uuid:{}'.format(sensor_uuid)) @@ -68,6 +76,8 @@ def _get_sensor_metadata(sensor_uuid, first_seen=True, last_seen=True, time_form meta_sensor['mail'] = r_serv_db.hget('metadata_uuid:{}'.format(sensor_uuid), 'user_mail') return meta_sensor +### BEGIN - SENSOR REGISTRATION ### + ## TODO: add description def register_sensor(req_dict): sensor_uuid = req_dict.get('uuid', None) @@ -167,3 +177,98 @@ def delete_registered_sensor(req_dict): def _delete_registered_sensor(sensor_uuid): r_serv_db.srem('registered_uuid', sensor_uuid) return ({'uuid': sensor_uuid}, 200) + +### --- END - SENSOR REGISTRATION --- ### + + +### BEGIN - SENSOR MONITORING ### +def get_sensors_monitoring_last_updated(): + res = r_serv_db.get('sensors_monitoring:last_updated') + if res: + return int(res) + else: + return 0 + +def get_all_sensors_to_monitor(): + return r_serv_db.smembers('to_monitor:sensors') + +def get_to_monitor_delta_time_by_uuid(sensor_uuid): + return int(r_serv_db.hget('to_monitor:sensor:{}'.format(sensor_uuid), 'delta_time')) + +def get_all_sensors_to_monitor_dict(): + dict_to_monitor = {} + for sensor_uuid in get_all_sensors_to_monitor(): + dict_to_monitor[sensor_uuid] = get_to_monitor_delta_time_by_uuid(sensor_uuid) + return dict_to_monitor + +def _check_sensor_delta(sensor_uuid, sensor_delta): + last_d4_packet = get_time_sensor_last_seen(sensor_uuid) + + # check sensor delta time between two D4 packets + check sensor connection + if int(time.time()) - last_d4_packet > sensor_delta or not d4_server.is_sensor_connected(sensor_uuid): + r_serv_db.sadd('sensors_monitoring:sensors_error', sensor_uuid) + handle_sensor_monitoring_error(sensor_uuid) + else: + r_serv_db.srem('sensors_monitoring:sensors_error', sensor_uuid) + +def handle_sensor_monitoring_error(sensor_uuid): + print('sensor monitoring error: {}'.format(sensor_uuid)) + ## TODO: ## + # MAILS + # UI Notifications + # SNMP + # Syslog message + ## ## ## ## + return None + +def is_sensor_monitored(sensor_uuid): + return r_serv_db.exists('to_monitor:sensors', sensor_uuid) + +def get_all_sensors_connection_errors(): + return r_serv_db.smembers('sensors_monitoring:sensors_error') + +def api_get_all_sensors_connection_errors(): + return list(get_all_sensors_connection_errors()), 200 + +def add_sensor_to_monitor(sensor_uuid, delta_time): + r_serv_db.sadd('to_monitor:sensors', sensor_uuid) + r_serv_db.hset('to_monitor:sensor:{}'.format(sensor_uuid), 'delta_time', delta_time) + r_serv_db.set('sensors_monitoring:last_updated', int(time.time())) + +def delete_sensor_to_monitor(sensor_uuid): + r_serv_db.srem('to_monitor:sensors', sensor_uuid) + r_serv_db.delete('to_monitor:sensor:{}'.format(sensor_uuid)) + r_serv_db.set('sensors_monitoring:last_updated', int(time.time())) + r_serv_db.srem('sensors_monitoring:sensors_error', sensor_uuid) + +def api_add_sensor_to_monitor(data_dict): + sensor_uuid = data_dict.get('uuid', None) + delta_time = data_dict.get('delta_time', None) + + if not is_valid_uuid_v4(sensor_uuid): + return ({"status": "error", "reason": "Invalid uuid"}, 400) + sensor_uuid = sensor_uuid.replace('-', '') + + # hmac key + if not delta_time: + return ({"status": "error", "reason": "Mandatory parameter(s) not provided"}, 400) + else: + try: + delta_time = int(delta_time) + if delta_time < 1: + return ({"status": "error", "reason": "Invalid delta_time"}, 400) + except Exception: + return ({"status": "error", "reason": "Invalid delta_time"}, 400) + add_sensor_to_monitor(sensor_uuid, delta_time) + +def api_delete_sensor_to_monitor(data_dict): + sensor_uuid = data_dict.get('uuid', None) + if not is_valid_uuid_v4(sensor_uuid): + return ({"status": "error", "reason": "Invalid uuid"}, 400) + sensor_uuid = sensor_uuid.replace('-', '') + if not is_sensor_monitored(sensor_uuid): + return ({"status": "error", "reason": "Sensor not monitored"}, 400) + delete_sensor_to_monitor(sensor_uuid) + + +### --- END - SENSOR REGISTRATION --- ### diff --git a/server/lib/d4_server.py b/server/lib/d4_server.py new file mode 100755 index 0000000..bb4349d --- /dev/null +++ b/server/lib/d4_server.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import sys +import time +import uuid +import redis + +from flask import escape + +sys.path.append(os.path.join(os.environ['D4_HOME'], 'lib/')) +import ConfigLoader + +### Config ### +config_loader = ConfigLoader.ConfigLoader() +r_stream = config_loader.get_redis_conn("Redis_STREAM") +config_loader = None +### ### + +### BEGIN - SENSOR CONNECTION ### + +def get_all_connected_sensors(r_list=False): + res = r_stream.smembers('active_connection') + if r_list: + if res: + return list(res) + else: + return [] + else: + return res + +def get_all_connected_sensors_by_type(d4_type, d4_extended_type=None): + # D4 extended type + if d4_type == 254 and d4_extended_type: + return r_stream.smembers('active_connection_extended_type:{}'.format(d4_extended_type)) + # type 1-253 + else: + return r_stream.smembers('active_connection:{}'.format(d4_type)) + +def is_sensor_connected(sensor_uuid): + return r_stream.sismember('active_connection', sensor_uuid) + +### --- END - SENSOR CONNECTION --- ### diff --git a/server/sensors_manager.py b/server/sensors_manager.py new file mode 100755 index 0000000..5f51a83 --- /dev/null +++ b/server/sensors_manager.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 + +import os +import sys +import time +import redis + +sys.path.append(os.path.join(os.environ['D4_HOME'], 'lib/')) +import ConfigLoader +import Sensor + +### Config ### +config_loader = ConfigLoader.ConfigLoader() +#redis_server_stream = config_loader.get_redis_conn("Redis_STREAM", decode_responses=False) +redis_server_metadata = config_loader.get_redis_conn("Redis_METADATA") +config_loader = None +### ### + +try: + redis_server_metadata.ping() +except redis.exceptions.ConnectionError: + print('Error: Redis server: Redis_METADATA, ConnectionError') + sys.exit(1) + + +def reload_all_sensors_to_monitor_dict(dict_to_monitor, last_updated): + if not dict_to_monitor: + dict_to_monitor = Sensor.get_all_sensors_to_monitor_dict() + else: + monitoring_last_updated = Sensor.get_sensors_monitoring_last_updated() + if monitoring_last_updated > last_updated: + dict_to_monitor = Sensor.get_all_sensors_to_monitor_dict() + last_updated = int(time.time()) + print('updated: List of sensors to monitor') + +if __name__ == "__main__": + + time_refresh = int(time.time()) + last_updated = time_refresh + all_sensors_to_monitor = Sensor.get_all_sensors_to_monitor_dict() + + while True: + + for sensor_uuid in all_sensors_to_monitor: + Sensor._check_sensor_delta(sensor_uuid, all_sensors_to_monitor[sensor_uuid]) + time.sleep(10) + + ## reload dict_to_monitor ## + curr_time = int(time.time()) + if curr_time - time_refresh >= 60: + time_refresh = curr_time + reload_all_sensors_to_monitor_dict(all_sensors_to_monitor, last_updated) + ##-- --## diff --git a/server/server.py b/server/server.py index 2d66d57..5460a5a 100755 --- a/server/server.py +++ b/server/server.py @@ -509,10 +509,11 @@ class D4_Server(Protocol, TimeoutMixin): redis_server_metadata.zincrby('stat_uuid_type:{}:{}'.format(date, data_header['uuid_header']), 1, data_header['type']) # + d4_packet_rcv_time = int(time.time()) if not redis_server_metadata.hexists('metadata_uuid:{}'.format(data_header['uuid_header']), 'first_seen'): - redis_server_metadata.hset('metadata_uuid:{}'.format(data_header['uuid_header']), 'first_seen', data_header['timestamp']) - redis_server_metadata.hset('metadata_uuid:{}'.format(data_header['uuid_header']), 'last_seen', data_header['timestamp']) - redis_server_metadata.hset('metadata_type_by_uuid:{}:{}'.format(data_header['uuid_header'], data_header['type']), 'last_seen', data_header['timestamp']) + redis_server_metadata.hset('metadata_uuid:{}'.format(data_header['uuid_header']), 'first_seen', d4_packet_rcv_time) + redis_server_metadata.hset('metadata_uuid:{}'.format(data_header['uuid_header']), 'last_seen', d4_packet_rcv_time) + redis_server_metadata.hset('metadata_type_by_uuid:{}:{}'.format(data_header['uuid_header'], data_header['type']), 'last_seen', d4_packet_rcv_time) if not self.data_saved: #UUID IP: ## TODO: use d4 timestamp ? @@ -523,7 +524,7 @@ class D4_Server(Protocol, TimeoutMixin): if self.update_stream_type: if not redis_server_metadata.hexists('metadata_type_by_uuid:{}:{}'.format(data_header['uuid_header'], data_header['type']), 'first_seen'): - redis_server_metadata.hset('metadata_type_by_uuid:{}:{}'.format(data_header['uuid_header'], data_header['type']), 'first_seen', data_header['timestamp']) + redis_server_metadata.hset('metadata_type_by_uuid:{}:{}'.format(data_header['uuid_header'], data_header['type']), 'first_seen', d4_packet_rcv_time) self.update_stream_type = False return 0 else: diff --git a/server/web/Flask_server.py b/server/web/Flask_server.py index d24d2ee..ab1ed65 100755 --- a/server/web/Flask_server.py +++ b/server/web/Flask_server.py @@ -35,6 +35,7 @@ import Analyzer_Queue from blueprints.restApi import restApi from blueprints.settings import settings from blueprints.analyzer_queue import analyzer_queue +from blueprints.D4_sensors import D4_sensors baseUrl = '' if baseUrl != '': @@ -114,6 +115,7 @@ login_manager.init_app(app) app.register_blueprint(restApi) app.register_blueprint(settings) app.register_blueprint(analyzer_queue) +app.register_blueprint(D4_sensors) # ========= =========# # ========= LOGIN MANAGER ======== @@ -281,7 +283,7 @@ def login(): if request.method == 'POST': username = request.form.get('username') password = request.form.get('password') - #next_page = request.form.get('next_page') + next_page = request.form.get('next_page') if username is not None: user = User.get(username) @@ -304,7 +306,10 @@ def login(): if user.request_password_change(): return redirect(url_for('change_password')) else: - return redirect(url_for('index')) + if next_page and next_page!='None': + return redirect(next_page) + else: + return redirect(url_for('index')) # login failed else: # set brute force protection @@ -320,9 +325,9 @@ def login(): return 'please provide a valid username' else: - #next_page = request.args.get('next') + next_page = request.args.get('next') error = request.args.get('error') - return render_template("login.html" , error=error) + return render_template("login.html" , error=error, next_page=next_page) @app.route('/change_password', methods=['POST', 'GET']) @login_required @@ -595,6 +600,8 @@ def uuid_management(): "temp_blacklist_uuid": temp_blacklist_uuid, "blacklisted_uuid": blacklisted_uuid, "blacklisted_ip_by_uuid": blacklisted_ip_by_uuid, "first_seen_gmt": first_seen_gmt, "last_seen_gmt": last_seen_gmt, "Error": Error} + + data_uuid['is_monitored'] = Sensor.is_sensor_monitored(uuid_sensor) if redis_server_stream.sismember('active_connection', uuid_sensor): active_connection = True diff --git a/server/web/blueprints/D4_sensors.py b/server/web/blueprints/D4_sensors.py new file mode 100644 index 0000000..e418600 --- /dev/null +++ b/server/web/blueprints/D4_sensors.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +''' + Flask functions and routes for all D4 sensors +''' + +import os +import re +import sys +import redis + +sys.path.append(os.path.join(os.environ['D4_HOME'], 'lib')) +import ConfigLoader +import Sensor + +from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response +from flask_login import login_required, current_user + +from Role_Manager import login_admin, login_user_basic + +# ============ BLUEPRINT ============ + +D4_sensors = Blueprint('D4_sensors', __name__, template_folder='templates') + +# ============ VARIABLES ============ + +### Config ### +config_loader = ConfigLoader.ConfigLoader() +r_serv_metadata = config_loader.get_redis_conn("Redis_METADATA") +r_serv_db = config_loader.get_redis_conn("Redis_SERV") +config_loader = None +### ### + +# ============ FUNCTIONS ============ + + +# ============= ROUTES ============== + +@D4_sensors.route("/sensors/monitoring/add", methods=['GET']) +@login_required +@login_user_basic +def add_sensor_to_monitor(): + sensor_uuid = request.args.get("uuid") + return render_template("sensors/add_sensor_to_monitor.html", + sensor_uuid=sensor_uuid) + +@D4_sensors.route("/sensors/monitoring/add_post", methods=['POST']) +@login_required +@login_user_basic +def add_sensor_to_monitor_post(): + sensor_uuid = request.form.get("uuid") + delta_time = request.form.get("delta_time") + res = Sensor.api_add_sensor_to_monitor({'uuid':sensor_uuid, 'delta_time': delta_time}) + if res: + Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + return redirect(url_for('uuid_management', uuid=sensor_uuid)) + +@D4_sensors.route("/sensors/monitoring/delete", methods=['GET']) +@login_required +@login_user_basic +def delete_sensor_to_monitor(): + sensor_uuid = request.args.get("uuid") + res = Sensor.api_delete_sensor_to_monitor({'uuid':sensor_uuid}) + if res: + Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + return redirect(url_for('uuid_management', uuid=sensor_uuid)) + + +# +# +# +# +# +# +# +# diff --git a/server/web/blueprints/analyzer_queue.py b/server/web/blueprints/analyzer_queue.py index 1ac0ee1..8d43423 100644 --- a/server/web/blueprints/analyzer_queue.py +++ b/server/web/blueprints/analyzer_queue.py @@ -18,7 +18,6 @@ from flask import Flask, render_template, jsonify, request, Blueprint, redirect, from flask_login import login_required, current_user from Role_Manager import login_admin, login_user_basic -from Role_Manager import create_user_db, edit_user_db, delete_user_db, check_password_strength, generate_new_token, gen_password, get_all_role # ============ BLUEPRINT ============ diff --git a/server/web/blueprints/restApi.py b/server/web/blueprints/restApi.py index 9b67140..59a4e1e 100644 --- a/server/web/blueprints/restApi.py +++ b/server/web/blueprints/restApi.py @@ -142,8 +142,8 @@ def is_valid_uuid_v4(header_uuid): except: return False -def one(): - return 1 +def build_json_response(resp_data, resp_code): + return Response(json.dumps(resp_data, indent=2, sort_keys=True), mimetype='application/json'), resp_code # ============= ROUTES ============== @@ -154,3 +154,9 @@ def add_sensor_register(): data = request.get_json() res = Sensor.register_sensor(data) return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + +@restApi.route("/api/v1/sensors/monitoring/errors", methods=['GET']) +#@token_required('login_user_basic') +def get_all_sensors_connection_errors(): + res = Sensor.api_get_all_sensors_connection_errors() + return build_json_response(res[0], res[1]) diff --git a/server/web/templates/login.html b/server/web/templates/login.html index b54c290..2a3e4a0 100644 --- a/server/web/templates/login.html +++ b/server/web/templates/login.html @@ -69,7 +69,8 @@

Please sign in

- + + diff --git a/server/web/templates/sensors/add_sensor_to_monitor.html b/server/web/templates/sensors/add_sensor_to_monitor.html new file mode 100644 index 0000000..6899ade --- /dev/null +++ b/server/web/templates/sensors/add_sensor_to_monitor.html @@ -0,0 +1,57 @@ + + + + + D4-Project + + + + + + + + + + + + + + {% include 'navbar.html' %} + + + +
+
+

Monitor a Sensor

+
+ + +
+ +
+
+   +
+ +
+ Maxinum Time (seconds) between two D4 packets +
+
+ +
+ +
+
+
+ +
+ + {% include 'navfooter.html' %} + + + diff --git a/server/web/templates/uuid_management.html b/server/web/templates/uuid_management.html index 8ed7974..681f6f2 100644 --- a/server/web/templates/uuid_management.html +++ b/server/web/templates/uuid_management.html @@ -101,6 +101,18 @@ +
+ {% if not data_uuid.get('is_monitored', False) %} + + + + {% else %} + + + + {% endif %} +
+