chg: [Sensors API + UI] add sensors monitoring

server/LAUNCH.sh

@@ -69,6 +69,8 @@ function launching_d4_server {
 
     screen -S "Server_D4" -X screen -t "Server_D4" bash -c "cd ${D4_HOME}; ./server.py -v 10; read x"
     sleep 0.1
+    screen -S "Server_D4" -X screen -t "sensors_manager" bash -c "cd ${D4_HOME}; ./sensors_manager.py; read x"
+    sleep 0.1
 }
 
 function launching_workers {

server/lib/Sensor.py

@@ -11,6 +11,7 @@ from flask import escape
 
 sys.path.append(os.path.join(os.environ['D4_HOME'], 'lib/'))
 import ConfigLoader
+import d4_server
 
 ### Config ###
 config_loader = ConfigLoader.ConfigLoader()
@@ -26,6 +27,13 @@ def is_valid_uuid_v4(UUID):
     except:
         return False
 
+def get_time_sensor_last_seen(sensor_uuid):
+    res = r_serv_db.hget('metadata_uuid:{}'.format(sensor_uuid), 'last_seen')
+    if res:
+        return int(res)
+    else:
+        return 0
+
 def _get_sensor_type(sensor_uuid, first_seen=True, last_seen=True, time_format='default'):
     uuid_type = []
     uuid_all_type = r_serv_db.smembers('all_types_by_uuid:{}'.format(sensor_uuid))
@@ -68,6 +76,8 @@ def _get_sensor_metadata(sensor_uuid, first_seen=True, last_seen=True, time_form
         meta_sensor['mail'] = r_serv_db.hget('metadata_uuid:{}'.format(sensor_uuid), 'user_mail')
     return meta_sensor
 
+### BEGIN - SENSOR REGISTRATION ###
+
 ## TODO: add description
 def register_sensor(req_dict):
     sensor_uuid = req_dict.get('uuid', None)
@@ -167,3 +177,98 @@ def delete_registered_sensor(req_dict):
 def _delete_registered_sensor(sensor_uuid):
     r_serv_db.srem('registered_uuid', sensor_uuid)
     return ({'uuid': sensor_uuid}, 200)
+
+### --- END - SENSOR REGISTRATION --- ###
+
+
+###  BEGIN - SENSOR MONITORING  ###
+def get_sensors_monitoring_last_updated():
+    res = r_serv_db.get('sensors_monitoring:last_updated')
+    if res:
+        return int(res)
+    else:
+        return 0
+
+def get_all_sensors_to_monitor():
+    return r_serv_db.smembers('to_monitor:sensors')
+
+def get_to_monitor_delta_time_by_uuid(sensor_uuid):
+    return int(r_serv_db.hget('to_monitor:sensor:{}'.format(sensor_uuid), 'delta_time'))
+
+def get_all_sensors_to_monitor_dict():
+    dict_to_monitor = {}
+    for sensor_uuid in get_all_sensors_to_monitor():
+        dict_to_monitor[sensor_uuid] = get_to_monitor_delta_time_by_uuid(sensor_uuid)
+    return dict_to_monitor
+
+def _check_sensor_delta(sensor_uuid, sensor_delta):
+    last_d4_packet = get_time_sensor_last_seen(sensor_uuid)
+
+    # check sensor delta time between two D4 packets + check sensor connection
+    if int(time.time()) - last_d4_packet > sensor_delta or not d4_server.is_sensor_connected(sensor_uuid):
+        r_serv_db.sadd('sensors_monitoring:sensors_error', sensor_uuid)
+        handle_sensor_monitoring_error(sensor_uuid)
+    else:
+        r_serv_db.srem('sensors_monitoring:sensors_error', sensor_uuid)
+
+def handle_sensor_monitoring_error(sensor_uuid):
+    print('sensor monitoring error: {}'.format(sensor_uuid))
+    ## TODO: ##
+    # MAILS
+    # UI Notifications
+    # SNMP
+    # Syslog message
+    ## ## ## ##
+    return None
+
+def is_sensor_monitored(sensor_uuid):
+    return r_serv_db.exists('to_monitor:sensors', sensor_uuid)
+
+def get_all_sensors_connection_errors():
+    return r_serv_db.smembers('sensors_monitoring:sensors_error')
+
+def api_get_all_sensors_connection_errors():
+    return list(get_all_sensors_connection_errors()), 200
+
+def add_sensor_to_monitor(sensor_uuid, delta_time):
+    r_serv_db.sadd('to_monitor:sensors', sensor_uuid)
+    r_serv_db.hset('to_monitor:sensor:{}'.format(sensor_uuid), 'delta_time', delta_time)
+    r_serv_db.set('sensors_monitoring:last_updated', int(time.time()))
+
+def delete_sensor_to_monitor(sensor_uuid):
+    r_serv_db.srem('to_monitor:sensors', sensor_uuid)
+    r_serv_db.delete('to_monitor:sensor:{}'.format(sensor_uuid))
+    r_serv_db.set('sensors_monitoring:last_updated', int(time.time()))
+    r_serv_db.srem('sensors_monitoring:sensors_error', sensor_uuid)
+
+def api_add_sensor_to_monitor(data_dict):
+    sensor_uuid = data_dict.get('uuid', None)
+    delta_time = data_dict.get('delta_time', None)
+
+    if not is_valid_uuid_v4(sensor_uuid):
+        return ({"status": "error", "reason": "Invalid uuid"}, 400)
+    sensor_uuid = sensor_uuid.replace('-', '')
+
+    # hmac key
+    if not delta_time:
+        return ({"status": "error", "reason": "Mandatory parameter(s) not provided"}, 400)
+    else:
+        try:
+            delta_time = int(delta_time)
+            if delta_time < 1:
+                return ({"status": "error", "reason": "Invalid delta_time"}, 400)
+        except Exception:
+            return ({"status": "error", "reason": "Invalid delta_time"}, 400)
+    add_sensor_to_monitor(sensor_uuid, delta_time)
+
+def api_delete_sensor_to_monitor(data_dict):
+    sensor_uuid = data_dict.get('uuid', None)
+    if not is_valid_uuid_v4(sensor_uuid):
+        return ({"status": "error", "reason": "Invalid uuid"}, 400)
+    sensor_uuid = sensor_uuid.replace('-', '')
+    if not is_sensor_monitored(sensor_uuid):
+        return ({"status": "error", "reason": "Sensor not monitored"}, 400)
+    delete_sensor_to_monitor(sensor_uuid)
+
+
+### --- END - SENSOR REGISTRATION --- ###

server/lib/d4_server.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import os
+import sys
+import time
+import uuid
+import redis
+
+from flask import escape
+
+sys.path.append(os.path.join(os.environ['D4_HOME'], 'lib/'))
+import ConfigLoader
+
+### Config ###
+config_loader = ConfigLoader.ConfigLoader()
+r_stream = config_loader.get_redis_conn("Redis_STREAM")
+config_loader = None
+###  ###
+
+###  BEGIN - SENSOR CONNECTION ###
+
+def get_all_connected_sensors(r_list=False):
+    res = r_stream.smembers('active_connection')
+    if r_list:
+        if res:
+            return list(res)
+        else:
+            return []
+    else:
+        return res
+
+def get_all_connected_sensors_by_type(d4_type, d4_extended_type=None):
+    # D4 extended type
+    if d4_type == 254 and d4_extended_type:
+        return r_stream.smembers('active_connection_extended_type:{}'.format(d4_extended_type))
+    # type 1-253
+    else:
+        return r_stream.smembers('active_connection:{}'.format(d4_type))
+
+def is_sensor_connected(sensor_uuid):
+    return r_stream.sismember('active_connection', sensor_uuid)
+
+### --- END - SENSOR CONNECTION --- ###

server/sensors_manager.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+import time
+import redis
+
+sys.path.append(os.path.join(os.environ['D4_HOME'], 'lib/'))
+import ConfigLoader
+import Sensor
+
+### Config ###
+config_loader = ConfigLoader.ConfigLoader()
+#redis_server_stream = config_loader.get_redis_conn("Redis_STREAM", decode_responses=False)
+redis_server_metadata = config_loader.get_redis_conn("Redis_METADATA")
+config_loader = None
+###  ###
+
+try:
+    redis_server_metadata.ping()
+except redis.exceptions.ConnectionError:
+    print('Error: Redis server: Redis_METADATA, ConnectionError')
+    sys.exit(1)
+
+
+def reload_all_sensors_to_monitor_dict(dict_to_monitor, last_updated):
+    if not dict_to_monitor:
+        dict_to_monitor = Sensor.get_all_sensors_to_monitor_dict()
+    else:
+        monitoring_last_updated = Sensor.get_sensors_monitoring_last_updated()
+        if monitoring_last_updated > last_updated:
+            dict_to_monitor = Sensor.get_all_sensors_to_monitor_dict()
+            last_updated = int(time.time())
+            print('updated: List of sensors to monitor')
+
+if __name__ == "__main__":
+
+    time_refresh = int(time.time())
+    last_updated = time_refresh
+    all_sensors_to_monitor = Sensor.get_all_sensors_to_monitor_dict()
+
+    while True:
+
+        for sensor_uuid in all_sensors_to_monitor:
+            Sensor._check_sensor_delta(sensor_uuid, all_sensors_to_monitor[sensor_uuid])
+        time.sleep(10)
+
+        ## reload dict_to_monitor ##
+        curr_time = int(time.time())
+        if curr_time - time_refresh >= 60:
+            time_refresh = curr_time
+            reload_all_sensors_to_monitor_dict(all_sensors_to_monitor, last_updated)
+        ##--  --##

server/server.py

@@ -509,10 +509,11 @@ class D4_Server(Protocol, TimeoutMixin):
                 redis_server_metadata.zincrby('stat_uuid_type:{}:{}'.format(date, data_header['uuid_header']), 1, data_header['type'])
 
                 #
+                d4_packet_rcv_time = int(time.time())
                 if not redis_server_metadata.hexists('metadata_uuid:{}'.format(data_header['uuid_header']), 'first_seen'):
-                    redis_server_metadata.hset('metadata_uuid:{}'.format(data_header['uuid_header']), 'first_seen', data_header['timestamp'])
-                redis_server_metadata.hset('metadata_uuid:{}'.format(data_header['uuid_header']), 'last_seen', data_header['timestamp'])
-                redis_server_metadata.hset('metadata_type_by_uuid:{}:{}'.format(data_header['uuid_header'], data_header['type']), 'last_seen', data_header['timestamp'])
+                    redis_server_metadata.hset('metadata_uuid:{}'.format(data_header['uuid_header']), 'first_seen', d4_packet_rcv_time)
+                redis_server_metadata.hset('metadata_uuid:{}'.format(data_header['uuid_header']), 'last_seen', d4_packet_rcv_time)
+                redis_server_metadata.hset('metadata_type_by_uuid:{}:{}'.format(data_header['uuid_header'], data_header['type']), 'last_seen', d4_packet_rcv_time)
 
                 if not self.data_saved:
                     #UUID IP:           ## TODO: use d4 timestamp ?
@@ -523,7 +524,7 @@ class D4_Server(Protocol, TimeoutMixin):
                 if self.update_stream_type:
 
                     if not redis_server_metadata.hexists('metadata_type_by_uuid:{}:{}'.format(data_header['uuid_header'], data_header['type']), 'first_seen'):
-                        redis_server_metadata.hset('metadata_type_by_uuid:{}:{}'.format(data_header['uuid_header'], data_header['type']), 'first_seen', data_header['timestamp'])
+                        redis_server_metadata.hset('metadata_type_by_uuid:{}:{}'.format(data_header['uuid_header'], data_header['type']), 'first_seen', d4_packet_rcv_time)
                     self.update_stream_type = False
                 return 0
             else:

server/web/Flask_server.py

@@ -35,6 +35,7 @@ import Analyzer_Queue
 from blueprints.restApi import restApi
 from blueprints.settings import settings
 from blueprints.analyzer_queue import analyzer_queue
+from blueprints.D4_sensors import D4_sensors
 
 baseUrl = ''
 if baseUrl != '':
@@ -114,6 +115,7 @@ login_manager.init_app(app)
 app.register_blueprint(restApi)
 app.register_blueprint(settings)
 app.register_blueprint(analyzer_queue)
+app.register_blueprint(D4_sensors)
 # =========       =========#
 
 # ========= LOGIN MANAGER ========
@@ -281,7 +283,7 @@ def login():
     if request.method == 'POST':
         username = request.form.get('username')
         password = request.form.get('password')
-        #next_page = request.form.get('next_page')
+        next_page = request.form.get('next_page')
 
         if username is not None:
             user = User.get(username)
@@ -304,7 +306,10 @@ def login():
                 if user.request_password_change():
                     return redirect(url_for('change_password'))
                 else:
-                    return redirect(url_for('index'))
+                    if next_page and next_page!='None':
+                        return redirect(next_page)
+                    else:
+                        return redirect(url_for('index'))
             # login failed
             else:
                 # set brute force protection
@@ -320,9 +325,9 @@ def login():
         return 'please provide a valid username'
 
     else:
-        #next_page = request.args.get('next')
+        next_page = request.args.get('next')
         error = request.args.get('error')
-        return render_template("login.html" , error=error)
+        return render_template("login.html" , error=error, next_page=next_page)
 
 @app.route('/change_password', methods=['POST', 'GET'])
 @login_required
@@ -595,6 +600,8 @@ def uuid_management():
                     "temp_blacklist_uuid": temp_blacklist_uuid,
                     "blacklisted_uuid": blacklisted_uuid, "blacklisted_ip_by_uuid": blacklisted_ip_by_uuid,
                     "first_seen_gmt": first_seen_gmt, "last_seen_gmt": last_seen_gmt, "Error": Error}
+                    
+        data_uuid['is_monitored'] = Sensor.is_sensor_monitored(uuid_sensor)
 
         if redis_server_stream.sismember('active_connection', uuid_sensor):
             active_connection = True

server/web/blueprints/D4_sensors.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+'''
+    Flask functions and routes for all D4 sensors
+'''
+
+import os
+import re
+import sys
+import redis
+
+sys.path.append(os.path.join(os.environ['D4_HOME'], 'lib'))
+import ConfigLoader
+import Sensor
+
+from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response
+from flask_login import login_required, current_user
+
+from Role_Manager import login_admin, login_user_basic
+
+# ============ BLUEPRINT ============
+
+D4_sensors = Blueprint('D4_sensors', __name__, template_folder='templates')
+
+# ============ VARIABLES ============
+
+### Config ###
+config_loader = ConfigLoader.ConfigLoader()
+r_serv_metadata = config_loader.get_redis_conn("Redis_METADATA")
+r_serv_db = config_loader.get_redis_conn("Redis_SERV")
+config_loader = None
+###  ###
+
+# ============ FUNCTIONS ============
+
+
+# ============= ROUTES ==============
+
+@D4_sensors.route("/sensors/monitoring/add", methods=['GET'])
+@login_required
+@login_user_basic
+def add_sensor_to_monitor():
+    sensor_uuid = request.args.get("uuid")
+    return render_template("sensors/add_sensor_to_monitor.html",
+                                sensor_uuid=sensor_uuid)
+
+@D4_sensors.route("/sensors/monitoring/add_post", methods=['POST'])
+@login_required
+@login_user_basic
+def add_sensor_to_monitor_post():
+    sensor_uuid = request.form.get("uuid")
+    delta_time = request.form.get("delta_time")
+    res = Sensor.api_add_sensor_to_monitor({'uuid':sensor_uuid, 'delta_time': delta_time})
+    if res:
+        Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
+    return redirect(url_for('uuid_management', uuid=sensor_uuid))
+
+@D4_sensors.route("/sensors/monitoring/delete", methods=['GET'])
+@login_required
+@login_user_basic
+def delete_sensor_to_monitor():
+    sensor_uuid = request.args.get("uuid")
+    res = Sensor.api_delete_sensor_to_monitor({'uuid':sensor_uuid})
+    if res:
+        Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
+    return redirect(url_for('uuid_management', uuid=sensor_uuid))
+
+
+#
+#
+#
+#
+#
+#
+#
+#

server/web/blueprints/analyzer_queue.py

@@ -18,7 +18,6 @@ from flask import Flask, render_template, jsonify, request, Blueprint, redirect,
 from flask_login import login_required, current_user
 
 from Role_Manager import login_admin, login_user_basic
-from Role_Manager import create_user_db, edit_user_db, delete_user_db, check_password_strength, generate_new_token, gen_password, get_all_role
 
 # ============ BLUEPRINT ============
 

server/web/blueprints/restApi.py

@@ -142,8 +142,8 @@ def is_valid_uuid_v4(header_uuid):
     except:
         return False
 
-def one():
-    return 1
+def build_json_response(resp_data, resp_code):
+    return Response(json.dumps(resp_data, indent=2, sort_keys=True), mimetype='application/json'), resp_code
 
 # ============= ROUTES ==============
 
@@ -154,3 +154,9 @@ def add_sensor_register():
     data = request.get_json()
     res = Sensor.register_sensor(data)
     return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
+
+@restApi.route("/api/v1/sensors/monitoring/errors", methods=['GET'])
+#@token_required('login_user_basic')
+def get_all_sensors_connection_errors():
+    res = Sensor.api_get_all_sensors_connection_errors()
+    return build_json_response(res[0], res[1])

server/web/templates/login.html

@@ -69,7 +69,8 @@
 				<form class="form-signin" action="{{ url_for('login')}}"  method="post">
 		      <img class="mb-4" src="{{ url_for('static', filename='img/d4-logo.png')}}" width="300">
 		      <h1 class="h3 mb-3 text-secondary">Please sign in</h1>
-		      <label for="inputEmail" class="sr-only">Email address</label>
+					<input type="text" id="next_page" name="next_page" value="{{next_page}}" hidden>
+					<label for="inputEmail" class="sr-only">Email address</label>
 		      <input type="email" id="inputEmail" name="username" class="form-control" placeholder="Email address" required autofocus>
 		      <label for="inputPassword" class="sr-only">Password</label>
 		      <input type="password" id="inputPassword" name="password" class="form-control {% if error %}is-invalid{% endif %}" placeholder="Password" required>

server/web/templates/sensors/add_sensor_to_monitor.html

@@ -0,0 +1,57 @@
+<!DOCTYPE html>
+
+<html>
+<head>
+	<title>D4-Project</title>
+	<link rel="icon" href="{{ url_for('static', filename='img/d4-logo.png')}}">
+	<!-- Core CSS -->
+	<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
+
+	<!-- JS -->
+	<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/bootstrap.min.js')}}"></script>
+</head>
+
+<body>
+
+	{% include 'navbar.html' %}
+
+	<form action="{{ url_for('D4_sensors.add_sensor_to_monitor_post') }}" method="post" enctype=multipart/form-data>
+
+		<div class="d-flex justify-content-center">
+			<div class="col-sm-6">
+				<h4 class="my-3">Monitor a Sensor</h4>
+				<div class="form-group">
+					<input class="form-control text-center bg-dark text-white" type="text" value="{{sensor_uuid}}" disabled>
+					<input type="text" name="uuid" id="uuid" value="{{sensor_uuid}}" hidden>
+				</div>
+
+				<div class="input-group mt-2 mb-2">
+					<div class="input-group-prepend">
+						<span class="input-group-text bg-light"><i class="fa fa-clock-o"></i>&nbsp;</span>
+					</div>
+					<input class="form-control" type="number" id="delta_time" value="3600" min="30" name="delta_time" required>
+					<div class="input-group-append">
+						<span class="input-group-text">Maxinum Time (seconds) between two D4 packets</span>
+					</div>
+				</div>
+
+				<div class="form-group">
+					<button class="btn btn-primary" type="submit">Monitor Sensor</button>
+				</div>
+			</div>
+		</div>
+
+	</form>
+
+	{% include 'navfooter.html' %}
+</body>
+
+<script>
+$(document).ready(function(){
+	$("#nav-sensor").addClass("active");
+});
+
+</script>

server/web/templates/uuid_management.html

@@ -101,6 +101,18 @@
 		</div>
 	</div>
 
+	<div class="d-flex justify-content-center mt-2">
+		{% if not data_uuid.get('is_monitored', False) %}
+			<a href="{{ url_for('D4_sensors.add_sensor_to_monitor') }}?uuid={{uuid_sensor}}">
+				<button type="button" class="btn btn-primary">Monitor Sensor</button>
+			</a>
+		{% else %}
+			<a href="{{ url_for('D4_sensors.delete_sensor_to_monitor') }}?uuid={{uuid_sensor}}">
+				<button type="button" class="btn btn-danger">Remove Sensor from monitoring</button>
+			</a>
+		{% endif %}
+	</div>
+
 	<div class="card-deck justify-content-center ml-0 mr-0">
 		<div class="card border-dark mt-3" style="max-width: 18rem;">
 		  <div class="card-body text-dark">