chg: [server + UI + API] add server mode: registration + shared-secret + API-UI: approve/delete pending sensors

server/lib/Sensor.py

@@ -2,6 +2,7 @@
 # -*-coding:UTF-8 -*
 
 import os
+import time
 import uuid
 import redis
 
@@ -13,7 +14,8 @@ port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380))
 r_serv_db = redis.StrictRedis(
     host=host_redis_metadata,
     port=port_redis_metadata,
-    db=0)
+    db=0,
+    decode_responses=True)
 
 def is_valid_uuid_v4(UUID):
     UUID = UUID.replace('-', '')
@@ -23,6 +25,20 @@ def is_valid_uuid_v4(UUID):
     except:
         return False
 
+def _get_sensor_metadata(sensor_uuid, first_seen=True, last_seen=True, mail=True, description=True):
+
+    meta_sensor = {}
+    meta_sensor['uuid'] = sensor_uuid
+    if first_seen:
+        meta_sensor['first_seen'] = r_serv_db.hget('metadata_uuid:{}'.format(sensor_uuid), 'first_seen')
+    if last_seen:
+        meta_sensor['last_seen'] = r_serv_db.hget('metadata_uuid:{}'.format(sensor_uuid), 'last_seen')
+    if description:
+        meta_sensor['description'] = r_serv_db.hget('metadata_uuid:{}'.format(sensor_uuid), 'description')
+    if mail:
+        meta_sensor['mail'] = r_serv_db.hget('metadata_uuid:{}'.format(sensor_uuid), 'user_mail')
+    return meta_sensor
+
 ## TODO: add description
 def register_sensor(req_dict):
     sensor_uuid = req_dict.get('uuid', None)
@@ -41,6 +57,8 @@ def register_sensor(req_dict):
         return ({"status": "error", "reason": "Mandatory parameter(s) not provided"}, 400)
     else:
         hmac_key = escape(hmac_key)
+        if len(hmac_key)>100:
+            hmac_key=hmac_key[:100]
 
 
     res = _register_sensor(sensor_uuid, hmac_key, user_id=user_id, description=None)
@@ -53,4 +71,43 @@ def _register_sensor(sensor_uuid, secret_key, user_id=None, description=None):
         r_serv_db.hset('metadata_uuid:{}'.format(sensor_uuid), 'user_mail', user_id)
     if description:
         r_serv_db.hset('metadata_uuid:{}'.format(sensor_uuid), 'description', description)
+    r_serv_db.sadd('sensor_pending_registration', sensor_uuid)
+    return ({'uuid': sensor_uuid}, 200)
+
+def get_pending_sensor():
+    return list(r_serv_db.smembers('sensor_pending_registration'))
+
+def approve_sensor(req_dict):
+    sensor_uuid = req_dict.get('uuid', None)
+    if not is_valid_uuid_v4(sensor_uuid):
+        return ({"status": "error", "reason": "Invalid uuid"}, 400)
+    sensor_uuid = sensor_uuid.replace('-', '')
+    # sensor not registred
+    #if r_serv_db.sismember('sensor_pending_registration', sensor_uuid):
+    #    return ({"status": "error", "reason": "Sensor not registred"}, 404)
+    # sensor already approved
+    if r_serv_db.sismember('registered_uuid', sensor_uuid):
+        return ({"status": "error", "reason": "Sensor already approved"}, 409)
+    return _approve_sensor(sensor_uuid)
+
+def _approve_sensor(sensor_uuid):
+    r_serv_db.sadd('registered_uuid', sensor_uuid)
+    r_serv_db.srem('sensor_pending_registration', sensor_uuid)
+    return ({'uuid': sensor_uuid}, 200)
+
+def delete_pending_sensor(req_dict):
+    sensor_uuid = req_dict.get('uuid', None)
+    if not is_valid_uuid_v4(sensor_uuid):
+        return ({"status": "error", "reason": "Invalid uuid"}, 400)
+    sensor_uuid = sensor_uuid.replace('-', '')
+    # sensor not registred
+    #if r_serv_db.sismember('sensor_pending_registration', sensor_uuid):
+    #    return ({"status": "error", "reason": "Sensor not registred"}, 404)
+    # sensor already approved
+    if not r_serv_db.sismember('sensor_pending_registration', sensor_uuid):
+        return ({"status": "error", "reason": "Not Pending Sensor"}, 409)
+    return _delete_pending_sensor(sensor_uuid)
+
+def _delete_pending_sensor(sensor_uuid):
+    r_serv_db.srem('sensor_pending_registration', sensor_uuid)
     return ({'uuid': sensor_uuid}, 200)

server/server.py

@@ -26,6 +26,9 @@ hmac_key = os.getenv('D4_HMAC_KEY', b'private key to change')
 accepted_type = [1, 2, 4, 8, 254]
 accepted_extended_type = ['ja3-jl']
 
+all_server_modes = ('registration', 'shared-secret')
+server_mode = 'registration'
+
 timeout_time = 30
 
 header_size = 62
@@ -39,6 +42,9 @@ port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379))
 host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost")
 port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380))
 
+
+### REDIS ###
+
 redis_server_stream = redis.StrictRedis(
                     host=host_redis_stream,
                     port=port_redis_stream,
@@ -61,19 +67,111 @@ except redis.exceptions.ConnectionError:
     print('Error: Redis server {}:{}, ConnectionError'.format(host_redis_metadata, port_redis_metadata))
     sys.exit(1)
 
+### REDIS ###
+
 # set hmac default key
 redis_server_metadata.set('server:hmac_default_key', hmac_key)
 
 # init redis_server_metadata
-redis_server_metadata.delete('server:accepted_type')
 for type in accepted_type:
     redis_server_metadata.sadd('server:accepted_type', type)
-redis_server_metadata.delete('server:accepted_extended_type')
 for type in accepted_extended_type:
     redis_server_metadata.sadd('server:accepted_extended_type', type)
 
 dict_all_connection =  {}
 
+### FUNCTIONS ###
+
+# kick sensors
+def kick_sensors():
+    for client_uuid in redis_server_stream.smembers('server:sensor_to_kick'):
+        client_uuid = client_uuid.decode()
+        for session_uuid in redis_server_stream.smembers('map:active_connection-uuid-session_uuid:{}'.format(client_uuid)):
+            session_uuid = session_uuid.decode()
+            logger.warning('Sensor kicked uuid={}, session_uuid={}'.format(client_uuid, session_uuid))
+            redis_server_stream.set('temp_blacklist_uuid:{}'.format(client_uuid), 'some random string')
+            redis_server_stream.expire('temp_blacklist_uuid:{}'.format(client_uuid), 30)
+            dict_all_connection[session_uuid].transport.abortConnection()
+        redis_server_stream.srem('server:sensor_to_kick', client_uuid)
+
+# Unpack D4 Header
+#def unpack_header(data):
+#    data_header = {}
+#    if len(data) >= header_size:
+#        data_header['version'] = struct.unpack('B', data[0:1])[0]
+#        data_header['type'] = struct.unpack('B', data[1:2])[0]
+#        data_header['uuid_header'] = data[2:18].hex()
+#        data_header['timestamp'] = struct.unpack('Q', data[18:26])[0]
+#        data_header['hmac_header'] = data[26:58]
+#        data_header['size'] = struct.unpack('I', data[58:62])[0]
+#    return data_header
+
+def is_valid_uuid_v4(header_uuid):
+    try:
+        uuid_test = uuid.UUID(hex=header_uuid, version=4)
+        return uuid_test.hex == header_uuid
+    except:
+        logger.info('Not UUID v4: uuid={}, session_uuid={}'.format(header_uuid, self.session_uuid))
+        return False
+
+# # TODO:  check timestamp
+def is_valid_header(uuid_to_check, type):
+    if is_valid_uuid_v4(uuid_to_check):
+        if redis_server_metadata.sismember('server:accepted_type', type):
+            return True
+        else:
+            logger.warning('Invalid type, the server don\'t accept this type: {}, uuid={}, session_uuid={}'.format(type, uuid_to_check, self.session_uuid))
+            return False
+    else:
+        logger.info('Invalid Header, uuid={}, session_uuid={}'.format(uuid_to_check, self.session_uuid))
+        return False
+
+def extract_ip(ip_string):
+    #remove interface
+    ip_string = ip_string.split('%')[0]
+    # IPv4
+    #extract ipv4
+    if '.' in ip_string:
+        return ip_string.split(':')[-1]
+    # IPv6
+    else:
+        return ip_string
+
+def server_mode_registration(header_uuid):
+    # only accept registred uuid
+    if server_mode == 'registration':
+        if not redis_server_metadata.sismember('registered_uuid', header_uuid):
+            error_msg = 'Not registred UUID={}, connection closed'.format(header_uuid)
+            print(error_msg)
+            logger.warning(error_msg)
+            #redis_server_metadata.hset('metadata_uuid:{}'.format(data_header['uuid_header']), 'Error', 'Error: This UUID is temporarily blacklisted')
+            return False
+        else:
+            return True
+    else:
+        return True
+
+def is_client_ip_blacklisted():
+    pass
+
+def is_uuid_blacklisted(uuid):
+    return redis_server_metadata.sismember('blacklist_uuid', data_header['uuid_header'])
+
+
+# return True if not blocked
+# False if blacklisted
+def check_blacklist():
+    pass
+
+# Kill Connection + create log
+#def manual_abort_connection(self, message, log_level='WARNING'):
+#    logger.log(message)
+#    self.transport.abortConnection()
+#    return 1
+
+###  ###
+
+
 class D4_Server(Protocol, TimeoutMixin):
 
     def __init__(self):
@@ -96,20 +194,12 @@ class D4_Server(Protocol, TimeoutMixin):
 
     def dataReceived(self, data):
         # check and kick sensor by uuid
-        for client_uuid in redis_server_stream.smembers('server:sensor_to_kick'):
+        kick_sensors()
-            client_uuid = client_uuid.decode()
-            for session_uuid in redis_server_stream.smembers('map:active_connection-uuid-session_uuid:{}'.format(client_uuid)):
-                session_uuid = session_uuid.decode()
-                logger.warning('Sensor kicked uuid={}, session_uuid={}'.format(client_uuid, session_uuid))
-                redis_server_stream.set('temp_blacklist_uuid:{}'.format(client_uuid), 'some random string')
-                redis_server_stream.expire('temp_blacklist_uuid:{}'.format(client_uuid), 30)
-                dict_all_connection[session_uuid].transport.abortConnection()
-            redis_server_stream.srem('server:sensor_to_kick', client_uuid)
 
         self.resetTimeout()
         if self.first_connection or self.ip is None:
             client_info = self.transport.client
-            self.ip = self.extract_ip(client_info[0])
+            self.ip = extract_ip(client_info[0])
             self.source_port = client_info[1]
             logger.debug('New connection, ip={}, port={} session_uuid={}'.format(self.ip, self.source_port, self.session_uuid))
         # check blacklisted_ip
@@ -171,37 +261,7 @@ class D4_Server(Protocol, TimeoutMixin):
             data_header['timestamp'] = struct.unpack('Q', data[18:26])[0]
             data_header['hmac_header'] = data[26:58]
             data_header['size'] = struct.unpack('I', data[58:62])[0]
-            return data_header
+        return data_header
-
-    def extract_ip(self, ip_string):
-        #remove interface
-        ip_string = ip_string.split('%')[0]
-        # IPv4
-        #extract ipv4
-        if '.' in ip_string:
-            return ip_string.split(':')[-1]
-        # IPv6
-        else:
-            return ip_string
-
-    def is_valid_uuid_v4(self, header_uuid):
-        try:
-            uuid_test = uuid.UUID(hex=header_uuid, version=4)
-            return uuid_test.hex == header_uuid
-        except:
-            logger.info('Not UUID v4: uuid={}, session_uuid={}'.format(header_uuid, self.session_uuid))
-            return False
-
-    # # TODO:  check timestamp
-    def is_valid_header(self, uuid_to_check, type):
-        if self.is_valid_uuid_v4(uuid_to_check):
-            if redis_server_metadata.sismember('server:accepted_type', type):
-                return True
-            else:
-                logger.warning('Invalid type, the server don\'t accept this type: {}, uuid={}, session_uuid={}'.format(type, uuid_to_check, self.session_uuid))
-        else:
-            logger.info('Invalid Header, uuid={}, session_uuid={}'.format(uuid_to_check, self.session_uuid))
-            return False
 
     def check_connection_validity(self, data_header):
         # blacklist ip by uuid
@@ -217,6 +277,11 @@ class D4_Server(Protocol, TimeoutMixin):
             self.transport.abortConnection()
             return False
 
+        # Check server mode
+        if not server_mode_registration(data_header['uuid_header']):
+            self.transport.abortConnection()
+            return False
+
         # check temp blacklist
         if redis_server_stream.exists('temp_blacklist_uuid:{}'.format(data_header['uuid_header'])):
             logger.warning('Temporarily Blacklisted UUID={}, connection closed'.format(data_header['uuid_header']))
@@ -246,7 +311,7 @@ class D4_Server(Protocol, TimeoutMixin):
             if data_header:
                 if not self.check_connection_validity(data_header):
                     return 1
-                if self.is_valid_header(data_header['uuid_header'], data_header['type']):
+                if is_valid_header(data_header['uuid_header'], data_header['type']):
 
                     # auto kill connection # TODO: map type
                     if self.first_connection:

server/web/Flask_server.py

@@ -17,17 +17,18 @@ import configparser
 
 import subprocess
 
-from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response
+from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, escape
 from flask_login import LoginManager, current_user, login_user, logout_user, login_required
 
 import bcrypt
 
 # Import Role_Manager
 from Role_Manager import create_user_db, check_password_strength, check_user_role_integrity
-from Role_Manager import login_user_basic
+from Role_Manager import login_user_basic, login_admin
 
 sys.path.append(os.path.join(os.environ['D4_HOME'], 'lib'))
 from User import User
+import Sensor
 
 # Import Blueprint
 from blueprints.restApi import restApi
@@ -709,6 +710,38 @@ def blacklisted_uuid():
                             unblacklisted_uuid=unblacklisted_uuid, blacklisted_uuid=blacklisted_uuid)
 
 
+@app.route('/server/pending_sensor')
+@login_required
+@login_admin
+def pending_sensors():
+    sensors = Sensor.get_pending_sensor()
+    all_pending = []
+    for sensor_uuid in sensors:
+        all_pending.append(Sensor._get_sensor_metadata(sensor_uuid, first_seen=False, last_seen=False))
+    return render_template("pending_sensor.html", all_pending=all_pending)
+
+@app.route('/server/approve_sensor')
+@login_required
+@login_admin
+def approve_sensor():
+    uuid_sensor = request.args.get('uuid')
+    res = Sensor.approve_sensor({'uuid': uuid_sensor})
+    if res[1] == 200:
+        return redirect(url_for('pending_sensors'))
+    else:
+        return jsonify(res[0])
+
+@app.route('/server/delete_pending_sensor')
+@login_required
+@login_admin
+def delete_pending_sensor():
+    uuid_sensor = request.args.get('uuid')
+    res = Sensor.delete_pending_sensor({'uuid': uuid_sensor})
+    if res[1] == 200:
+        return redirect(url_for('pending_sensors'))
+    else:
+        return jsonify(res[0])
+
 @app.route('/uuid_change_stream_max_size')
 @login_required
 @login_user_basic
@@ -1042,7 +1075,10 @@ def set_uuid_hmac_key():
     uuid_sensor = request.args.get('uuid')
     user = request.args.get('redirect')
     key = request.args.get('key')
-    redis_server_metadata.hset('metadata_uuid:{}'.format(uuid_sensor), 'hmac_key', key)
+    hmac_key = escape(key)
+    if len(hmac_key)>100:
+        hmac_key=hmac_key[:100]
+    redis_server_metadata.hset('metadata_uuid:{}'.format(uuid_sensor), 'hmac_key', hmac_key)
     if user:
         return redirect(url_for('uuid_management', uuid=uuid_sensor))
 

server/web/templates/pending_sensor.html

@@ -0,0 +1,96 @@
+<!DOCTYPE html>
+
+<html>
+<head>
+	<title>D4-Project</title>
+	<link rel="icon" href="{{ url_for('static', filename='img/d4-logo.png')}}">
+	<!-- Core CSS -->
+	<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
+
+	<!-- JS -->
+	<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/bootstrap.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
+
+  <style>
+	.popover{
+		max-width: 100%;
+	}
+  </style>
+
+
+</head>
+
+<body>
+
+	{% include 'navbar.html' %}
+
+	<div class="d-flex justify-content-center">
+		<div class="card border-dark mt-3 text-center" style="max-width: 30rem;">
+			<div class="card-body text-dark">
+				<h5 class="card-title">Approve New Sensor UUID</h5>
+				<input class="form-control" type="text" id="uuid" value="" required>
+				<button type="button" class="btn btn-outline-secondary mt-1" onclick="window.location.href ='{{ url_for('approve_sensor') }}?uuid='+$('#uuid').val();">Approve UUID</button>
+			</div>
+		</div>
+	</div>
+
+	<div class="py-3 mx-2">
+		<table class="table table-striped table-bordered table-hover text-center" id="myTable_1">
+			<thead>
+				<tr>
+					<th class="bg-info text-white">UUID</th>
+					<th class="bg-info text-white">description</th>
+					<th class="bg-info text-white">mail</th>
+					<th class="bg-info text-white"></th>
+				</tr>
+			</thead>
+			<tbody>
+				{% for row_uuid in all_pending %}
+					<tr data-trigger="hover" title="" data-content="test content" data-original-title="test title">
+						<td>
+							<a class="" href="{{ url_for('uuid_management') }}?uuid={{row_uuid['uuid']}}">
+								{{row_uuid['uuid']}}
+							</a>
+						</td>
+						<td>{{row_uuid['description']}}</td>
+						<td>{{row_uuid['mail']}}</td>
+						<td>
+							<a href="{{ url_for('approve_sensor') }}?uuid={{row_uuid['uuid']}}">
+								<button type="button" class="btn btn-outline-info"><i class="fa fa-plus"></i></button>
+							</a>
+							<a href="{{ url_for('delete_pending_sensor') }}?uuid={{row_uuid['uuid']}}">
+								<button type="button" class="btn btn-outline-danger"><i class="fa fa-trash"></i></button>
+							</a>
+						</td>
+					</tr>
+				{% endfor %}
+			</tbody>
+		</table>
+	</div>
+
+
+	{% include 'navfooter.html' %}
+</body>
+
+<script>
+$(document).ready(function(){
+	table = $('#myTable_1').DataTable(
+		{
+			"aLengthMenu": [[5, 10, 15, 20, -1], [5, 10, 15, 20, "All"]],
+			"iDisplayLength": 10,
+			"order": [[ 0, "asc" ]]
+		}
+	);
+	$('[data-toggle="popover"]').popover({
+		placement: 'top',
+		container: 'body',
+		html : false,
+	})
+});
+
+</script>