From 3ce8557cffee37a39f770d9f49f1822e5c034a25 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Mon, 30 Sep 2019 17:07:25 +0200 Subject: [PATCH] chg: [server + UI + API] add server mode: registration + shared-secret + API-UI: approve/delete pending sensors --- server/lib/Sensor.py | 59 ++++++++- server/server.py | 153 ++++++++++++++++------- server/web/Flask_server.py | 42 ++++++- server/web/templates/pending_sensor.html | 96 ++++++++++++++ 4 files changed, 302 insertions(+), 48 deletions(-) create mode 100644 server/web/templates/pending_sensor.html diff --git a/server/lib/Sensor.py b/server/lib/Sensor.py index 82a697a..0940ecd 100755 --- a/server/lib/Sensor.py +++ b/server/lib/Sensor.py @@ -2,6 +2,7 @@ # -*-coding:UTF-8 -* import os +import time import uuid import redis @@ -13,7 +14,8 @@ port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380)) r_serv_db = redis.StrictRedis( host=host_redis_metadata, port=port_redis_metadata, - db=0) + db=0, + decode_responses=True) def is_valid_uuid_v4(UUID): UUID = UUID.replace('-', '') @@ -23,6 +25,20 @@ def is_valid_uuid_v4(UUID): except: return False +def _get_sensor_metadata(sensor_uuid, first_seen=True, last_seen=True, mail=True, description=True): + + meta_sensor = {} + meta_sensor['uuid'] = sensor_uuid + if first_seen: + meta_sensor['first_seen'] = r_serv_db.hget('metadata_uuid:{}'.format(sensor_uuid), 'first_seen') + if last_seen: + meta_sensor['last_seen'] = r_serv_db.hget('metadata_uuid:{}'.format(sensor_uuid), 'last_seen') + if description: + meta_sensor['description'] = r_serv_db.hget('metadata_uuid:{}'.format(sensor_uuid), 'description') + if mail: + meta_sensor['mail'] = r_serv_db.hget('metadata_uuid:{}'.format(sensor_uuid), 'user_mail') + return meta_sensor + ## TODO: add description def register_sensor(req_dict): sensor_uuid = req_dict.get('uuid', None) @@ -41,6 +57,8 @@ def register_sensor(req_dict): return ({"status": "error", "reason": "Mandatory parameter(s) not provided"}, 400) else: hmac_key = escape(hmac_key) + if len(hmac_key)>100: + hmac_key=hmac_key[:100] res = _register_sensor(sensor_uuid, hmac_key, user_id=user_id, description=None) @@ -53,4 +71,43 @@ def _register_sensor(sensor_uuid, secret_key, user_id=None, description=None): r_serv_db.hset('metadata_uuid:{}'.format(sensor_uuid), 'user_mail', user_id) if description: r_serv_db.hset('metadata_uuid:{}'.format(sensor_uuid), 'description', description) + r_serv_db.sadd('sensor_pending_registration', sensor_uuid) + return ({'uuid': sensor_uuid}, 200) + +def get_pending_sensor(): + return list(r_serv_db.smembers('sensor_pending_registration')) + +def approve_sensor(req_dict): + sensor_uuid = req_dict.get('uuid', None) + if not is_valid_uuid_v4(sensor_uuid): + return ({"status": "error", "reason": "Invalid uuid"}, 400) + sensor_uuid = sensor_uuid.replace('-', '') + # sensor not registred + #if r_serv_db.sismember('sensor_pending_registration', sensor_uuid): + # return ({"status": "error", "reason": "Sensor not registred"}, 404) + # sensor already approved + if r_serv_db.sismember('registered_uuid', sensor_uuid): + return ({"status": "error", "reason": "Sensor already approved"}, 409) + return _approve_sensor(sensor_uuid) + +def _approve_sensor(sensor_uuid): + r_serv_db.sadd('registered_uuid', sensor_uuid) + r_serv_db.srem('sensor_pending_registration', sensor_uuid) + return ({'uuid': sensor_uuid}, 200) + +def delete_pending_sensor(req_dict): + sensor_uuid = req_dict.get('uuid', None) + if not is_valid_uuid_v4(sensor_uuid): + return ({"status": "error", "reason": "Invalid uuid"}, 400) + sensor_uuid = sensor_uuid.replace('-', '') + # sensor not registred + #if r_serv_db.sismember('sensor_pending_registration', sensor_uuid): + # return ({"status": "error", "reason": "Sensor not registred"}, 404) + # sensor already approved + if not r_serv_db.sismember('sensor_pending_registration', sensor_uuid): + return ({"status": "error", "reason": "Not Pending Sensor"}, 409) + return _delete_pending_sensor(sensor_uuid) + +def _delete_pending_sensor(sensor_uuid): + r_serv_db.srem('sensor_pending_registration', sensor_uuid) return ({'uuid': sensor_uuid}, 200) diff --git a/server/server.py b/server/server.py index 4cd04fa..98f11ca 100755 --- a/server/server.py +++ b/server/server.py @@ -26,6 +26,9 @@ hmac_key = os.getenv('D4_HMAC_KEY', b'private key to change') accepted_type = [1, 2, 4, 8, 254] accepted_extended_type = ['ja3-jl'] +all_server_modes = ('registration', 'shared-secret') +server_mode = 'registration' + timeout_time = 30 header_size = 62 @@ -39,6 +42,9 @@ port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379)) host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost") port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380)) + +### REDIS ### + redis_server_stream = redis.StrictRedis( host=host_redis_stream, port=port_redis_stream, @@ -61,19 +67,111 @@ except redis.exceptions.ConnectionError: print('Error: Redis server {}:{}, ConnectionError'.format(host_redis_metadata, port_redis_metadata)) sys.exit(1) +### REDIS ### + # set hmac default key redis_server_metadata.set('server:hmac_default_key', hmac_key) # init redis_server_metadata -redis_server_metadata.delete('server:accepted_type') for type in accepted_type: redis_server_metadata.sadd('server:accepted_type', type) -redis_server_metadata.delete('server:accepted_extended_type') for type in accepted_extended_type: redis_server_metadata.sadd('server:accepted_extended_type', type) dict_all_connection = {} +### FUNCTIONS ### + +# kick sensors +def kick_sensors(): + for client_uuid in redis_server_stream.smembers('server:sensor_to_kick'): + client_uuid = client_uuid.decode() + for session_uuid in redis_server_stream.smembers('map:active_connection-uuid-session_uuid:{}'.format(client_uuid)): + session_uuid = session_uuid.decode() + logger.warning('Sensor kicked uuid={}, session_uuid={}'.format(client_uuid, session_uuid)) + redis_server_stream.set('temp_blacklist_uuid:{}'.format(client_uuid), 'some random string') + redis_server_stream.expire('temp_blacklist_uuid:{}'.format(client_uuid), 30) + dict_all_connection[session_uuid].transport.abortConnection() + redis_server_stream.srem('server:sensor_to_kick', client_uuid) + +# Unpack D4 Header +#def unpack_header(data): +# data_header = {} +# if len(data) >= header_size: +# data_header['version'] = struct.unpack('B', data[0:1])[0] +# data_header['type'] = struct.unpack('B', data[1:2])[0] +# data_header['uuid_header'] = data[2:18].hex() +# data_header['timestamp'] = struct.unpack('Q', data[18:26])[0] +# data_header['hmac_header'] = data[26:58] +# data_header['size'] = struct.unpack('I', data[58:62])[0] +# return data_header + +def is_valid_uuid_v4(header_uuid): + try: + uuid_test = uuid.UUID(hex=header_uuid, version=4) + return uuid_test.hex == header_uuid + except: + logger.info('Not UUID v4: uuid={}, session_uuid={}'.format(header_uuid, self.session_uuid)) + return False + +# # TODO: check timestamp +def is_valid_header(uuid_to_check, type): + if is_valid_uuid_v4(uuid_to_check): + if redis_server_metadata.sismember('server:accepted_type', type): + return True + else: + logger.warning('Invalid type, the server don\'t accept this type: {}, uuid={}, session_uuid={}'.format(type, uuid_to_check, self.session_uuid)) + return False + else: + logger.info('Invalid Header, uuid={}, session_uuid={}'.format(uuid_to_check, self.session_uuid)) + return False + +def extract_ip(ip_string): + #remove interface + ip_string = ip_string.split('%')[0] + # IPv4 + #extract ipv4 + if '.' in ip_string: + return ip_string.split(':')[-1] + # IPv6 + else: + return ip_string + +def server_mode_registration(header_uuid): + # only accept registred uuid + if server_mode == 'registration': + if not redis_server_metadata.sismember('registered_uuid', header_uuid): + error_msg = 'Not registred UUID={}, connection closed'.format(header_uuid) + print(error_msg) + logger.warning(error_msg) + #redis_server_metadata.hset('metadata_uuid:{}'.format(data_header['uuid_header']), 'Error', 'Error: This UUID is temporarily blacklisted') + return False + else: + return True + else: + return True + +def is_client_ip_blacklisted(): + pass + +def is_uuid_blacklisted(uuid): + return redis_server_metadata.sismember('blacklist_uuid', data_header['uuid_header']) + + +# return True if not blocked +# False if blacklisted +def check_blacklist(): + pass + +# Kill Connection + create log +#def manual_abort_connection(self, message, log_level='WARNING'): +# logger.log(message) +# self.transport.abortConnection() +# return 1 + +### ### + + class D4_Server(Protocol, TimeoutMixin): def __init__(self): @@ -96,20 +194,12 @@ class D4_Server(Protocol, TimeoutMixin): def dataReceived(self, data): # check and kick sensor by uuid - for client_uuid in redis_server_stream.smembers('server:sensor_to_kick'): - client_uuid = client_uuid.decode() - for session_uuid in redis_server_stream.smembers('map:active_connection-uuid-session_uuid:{}'.format(client_uuid)): - session_uuid = session_uuid.decode() - logger.warning('Sensor kicked uuid={}, session_uuid={}'.format(client_uuid, session_uuid)) - redis_server_stream.set('temp_blacklist_uuid:{}'.format(client_uuid), 'some random string') - redis_server_stream.expire('temp_blacklist_uuid:{}'.format(client_uuid), 30) - dict_all_connection[session_uuid].transport.abortConnection() - redis_server_stream.srem('server:sensor_to_kick', client_uuid) + kick_sensors() self.resetTimeout() if self.first_connection or self.ip is None: client_info = self.transport.client - self.ip = self.extract_ip(client_info[0]) + self.ip = extract_ip(client_info[0]) self.source_port = client_info[1] logger.debug('New connection, ip={}, port={} session_uuid={}'.format(self.ip, self.source_port, self.session_uuid)) # check blacklisted_ip @@ -171,37 +261,7 @@ class D4_Server(Protocol, TimeoutMixin): data_header['timestamp'] = struct.unpack('Q', data[18:26])[0] data_header['hmac_header'] = data[26:58] data_header['size'] = struct.unpack('I', data[58:62])[0] - return data_header - - def extract_ip(self, ip_string): - #remove interface - ip_string = ip_string.split('%')[0] - # IPv4 - #extract ipv4 - if '.' in ip_string: - return ip_string.split(':')[-1] - # IPv6 - else: - return ip_string - - def is_valid_uuid_v4(self, header_uuid): - try: - uuid_test = uuid.UUID(hex=header_uuid, version=4) - return uuid_test.hex == header_uuid - except: - logger.info('Not UUID v4: uuid={}, session_uuid={}'.format(header_uuid, self.session_uuid)) - return False - - # # TODO: check timestamp - def is_valid_header(self, uuid_to_check, type): - if self.is_valid_uuid_v4(uuid_to_check): - if redis_server_metadata.sismember('server:accepted_type', type): - return True - else: - logger.warning('Invalid type, the server don\'t accept this type: {}, uuid={}, session_uuid={}'.format(type, uuid_to_check, self.session_uuid)) - else: - logger.info('Invalid Header, uuid={}, session_uuid={}'.format(uuid_to_check, self.session_uuid)) - return False + return data_header def check_connection_validity(self, data_header): # blacklist ip by uuid @@ -217,6 +277,11 @@ class D4_Server(Protocol, TimeoutMixin): self.transport.abortConnection() return False + # Check server mode + if not server_mode_registration(data_header['uuid_header']): + self.transport.abortConnection() + return False + # check temp blacklist if redis_server_stream.exists('temp_blacklist_uuid:{}'.format(data_header['uuid_header'])): logger.warning('Temporarily Blacklisted UUID={}, connection closed'.format(data_header['uuid_header'])) @@ -246,7 +311,7 @@ class D4_Server(Protocol, TimeoutMixin): if data_header: if not self.check_connection_validity(data_header): return 1 - if self.is_valid_header(data_header['uuid_header'], data_header['type']): + if is_valid_header(data_header['uuid_header'], data_header['type']): # auto kill connection # TODO: map type if self.first_connection: diff --git a/server/web/Flask_server.py b/server/web/Flask_server.py index c61a513..e5d4b31 100755 --- a/server/web/Flask_server.py +++ b/server/web/Flask_server.py @@ -17,17 +17,18 @@ import configparser import subprocess -from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response +from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, escape from flask_login import LoginManager, current_user, login_user, logout_user, login_required import bcrypt # Import Role_Manager from Role_Manager import create_user_db, check_password_strength, check_user_role_integrity -from Role_Manager import login_user_basic +from Role_Manager import login_user_basic, login_admin sys.path.append(os.path.join(os.environ['D4_HOME'], 'lib')) from User import User +import Sensor # Import Blueprint from blueprints.restApi import restApi @@ -709,6 +710,38 @@ def blacklisted_uuid(): unblacklisted_uuid=unblacklisted_uuid, blacklisted_uuid=blacklisted_uuid) +@app.route('/server/pending_sensor') +@login_required +@login_admin +def pending_sensors(): + sensors = Sensor.get_pending_sensor() + all_pending = [] + for sensor_uuid in sensors: + all_pending.append(Sensor._get_sensor_metadata(sensor_uuid, first_seen=False, last_seen=False)) + return render_template("pending_sensor.html", all_pending=all_pending) + +@app.route('/server/approve_sensor') +@login_required +@login_admin +def approve_sensor(): + uuid_sensor = request.args.get('uuid') + res = Sensor.approve_sensor({'uuid': uuid_sensor}) + if res[1] == 200: + return redirect(url_for('pending_sensors')) + else: + return jsonify(res[0]) + +@app.route('/server/delete_pending_sensor') +@login_required +@login_admin +def delete_pending_sensor(): + uuid_sensor = request.args.get('uuid') + res = Sensor.delete_pending_sensor({'uuid': uuid_sensor}) + if res[1] == 200: + return redirect(url_for('pending_sensors')) + else: + return jsonify(res[0]) + @app.route('/uuid_change_stream_max_size') @login_required @login_user_basic @@ -1042,7 +1075,10 @@ def set_uuid_hmac_key(): uuid_sensor = request.args.get('uuid') user = request.args.get('redirect') key = request.args.get('key') - redis_server_metadata.hset('metadata_uuid:{}'.format(uuid_sensor), 'hmac_key', key) + hmac_key = escape(key) + if len(hmac_key)>100: + hmac_key=hmac_key[:100] + redis_server_metadata.hset('metadata_uuid:{}'.format(uuid_sensor), 'hmac_key', hmac_key) if user: return redirect(url_for('uuid_management', uuid=uuid_sensor)) diff --git a/server/web/templates/pending_sensor.html b/server/web/templates/pending_sensor.html new file mode 100644 index 0000000..5514d24 --- /dev/null +++ b/server/web/templates/pending_sensor.html @@ -0,0 +1,96 @@ + + + + + D4-Project + + + + + + + + + + + + + + + + + + + + + {% include 'navbar.html' %} + +
+
+
+
Approve New Sensor UUID
+ + +
+
+
+ +
+ + + + + + + + + + + {% for row_uuid in all_pending %} + + + + + + + {% endfor %} + +
UUIDdescriptionmail
+ + {{row_uuid['uuid']}} + + {{row_uuid['description']}}{{row_uuid['mail']}} + + + + + + +
+
+ + + {% include 'navfooter.html' %} + + +