mirror of https://github.com/D4-project/d4-core
chg: [server + UI + API] add server mode: registration + shared-secret + API-UI: approve/delete pending sensors
Terrtia
parent
336fc7655a
commit
3ce8557cff
|
|
@ -2,6 +2,7 @@
|
|||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import time
|
||||
import uuid
|
||||
import redis
|
||||
|
||||
|
|
@ -13,7 +14,8 @@ port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380))
|
|||
r_serv_db = redis.StrictRedis(
|
||||
host=host_redis_metadata,
|
||||
port=port_redis_metadata,
|
||||
db=0)
|
||||
db=0,
|
||||
decode_responses=True)
|
||||
|
||||
def is_valid_uuid_v4(UUID):
|
||||
UUID = UUID.replace('-', '')
|
||||
|
|
@ -23,6 +25,20 @@ def is_valid_uuid_v4(UUID):
|
|||
except:
|
||||
return False
|
||||
|
||||
def _get_sensor_metadata(sensor_uuid, first_seen=True, last_seen=True, mail=True, description=True):
|
||||
|
||||
meta_sensor = {}
|
||||
meta_sensor['uuid'] = sensor_uuid
|
||||
if first_seen:
|
||||
meta_sensor['first_seen'] = r_serv_db.hget('metadata_uuid:{}'.format(sensor_uuid), 'first_seen')
|
||||
if last_seen:
|
||||
meta_sensor['last_seen'] = r_serv_db.hget('metadata_uuid:{}'.format(sensor_uuid), 'last_seen')
|
||||
if description:
|
||||
meta_sensor['description'] = r_serv_db.hget('metadata_uuid:{}'.format(sensor_uuid), 'description')
|
||||
if mail:
|
||||
meta_sensor['mail'] = r_serv_db.hget('metadata_uuid:{}'.format(sensor_uuid), 'user_mail')
|
||||
return meta_sensor
|
||||
|
||||
## TODO: add description
|
||||
def register_sensor(req_dict):
|
||||
sensor_uuid = req_dict.get('uuid', None)
|
||||
|
|
@ -41,6 +57,8 @@ def register_sensor(req_dict):
|
|||
return ({"status": "error", "reason": "Mandatory parameter(s) not provided"}, 400)
|
||||
else:
|
||||
hmac_key = escape(hmac_key)
|
||||
if len(hmac_key)>100:
|
||||
hmac_key=hmac_key[:100]
|
||||
|
||||
|
||||
res = _register_sensor(sensor_uuid, hmac_key, user_id=user_id, description=None)
|
||||
|
|
@ -53,4 +71,43 @@ def _register_sensor(sensor_uuid, secret_key, user_id=None, description=None):
|
|||
r_serv_db.hset('metadata_uuid:{}'.format(sensor_uuid), 'user_mail', user_id)
|
||||
if description:
|
||||
r_serv_db.hset('metadata_uuid:{}'.format(sensor_uuid), 'description', description)
|
||||
r_serv_db.sadd('sensor_pending_registration', sensor_uuid)
|
||||
return ({'uuid': sensor_uuid}, 200)
|
||||
|
||||
def get_pending_sensor():
|
||||
return list(r_serv_db.smembers('sensor_pending_registration'))
|
||||
|
||||
def approve_sensor(req_dict):
|
||||
sensor_uuid = req_dict.get('uuid', None)
|
||||
if not is_valid_uuid_v4(sensor_uuid):
|
||||
return ({"status": "error", "reason": "Invalid uuid"}, 400)
|
||||
sensor_uuid = sensor_uuid.replace('-', '')
|
||||
# sensor not registred
|
||||
#if r_serv_db.sismember('sensor_pending_registration', sensor_uuid):
|
||||
# return ({"status": "error", "reason": "Sensor not registred"}, 404)
|
||||
# sensor already approved
|
||||
if r_serv_db.sismember('registered_uuid', sensor_uuid):
|
||||
return ({"status": "error", "reason": "Sensor already approved"}, 409)
|
||||
return _approve_sensor(sensor_uuid)
|
||||
|
||||
def _approve_sensor(sensor_uuid):
|
||||
r_serv_db.sadd('registered_uuid', sensor_uuid)
|
||||
r_serv_db.srem('sensor_pending_registration', sensor_uuid)
|
||||
return ({'uuid': sensor_uuid}, 200)
|
||||
|
||||
def delete_pending_sensor(req_dict):
|
||||
sensor_uuid = req_dict.get('uuid', None)
|
||||
if not is_valid_uuid_v4(sensor_uuid):
|
||||
return ({"status": "error", "reason": "Invalid uuid"}, 400)
|
||||
sensor_uuid = sensor_uuid.replace('-', '')
|
||||
# sensor not registred
|
||||
#if r_serv_db.sismember('sensor_pending_registration', sensor_uuid):
|
||||
# return ({"status": "error", "reason": "Sensor not registred"}, 404)
|
||||
# sensor already approved
|
||||
if not r_serv_db.sismember('sensor_pending_registration', sensor_uuid):
|
||||
return ({"status": "error", "reason": "Not Pending Sensor"}, 409)
|
||||
return _delete_pending_sensor(sensor_uuid)
|
||||
|
||||
def _delete_pending_sensor(sensor_uuid):
|
||||
r_serv_db.srem('sensor_pending_registration', sensor_uuid)
|
||||
return ({'uuid': sensor_uuid}, 200)
|
||||
|
|
|
|||
153
server/server.py
153
server/server.py
|
|
@ -26,6 +26,9 @@ hmac_key = os.getenv('D4_HMAC_KEY', b'private key to change')
|
|||
accepted_type = [1, 2, 4, 8, 254]
|
||||
accepted_extended_type = ['ja3-jl']
|
||||
|
||||
all_server_modes = ('registration', 'shared-secret')
|
||||
server_mode = 'registration'
|
||||
|
||||
timeout_time = 30
|
||||
|
||||
header_size = 62
|
||||
|
|
@ -39,6 +42,9 @@ port_redis_stream = int(os.getenv('D4_REDIS_STREAM_PORT', 6379))
|
|||
host_redis_metadata = os.getenv('D4_REDIS_METADATA_HOST', "localhost")
|
||||
port_redis_metadata = int(os.getenv('D4_REDIS_METADATA_PORT', 6380))
|
||||
|
||||
|
||||
### REDIS ###
|
||||
|
||||
redis_server_stream = redis.StrictRedis(
|
||||
host=host_redis_stream,
|
||||
port=port_redis_stream,
|
||||
|
|
@ -61,19 +67,111 @@ except redis.exceptions.ConnectionError:
|
|||
print('Error: Redis server {}:{}, ConnectionError'.format(host_redis_metadata, port_redis_metadata))
|
||||
sys.exit(1)
|
||||
|
||||
### REDIS ###
|
||||
|
||||
# set hmac default key
|
||||
redis_server_metadata.set('server:hmac_default_key', hmac_key)
|
||||
|
||||
# init redis_server_metadata
|
||||
redis_server_metadata.delete('server:accepted_type')
|
||||
for type in accepted_type:
|
||||
redis_server_metadata.sadd('server:accepted_type', type)
|
||||
redis_server_metadata.delete('server:accepted_extended_type')
|
||||
for type in accepted_extended_type:
|
||||
redis_server_metadata.sadd('server:accepted_extended_type', type)
|
||||
|
||||
dict_all_connection = {}
|
||||
|
||||
### FUNCTIONS ###
|
||||
|
||||
# kick sensors
|
||||
def kick_sensors():
|
||||
for client_uuid in redis_server_stream.smembers('server:sensor_to_kick'):
|
||||
client_uuid = client_uuid.decode()
|
||||
for session_uuid in redis_server_stream.smembers('map:active_connection-uuid-session_uuid:{}'.format(client_uuid)):
|
||||
session_uuid = session_uuid.decode()
|
||||
logger.warning('Sensor kicked uuid={}, session_uuid={}'.format(client_uuid, session_uuid))
|
||||
redis_server_stream.set('temp_blacklist_uuid:{}'.format(client_uuid), 'some random string')
|
||||
redis_server_stream.expire('temp_blacklist_uuid:{}'.format(client_uuid), 30)
|
||||
dict_all_connection[session_uuid].transport.abortConnection()
|
||||
redis_server_stream.srem('server:sensor_to_kick', client_uuid)
|
||||
|
||||
# Unpack D4 Header
|
||||
#def unpack_header(data):
|
||||
# data_header = {}
|
||||
# if len(data) >= header_size:
|
||||
# data_header['version'] = struct.unpack('B', data[0:1])[0]
|
||||
# data_header['type'] = struct.unpack('B', data[1:2])[0]
|
||||
# data_header['uuid_header'] = data[2:18].hex()
|
||||
# data_header['timestamp'] = struct.unpack('Q', data[18:26])[0]
|
||||
# data_header['hmac_header'] = data[26:58]
|
||||
# data_header['size'] = struct.unpack('I', data[58:62])[0]
|
||||
# return data_header
|
||||
|
||||
def is_valid_uuid_v4(header_uuid):
|
||||
try:
|
||||
uuid_test = uuid.UUID(hex=header_uuid, version=4)
|
||||
return uuid_test.hex == header_uuid
|
||||
except:
|
||||
logger.info('Not UUID v4: uuid={}, session_uuid={}'.format(header_uuid, self.session_uuid))
|
||||
return False
|
||||
|
||||
# # TODO: check timestamp
|
||||
def is_valid_header(uuid_to_check, type):
|
||||
if is_valid_uuid_v4(uuid_to_check):
|
||||
if redis_server_metadata.sismember('server:accepted_type', type):
|
||||
return True
|
||||
else:
|
||||
logger.warning('Invalid type, the server don\'t accept this type: {}, uuid={}, session_uuid={}'.format(type, uuid_to_check, self.session_uuid))
|
||||
return False
|
||||
else:
|
||||
logger.info('Invalid Header, uuid={}, session_uuid={}'.format(uuid_to_check, self.session_uuid))
|
||||
return False
|
||||
|
||||
def extract_ip(ip_string):
|
||||
#remove interface
|
||||
ip_string = ip_string.split('%')[0]
|
||||
# IPv4
|
||||
#extract ipv4
|
||||
if '.' in ip_string:
|
||||
return ip_string.split(':')[-1]
|
||||
# IPv6
|
||||
else:
|
||||
return ip_string
|
||||
|
||||
def server_mode_registration(header_uuid):
|
||||
# only accept registred uuid
|
||||
if server_mode == 'registration':
|
||||
if not redis_server_metadata.sismember('registered_uuid', header_uuid):
|
||||
error_msg = 'Not registred UUID={}, connection closed'.format(header_uuid)
|
||||
print(error_msg)
|
||||
logger.warning(error_msg)
|
||||
#redis_server_metadata.hset('metadata_uuid:{}'.format(data_header['uuid_header']), 'Error', 'Error: This UUID is temporarily blacklisted')
|
||||
return False
|
||||
else:
|
||||
return True
|
||||
else:
|
||||
return True
|
||||
|
||||
def is_client_ip_blacklisted():
|
||||
pass
|
||||
|
||||
def is_uuid_blacklisted(uuid):
|
||||
return redis_server_metadata.sismember('blacklist_uuid', data_header['uuid_header'])
|
||||
|
||||
|
||||
# return True if not blocked
|
||||
# False if blacklisted
|
||||
def check_blacklist():
|
||||
pass
|
||||
|
||||
# Kill Connection + create log
|
||||
#def manual_abort_connection(self, message, log_level='WARNING'):
|
||||
# logger.log(message)
|
||||
# self.transport.abortConnection()
|
||||
# return 1
|
||||
|
||||
### ###
|
||||
|
||||
|
||||
class D4_Server(Protocol, TimeoutMixin):
|
||||
|
||||
def __init__(self):
|
||||
|
|
@ -96,20 +194,12 @@ class D4_Server(Protocol, TimeoutMixin):
|
|||
|
||||
def dataReceived(self, data):
|
||||
# check and kick sensor by uuid
|
||||
for client_uuid in redis_server_stream.smembers('server:sensor_to_kick'):
|
||||
client_uuid = client_uuid.decode()
|
||||
for session_uuid in redis_server_stream.smembers('map:active_connection-uuid-session_uuid:{}'.format(client_uuid)):
|
||||
session_uuid = session_uuid.decode()
|
||||
logger.warning('Sensor kicked uuid={}, session_uuid={}'.format(client_uuid, session_uuid))
|
||||
redis_server_stream.set('temp_blacklist_uuid:{}'.format(client_uuid), 'some random string')
|
||||
redis_server_stream.expire('temp_blacklist_uuid:{}'.format(client_uuid), 30)
|
||||
dict_all_connection[session_uuid].transport.abortConnection()
|
||||
redis_server_stream.srem('server:sensor_to_kick', client_uuid)
|
||||
kick_sensors()
|
||||
|
||||
self.resetTimeout()
|
||||
if self.first_connection or self.ip is None:
|
||||
client_info = self.transport.client
|
||||
self.ip = self.extract_ip(client_info[0])
|
||||
self.ip = extract_ip(client_info[0])
|
||||
self.source_port = client_info[1]
|
||||
logger.debug('New connection, ip={}, port={} session_uuid={}'.format(self.ip, self.source_port, self.session_uuid))
|
||||
# check blacklisted_ip
|
||||
|
|
@ -171,37 +261,7 @@ class D4_Server(Protocol, TimeoutMixin):
|
|||
data_header['timestamp'] = struct.unpack('Q', data[18:26])[0]
|
||||
data_header['hmac_header'] = data[26:58]
|
||||
data_header['size'] = struct.unpack('I', data[58:62])[0]
|
||||
return data_header
|
||||
|
||||
def extract_ip(self, ip_string):
|
||||
#remove interface
|
||||
ip_string = ip_string.split('%')[0]
|
||||
# IPv4
|
||||
#extract ipv4
|
||||
if '.' in ip_string:
|
||||
return ip_string.split(':')[-1]
|
||||
# IPv6
|
||||
else:
|
||||
return ip_string
|
||||
|
||||
def is_valid_uuid_v4(self, header_uuid):
|
||||
try:
|
||||
uuid_test = uuid.UUID(hex=header_uuid, version=4)
|
||||
return uuid_test.hex == header_uuid
|
||||
except:
|
||||
logger.info('Not UUID v4: uuid={}, session_uuid={}'.format(header_uuid, self.session_uuid))
|
||||
return False
|
||||
|
||||
# # TODO: check timestamp
|
||||
def is_valid_header(self, uuid_to_check, type):
|
||||
if self.is_valid_uuid_v4(uuid_to_check):
|
||||
if redis_server_metadata.sismember('server:accepted_type', type):
|
||||
return True
|
||||
else:
|
||||
logger.warning('Invalid type, the server don\'t accept this type: {}, uuid={}, session_uuid={}'.format(type, uuid_to_check, self.session_uuid))
|
||||
else:
|
||||
logger.info('Invalid Header, uuid={}, session_uuid={}'.format(uuid_to_check, self.session_uuid))
|
||||
return False
|
||||
return data_header
|
||||
|
||||
def check_connection_validity(self, data_header):
|
||||
# blacklist ip by uuid
|
||||
|
|
@ -217,6 +277,11 @@ class D4_Server(Protocol, TimeoutMixin):
|
|||
self.transport.abortConnection()
|
||||
return False
|
||||
|
||||
# Check server mode
|
||||
if not server_mode_registration(data_header['uuid_header']):
|
||||
self.transport.abortConnection()
|
||||
return False
|
||||
|
||||
# check temp blacklist
|
||||
if redis_server_stream.exists('temp_blacklist_uuid:{}'.format(data_header['uuid_header'])):
|
||||
logger.warning('Temporarily Blacklisted UUID={}, connection closed'.format(data_header['uuid_header']))
|
||||
|
|
@ -246,7 +311,7 @@ class D4_Server(Protocol, TimeoutMixin):
|
|||
if data_header:
|
||||
if not self.check_connection_validity(data_header):
|
||||
return 1
|
||||
if self.is_valid_header(data_header['uuid_header'], data_header['type']):
|
||||
if is_valid_header(data_header['uuid_header'], data_header['type']):
|
||||
|
||||
# auto kill connection # TODO: map type
|
||||
if self.first_connection:
|
||||
|
|
|
|||
|
|
@ -17,17 +17,18 @@ import configparser
|
|||
|
||||
import subprocess
|
||||
|
||||
from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response
|
||||
from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, escape
|
||||
from flask_login import LoginManager, current_user, login_user, logout_user, login_required
|
||||
|
||||
import bcrypt
|
||||
|
||||
# Import Role_Manager
|
||||
from Role_Manager import create_user_db, check_password_strength, check_user_role_integrity
|
||||
from Role_Manager import login_user_basic
|
||||
from Role_Manager import login_user_basic, login_admin
|
||||
|
||||
sys.path.append(os.path.join(os.environ['D4_HOME'], 'lib'))
|
||||
from User import User
|
||||
import Sensor
|
||||
|
||||
# Import Blueprint
|
||||
from blueprints.restApi import restApi
|
||||
|
|
@ -709,6 +710,38 @@ def blacklisted_uuid():
|
|||
unblacklisted_uuid=unblacklisted_uuid, blacklisted_uuid=blacklisted_uuid)
|
||||
|
||||
|
||||
@app.route('/server/pending_sensor')
|
||||
@login_required
|
||||
@login_admin
|
||||
def pending_sensors():
|
||||
sensors = Sensor.get_pending_sensor()
|
||||
all_pending = []
|
||||
for sensor_uuid in sensors:
|
||||
all_pending.append(Sensor._get_sensor_metadata(sensor_uuid, first_seen=False, last_seen=False))
|
||||
return render_template("pending_sensor.html", all_pending=all_pending)
|
||||
|
||||
@app.route('/server/approve_sensor')
|
||||
@login_required
|
||||
@login_admin
|
||||
def approve_sensor():
|
||||
uuid_sensor = request.args.get('uuid')
|
||||
res = Sensor.approve_sensor({'uuid': uuid_sensor})
|
||||
if res[1] == 200:
|
||||
return redirect(url_for('pending_sensors'))
|
||||
else:
|
||||
return jsonify(res[0])
|
||||
|
||||
@app.route('/server/delete_pending_sensor')
|
||||
@login_required
|
||||
@login_admin
|
||||
def delete_pending_sensor():
|
||||
uuid_sensor = request.args.get('uuid')
|
||||
res = Sensor.delete_pending_sensor({'uuid': uuid_sensor})
|
||||
if res[1] == 200:
|
||||
return redirect(url_for('pending_sensors'))
|
||||
else:
|
||||
return jsonify(res[0])
|
||||
|
||||
@app.route('/uuid_change_stream_max_size')
|
||||
@login_required
|
||||
@login_user_basic
|
||||
|
|
@ -1042,7 +1075,10 @@ def set_uuid_hmac_key():
|
|||
uuid_sensor = request.args.get('uuid')
|
||||
user = request.args.get('redirect')
|
||||
key = request.args.get('key')
|
||||
redis_server_metadata.hset('metadata_uuid:{}'.format(uuid_sensor), 'hmac_key', key)
|
||||
hmac_key = escape(key)
|
||||
if len(hmac_key)>100:
|
||||
hmac_key=hmac_key[:100]
|
||||
redis_server_metadata.hset('metadata_uuid:{}'.format(uuid_sensor), 'hmac_key', hmac_key)
|
||||
if user:
|
||||
return redirect(url_for('uuid_management', uuid=uuid_sensor))
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,96 @@
|
|||
<!DOCTYPE html>
|
||||
|
||||
<html>
|
||||
<head>
|
||||
<title>D4-Project</title>
|
||||
<link rel="icon" href="{{ url_for('static', filename='img/d4-logo.png')}}">
|
||||
<!-- Core CSS -->
|
||||
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
|
||||
|
||||
<!-- JS -->
|
||||
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/bootstrap.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
|
||||
|
||||
<style>
|
||||
.popover{
|
||||
max-width: 100%;
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<body>
|
||||
|
||||
{% include 'navbar.html' %}
|
||||
|
||||
<div class="d-flex justify-content-center">
|
||||
<div class="card border-dark mt-3 text-center" style="max-width: 30rem;">
|
||||
<div class="card-body text-dark">
|
||||
<h5 class="card-title">Approve New Sensor UUID</h5>
|
||||
<input class="form-control" type="text" id="uuid" value="" required>
|
||||
<button type="button" class="btn btn-outline-secondary mt-1" onclick="window.location.href ='{{ url_for('approve_sensor') }}?uuid='+$('#uuid').val();">Approve UUID</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="py-3 mx-2">
|
||||
<table class="table table-striped table-bordered table-hover text-center" id="myTable_1">
|
||||
<thead>
|
||||
<tr>
|
||||
<th class="bg-info text-white">UUID</th>
|
||||
<th class="bg-info text-white">description</th>
|
||||
<th class="bg-info text-white">mail</th>
|
||||
<th class="bg-info text-white"></th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for row_uuid in all_pending %}
|
||||
<tr data-trigger="hover" title="" data-content="test content" data-original-title="test title">
|
||||
<td>
|
||||
<a class="" href="{{ url_for('uuid_management') }}?uuid={{row_uuid['uuid']}}">
|
||||
{{row_uuid['uuid']}}
|
||||
</a>
|
||||
</td>
|
||||
<td>{{row_uuid['description']}}</td>
|
||||
<td>{{row_uuid['mail']}}</td>
|
||||
<td>
|
||||
<a href="{{ url_for('approve_sensor') }}?uuid={{row_uuid['uuid']}}">
|
||||
<button type="button" class="btn btn-outline-info"><i class="fa fa-plus"></i></button>
|
||||
</a>
|
||||
<a href="{{ url_for('delete_pending_sensor') }}?uuid={{row_uuid['uuid']}}">
|
||||
<button type="button" class="btn btn-outline-danger"><i class="fa fa-trash"></i></button>
|
||||
</a>
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
|
||||
{% include 'navfooter.html' %}
|
||||
</body>
|
||||
|
||||
<script>
|
||||
$(document).ready(function(){
|
||||
table = $('#myTable_1').DataTable(
|
||||
{
|
||||
"aLengthMenu": [[5, 10, 15, 20, -1], [5, 10, 15, 20, "All"]],
|
||||
"iDisplayLength": 10,
|
||||
"order": [[ 0, "asc" ]]
|
||||
}
|
||||
);
|
||||
$('[data-toggle="popover"]').popover({
|
||||
placement: 'top',
|
||||
container: 'body',
|
||||
html : false,
|
||||
})
|
||||
});
|
||||
|
||||
</script>
|
||||
Loading…
Reference in New Issue