Compare commits

..

No commits in common. "master" and "v1.1" have entirely different histories.
master ... v1.1

24 changed files with 59 additions and 627 deletions

View File

@ -64,31 +64,10 @@ git submodule init
git submodule update
~~~~
Build the d4 client. This will create the `d4` binary.
~~~~
make
~~~~
Then register the sensor with the server. Replace `API_TOKEN`, `VALID_UUID4` (create a random UUID via [UUIDgenerator](https://www.uuidgenerator.net/)) and `VALID_HMAC_KEY`.
~~~~
curl -k https://127.0.0.1:7000/api/v1/add/sensor/register --header "Authorization: API_TOKEN" -H "Content-Type: application/json" --data '{"uuid":"VALID_UUID4","hmac_key":"VALID_HMAC_KEY"}' -X POST
~~~~
If the registration went correctly the UUID is returned. Do not forget to approve the registration in the D4 server web interface.
Update the configuration file
~~~~
cp -r conf.sample conf
echo VALID_UUID4 > conf/uuid
echo VALID_HMAC_KEY > conf/key
~~~~
## D4 core server
D4 core server is a complete server to handle clients (sensors) including the decapsulation of the [D4 protocol](https://github.com/D4-project/architecture/tree/master/format), control of sensor registrations, management of decoding protocols and dispatching to adequate decoders/analysers.
D4 core server is a complete server to handle clients (sensors) including the decapsulation of the [D4 protocol](https://github.com/D4-project/architecture/tree/master/format), control of
sensor registrations, management of decoding protocols and dispatching to adequate decoders/analysers.
### Requirements
@ -97,6 +76,7 @@ D4 core server is a complete server to handle clients (sensors) including the de
### Installation
- [Install D4 Server](https://github.com/D4-project/d4-core/tree/master/server)
### D4 core server Screenshots

View File

@ -32,7 +32,7 @@ clean:
- rm -rf *.o hmac
d4: d4.o sha2.o hmac.o unpack.o unparse.o pack.o gen_uuid.o randutils.o parse.o
$(CC) -Wall -o d4 d4.o hmac.o sha2.o unpack.o pack.o unparse.o gen_uuid.o randutils.o parse.o
gcc -Wall -o d4 d4.o hmac.o sha2.o unpack.o pack.o unparse.o gen_uuid.o randutils.o parse.o
d4.o: d4.c
$(CC) -Wall -c d4.c
gcc -Wall -c d4.c

View File

@ -210,7 +210,7 @@ void d4_transfert(d4_t* d4)
//In case of errors see block of 0 bytes
bzero(buf, d4->snaplen);
nread = read(d4->source.fd, buf, d4->snaplen);
if ( nread >= 0 ) {
if ( nread > 0 ) {
d4_update_header(d4, nread);
//Do HMAC on header and payload. HMAC field is 0 during computation
if (d4->ctx) {
@ -238,11 +238,6 @@ void d4_transfert(d4_t* d4)
fprintf(stderr,"Incomplete header written. abort to let consumer known that the packet is corrupted\n");
abort();
}
// no data - create empty D4 packet
if ( nread == 0 ) {
//FIXME no data available, sleep, abort, retry
break;
}
} else{
//FIXME no data available, sleep, abort, retry
break;

View File

@ -69,8 +69,6 @@ function launching_d4_server {
screen -S "Server_D4" -X screen -t "Server_D4" bash -c "cd ${D4_HOME}; ./server.py -v 10; read x"
sleep 0.1
screen -S "Server_D4" -X screen -t "sensors_manager" bash -c "cd ${D4_HOME}; ./sensors_manager.py; read x"
sleep 0.1
}
function launching_workers {

View File

@ -15,24 +15,11 @@ sensor registrations, management of decoding protocols and dispatching to adequa
### Installation
###### Install D4 server
Clone the repository and install necessary packages. Installation requires *sudo* permissions.
~~~~
git clone https://github.com/D4-project/d4-core.git
cd d4-core
cd server
./install_server.sh
~~~~
When the installation is finished, scroll back to where `+ ./create_default_user.py` is displayed. The next lines contain the default generated user and should resemble the snippet below. Take a temporary note of the password, you are required to **change the password** on first login.
~~~~
new user created: admin@admin.test
password: <redacted>
token: <redacted>
~~~~
Then create or add a pem in [d4-core/server](https://github.com/D4-project/d4-core/tree/master/server) :
Create or add a pem in [d4-core/server](https://github.com/D4-project/d4-core/tree/master/server) :
~~~~
cd gen_cert
./gen_root.sh
@ -40,6 +27,7 @@ cd gen_cert
cd ..
~~~~
###### Launch D4 server
~~~~
./LAUNCH.sh -l
@ -47,14 +35,6 @@ cd ..
The web interface is accessible via `http://127.0.0.1:7000/`
If you cannot access the web interface on localhost (for example because the system is running on a remote host), then stop the server, change the listening host IP and restart the server. In the below example it's changed to `0.0.0.0` (all interfaces). Make sure that the IP is not unintentionally publicly exposed.
~~~~
./LAUNCH.sh -k
sed -i '/\[Flask_Server\]/{:a;N;/host = 127\.0\.0\.1/!ba;s/host = 127\.0\.0\.1/host = 0.0.0.0/}' configs/server.conf
./LAUNCH.sh -l
~~~~
### Updating web assets
To update javascript libs run:
~~~~
@ -66,6 +46,7 @@ cd web
[API Documentation](https://github.com/D4-project/d4-core/tree/master/server/documentation/README.md)
### Notes
- All server logs are located in ``d4-core/server/logs/``
@ -104,7 +85,3 @@ Run the following command as root:
~~~~
aa-complain /usr/sbin/tcpdump
~~~~
###### WARNING - Not registered UUID=UUID4, connection closed
This happens after you have registered a new sensor, but have not approved the registration. In order to approve the sensor, go in the web interface to **Server Management**, and click **Pending Sensors**.

View File

@ -4,15 +4,12 @@ use_default_save_directory = yes
save_directory = None
[D4_Server]
server_port=4443
# registration or shared-secret
server_mode = registration
default_hmac_key = private key to change
analyzer_queues_max_size = 100000000
analyzer_queues_max_size = 10000
[Flask_Server]
# UI port number
host = 127.0.0.1
port = 7000
[Redis_STREAM]

View File

@ -30,8 +30,8 @@ git checkout 5.0
make
popd
# LAUNCH
bash LAUNCH.sh -l &
# LAUNCH Redis
bash ${AIL_BIN}LAUNCH.sh -lrv &
wait
echo ""

View File

@ -11,7 +11,6 @@ from flask import escape
sys.path.append(os.path.join(os.environ['D4_HOME'], 'lib/'))
import ConfigLoader
import d4_server
### Config ###
config_loader = ConfigLoader.ConfigLoader()
@ -27,13 +26,6 @@ def is_valid_uuid_v4(UUID):
except:
return False
def get_time_sensor_last_seen(sensor_uuid):
res = r_serv_db.hget('metadata_uuid:{}'.format(sensor_uuid), 'last_seen')
if res:
return int(res)
else:
return 0
def _get_sensor_type(sensor_uuid, first_seen=True, last_seen=True, time_format='default'):
uuid_type = []
uuid_all_type = r_serv_db.smembers('all_types_by_uuid:{}'.format(sensor_uuid))
@ -76,8 +68,6 @@ def _get_sensor_metadata(sensor_uuid, first_seen=True, last_seen=True, time_form
meta_sensor['mail'] = r_serv_db.hget('metadata_uuid:{}'.format(sensor_uuid), 'user_mail')
return meta_sensor
### BEGIN - SENSOR REGISTRATION ###
## TODO: add description
def register_sensor(req_dict):
sensor_uuid = req_dict.get('uuid', None)
@ -90,7 +80,7 @@ def register_sensor(req_dict):
sensor_uuid = sensor_uuid.replace('-', '')
# sensor already exist
if r_serv_db.exists('metadata_uuid:{}'.format(sensor_uuid)):
return ({"status": "error", "reason": "Sensor already registered"}, 409)
return ({"status": "error", "reason": "Sensor already registred"}, 409)
# hmac key
if not hmac_key:
@ -177,99 +167,3 @@ def delete_registered_sensor(req_dict):
def _delete_registered_sensor(sensor_uuid):
r_serv_db.srem('registered_uuid', sensor_uuid)
return ({'uuid': sensor_uuid}, 200)
### --- END - SENSOR REGISTRATION --- ###
### BEGIN - SENSOR MONITORING ###
def get_sensors_monitoring_last_updated():
res = r_serv_db.get('sensors_monitoring:last_updated')
if res:
return int(res)
else:
return 0
def get_all_sensors_to_monitor():
return r_serv_db.smembers('to_monitor:sensors')
def get_to_monitor_delta_time_by_uuid(sensor_uuid):
return int(r_serv_db.hget('to_monitor:sensor:{}'.format(sensor_uuid), 'delta_time'))
def get_all_sensors_to_monitor_dict():
dict_to_monitor = {}
for sensor_uuid in get_all_sensors_to_monitor():
dict_to_monitor[sensor_uuid] = get_to_monitor_delta_time_by_uuid(sensor_uuid)
return dict_to_monitor
def _check_sensor_delta(sensor_uuid, sensor_delta):
last_d4_packet = get_time_sensor_last_seen(sensor_uuid)
# check sensor delta time between two D4 packets + check sensor connection
if int(time.time()) - last_d4_packet > sensor_delta or not d4_server.is_sensor_connected(sensor_uuid):
r_serv_db.sadd('sensors_monitoring:sensors_error', sensor_uuid)
handle_sensor_monitoring_error(sensor_uuid)
else:
r_serv_db.srem('sensors_monitoring:sensors_error', sensor_uuid)
def handle_sensor_monitoring_error(sensor_uuid):
print('sensor monitoring error: {}'.format(sensor_uuid))
## TODO: ##
# MAILS
# UI Notifications
# SNMP
# Syslog message
## ## ## ##
return None
def is_sensor_monitored(sensor_uuid):
return r_serv_db.sismember('to_monitor:sensors', sensor_uuid)
def get_all_sensors_connection_errors():
return r_serv_db.smembers('sensors_monitoring:sensors_error')
def api_get_all_sensors_connection_errors():
return list(get_all_sensors_connection_errors()), 200
def add_sensor_to_monitor(sensor_uuid, delta_time):
r_serv_db.sadd('to_monitor:sensors', sensor_uuid)
r_serv_db.hset('to_monitor:sensor:{}'.format(sensor_uuid), 'delta_time', delta_time)
r_serv_db.set('sensors_monitoring:last_updated', int(time.time()))
r_serv_db.srem('sensors_monitoring:sensors_error', sensor_uuid)
def delete_sensor_to_monitor(sensor_uuid):
r_serv_db.srem('to_monitor:sensors', sensor_uuid)
r_serv_db.delete('to_monitor:sensor:{}'.format(sensor_uuid))
r_serv_db.set('sensors_monitoring:last_updated', int(time.time()))
r_serv_db.srem('sensors_monitoring:sensors_error', sensor_uuid)
def api_add_sensor_to_monitor(data_dict):
sensor_uuid = data_dict.get('uuid', None)
delta_time = data_dict.get('delta_time', None)
if not is_valid_uuid_v4(sensor_uuid):
return ({"status": "error", "reason": "Invalid uuid"}, 400)
sensor_uuid = sensor_uuid.replace('-', '')
# hmac key
if not delta_time:
return ({"status": "error", "reason": "Mandatory parameter(s) not provided"}, 400)
else:
try:
delta_time = int(delta_time)
if delta_time < 1:
return ({"status": "error", "reason": "Invalid delta_time"}, 400)
except Exception:
return ({"status": "error", "reason": "Invalid delta_time"}, 400)
add_sensor_to_monitor(sensor_uuid, delta_time)
def api_delete_sensor_to_monitor(data_dict):
sensor_uuid = data_dict.get('uuid', None)
if not is_valid_uuid_v4(sensor_uuid):
return ({"status": "error", "reason": "Invalid uuid"}, 400)
sensor_uuid = sensor_uuid.replace('-', '')
if not is_sensor_monitored(sensor_uuid):
return ({"status": "error", "reason": "Sensor not monitored"}, 400)
delete_sensor_to_monitor(sensor_uuid)
### --- END - SENSOR REGISTRATION --- ###

View File

@ -1,44 +0,0 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
import os
import sys
import time
import uuid
import redis
from flask import escape
sys.path.append(os.path.join(os.environ['D4_HOME'], 'lib/'))
import ConfigLoader
### Config ###
config_loader = ConfigLoader.ConfigLoader()
r_stream = config_loader.get_redis_conn("Redis_STREAM")
config_loader = None
### ###
### BEGIN - SENSOR CONNECTION ###
def get_all_connected_sensors(r_list=False):
res = r_stream.smembers('active_connection')
if r_list:
if res:
return list(res)
else:
return []
else:
return res
def get_all_connected_sensors_by_type(d4_type, d4_extended_type=None):
# D4 extended type
if d4_type == 254 and d4_extended_type:
return r_stream.smembers('active_connection_extended_type:{}'.format(d4_extended_type))
# type 1-253
else:
return r_stream.smembers('active_connection:{}'.format(d4_type))
def is_sensor_connected(sensor_uuid):
return r_stream.sismember('active_connection', sensor_uuid)
### --- END - SENSOR CONNECTION --- ###

View File

@ -1,9 +1,8 @@
twisted[tls]
redis
flask==2.2.2
flask
flask-login
bcrypt
Werkzeug==2.2.2
#sudo python3 -m pip install --upgrade service_identity

View File

@ -1,54 +0,0 @@
#!/usr/bin/env python3
import os
import sys
import time
import redis
sys.path.append(os.path.join(os.environ['D4_HOME'], 'lib/'))
import ConfigLoader
import Sensor
### Config ###
config_loader = ConfigLoader.ConfigLoader()
#redis_server_stream = config_loader.get_redis_conn("Redis_STREAM", decode_responses=False)
redis_server_metadata = config_loader.get_redis_conn("Redis_METADATA")
config_loader = None
### ###
try:
redis_server_metadata.ping()
except redis.exceptions.ConnectionError:
print('Error: Redis server: Redis_METADATA, ConnectionError')
sys.exit(1)
def reload_all_sensors_to_monitor_dict(dict_to_monitor, last_updated):
if not dict_to_monitor:
dict_to_monitor = Sensor.get_all_sensors_to_monitor_dict()
else:
monitoring_last_updated = Sensor.get_sensors_monitoring_last_updated()
if monitoring_last_updated > last_updated:
dict_to_monitor = Sensor.get_all_sensors_to_monitor_dict()
last_updated = int(time.time())
print('updated: List of sensors to monitor')
return dict_to_monitor
if __name__ == "__main__":
time_refresh = int(time.time())
last_updated = time_refresh
all_sensors_to_monitor = Sensor.get_all_sensors_to_monitor_dict()
while True:
for sensor_uuid in all_sensors_to_monitor:
Sensor._check_sensor_delta(sensor_uuid, all_sensors_to_monitor[sensor_uuid])
time.sleep(10)
## reload dict_to_monitor ##
curr_time = int(time.time())
if curr_time - time_refresh >= 60:
time_refresh = curr_time
all_sensors_to_monitor = reload_all_sensors_to_monitor_dict(all_sensors_to_monitor, last_updated)
##-- --##

View File

@ -13,8 +13,6 @@ import argparse
import logging
import logging.handlers
import configparser
from twisted.internet import ssl, task, protocol, endpoints, defer
from twisted.python import log
from twisted.python.modules import getModule
@ -26,6 +24,7 @@ sys.path.append(os.path.join(os.environ['D4_HOME'], 'lib/'))
import ConfigLoader
hmac_reset = bytearray(32)
hmac_key = os.getenv('D4_HMAC_KEY', b'private key to change')
accepted_type = [1, 2, 4, 8, 254]
accepted_extended_type = ['ja3-jl']
@ -47,16 +46,7 @@ redis_server_stream = config_loader.get_redis_conn("Redis_STREAM", decode_respon
redis_server_metadata = config_loader.get_redis_conn("Redis_METADATA", decode_responses=False)
# get server_mode
try:
D4server_port = config_loader.get_config_int("D4_Server", "server_port")
except configparser.NoOptionError:
D4server_port = 4443
server_mode = config_loader.get_config_str("D4_Server", "server_mode")
try:
hmac_key = config_loader.get_config_str("D4_Server", "default_hmac_key")
except configparser.NoOptionError:
hmac_key = 'private key to change'
config_loader = None
### ###
@ -144,10 +134,10 @@ def extract_ip(ip_string):
return ip_string
def server_mode_registration(header_uuid):
# only accept registered uuid
# only accept registred uuid
if server_mode == 'registration':
if not redis_server_metadata.sismember('registered_uuid', header_uuid):
error_msg = 'Not registered UUID={}, connection closed'.format(header_uuid)
error_msg = 'Not registred UUID={}, connection closed'.format(header_uuid)
print(error_msg)
logger.warning(error_msg)
#redis_server_metadata.hset('metadata_uuid:{}'.format(data_header['uuid_header']), 'Error', 'Error: This UUID is temporarily blacklisted')
@ -255,8 +245,8 @@ class D4_Server(Protocol, TimeoutMixin):
if not redis_server_stream.exists('active_connection_by_uuid:{}'.format(self.uuid)):
redis_server_stream.srem('active_connection', self.uuid)
logger.debug('Connection closed: session_uuid={}'.format(self.session_uuid))
dict_all_connection.pop(self.session_uuid)
logger.debug('Connection closed: session_uuid={}'.format(self.session_uuid))
dict_all_connection.pop(self.session_uuid)
def unpack_header(self, data):
data_header = {}
@ -269,20 +259,6 @@ class D4_Server(Protocol, TimeoutMixin):
data_header['size'] = struct.unpack('I', data[58:62])[0]
return data_header
def check_hmac_key(self, hmac_header, data):
if self.hmac_key is None:
self.hmac_key = redis_server_metadata.hget('metadata_uuid:{}'.format(self.uuid), 'hmac_key')
if self.hmac_key is None:
self.hmac_key = redis_server_metadata.get('server:hmac_default_key')
# set hmac_header to 0
data = data.replace(hmac_header, hmac_reset, 1)
HMAC = hmac.new(self.hmac_key, msg=data, digestmod='sha256')
hmac_header = hmac_header.hex()
# hmac match
return hmac_header == HMAC.hexdigest()
def check_connection_validity(self, data_header):
# blacklist ip by uuid
if redis_server_metadata.sismember('blacklist_ip_by_uuid', data_header['uuid_header']):
@ -359,14 +335,8 @@ class D4_Server(Protocol, TimeoutMixin):
self.type = data_header['type']
self.uuid = data_header['uuid_header']
# # check HMAC /!\ incomplete data
# if not self.check_hmac_key(data_header['hmac_header'], data):
# print('hmac do not match')
# print(data)
# logger.debug("HMAC don't match, uuid={}, session_uuid={}".format(self.uuid, self.session_uuid))
# redis_server_metadata.hset('metadata_uuid:{}'.format(data_header['uuid_header']), 'Error', 'Error: HMAC don\'t match')
# self.transport.abortConnection()
# return 1
# worker entry point: map type:session_uuid
redis_server_stream.sadd('session_uuid:{}'.format(data_header['type']), self.session_uuid.encode())
## save active connection ##
#active Connection
@ -493,6 +463,15 @@ class D4_Server(Protocol, TimeoutMixin):
def process_d4_data(self, data, data_header, ip):
# empty buffer
self.buffer = b''
# set hmac_header to 0
data = data.replace(data_header['hmac_header'], hmac_reset, 1)
if self.hmac_key is None:
self.hmac_key = redis_server_metadata.hget('metadata_uuid:{}'.format(data_header['uuid_header']), 'hmac_key')
if self.hmac_key is None:
self.hmac_key = redis_server_metadata.get('server:hmac_default_key')
HMAC = hmac.new(self.hmac_key, msg=data, digestmod='sha256')
data_header['hmac_header'] = data_header['hmac_header'].hex()
### Debug ###
#print('hexdigest: {}'.format( HMAC.hexdigest() ))
@ -505,7 +484,7 @@ class D4_Server(Protocol, TimeoutMixin):
### ###
# hmac match
if self.check_hmac_key(data_header['hmac_header'], data):
if data_header['hmac_header'] == HMAC.hexdigest():
if not self.stream_max_size:
temp = redis_server_metadata.hget('stream_max_size_by_uuid', data_header['uuid_header'])
if temp is not None:
@ -530,16 +509,12 @@ class D4_Server(Protocol, TimeoutMixin):
redis_server_metadata.zincrby('stat_uuid_type:{}:{}'.format(date, data_header['uuid_header']), 1, data_header['type'])
#
d4_packet_rcv_time = int(time.time())
if not redis_server_metadata.hexists('metadata_uuid:{}'.format(data_header['uuid_header']), 'first_seen'):
redis_server_metadata.hset('metadata_uuid:{}'.format(data_header['uuid_header']), 'first_seen', d4_packet_rcv_time)
redis_server_metadata.hset('metadata_uuid:{}'.format(data_header['uuid_header']), 'last_seen', d4_packet_rcv_time)
redis_server_metadata.hset('metadata_type_by_uuid:{}:{}'.format(data_header['uuid_header'], data_header['type']), 'last_seen', d4_packet_rcv_time)
redis_server_metadata.hset('metadata_uuid:{}'.format(data_header['uuid_header']), 'first_seen', data_header['timestamp'])
redis_server_metadata.hset('metadata_uuid:{}'.format(data_header['uuid_header']), 'last_seen', data_header['timestamp'])
redis_server_metadata.hset('metadata_type_by_uuid:{}:{}'.format(data_header['uuid_header'], data_header['type']), 'last_seen', data_header['timestamp'])
if not self.data_saved:
# worker entry point: map type:session_uuid
redis_server_stream.sadd('session_uuid:{}'.format(data_header['type']), self.session_uuid.encode())
#UUID IP: ## TODO: use d4 timestamp ?
redis_server_metadata.lpush('list_uuid_ip:{}'.format(data_header['uuid_header']), '{}-{}'.format(ip, datetime.datetime.now().strftime("%Y%m%d%H%M%S")))
redis_server_metadata.ltrim('list_uuid_ip:{}'.format(data_header['uuid_header']), 0, 15)
@ -548,7 +523,7 @@ class D4_Server(Protocol, TimeoutMixin):
if self.update_stream_type:
if not redis_server_metadata.hexists('metadata_type_by_uuid:{}:{}'.format(data_header['uuid_header'], data_header['type']), 'first_seen'):
redis_server_metadata.hset('metadata_type_by_uuid:{}:{}'.format(data_header['uuid_header'], data_header['type']), 'first_seen', d4_packet_rcv_time)
redis_server_metadata.hset('metadata_type_by_uuid:{}:{}'.format(data_header['uuid_header'], data_header['type']), 'first_seen', data_header['timestamp'])
self.update_stream_type = False
return 0
else:
@ -579,7 +554,7 @@ def main(reactor):
certificate = ssl.PrivateCertificate.loadPEM(certData)
factory = protocol.Factory.forProtocol(D4_Server)
# use interface to support both IPv4 and IPv6
reactor.listenSSL(D4server_port, factory, certificate.options(), interface='::')
reactor.listenSSL(4443, factory, certificate.options(), interface='::')
return defer.Deferred()

View File

@ -35,7 +35,6 @@ import Analyzer_Queue
from blueprints.restApi import restApi
from blueprints.settings import settings
from blueprints.analyzer_queue import analyzer_queue
from blueprints.D4_sensors import D4_sensors
baseUrl = ''
if baseUrl != '':
@ -65,12 +64,6 @@ server_mode = config_loader.get_config_str("D4_Server", "server_mode")
if server_mode not in all_server_modes:
print('Error: incorrect server_mode')
try:
FLASK_HOST = config_loader.get_config_str("Flask_Server", "host")
except Exception as e:
print(e)
FLASK_HOST = '127.0.0.1'
try:
FLASK_PORT = config_loader.get_config_int("Flask_Server", "port")
except Exception:
@ -116,7 +109,6 @@ login_manager.init_app(app)
app.register_blueprint(restApi)
app.register_blueprint(settings)
app.register_blueprint(analyzer_queue)
app.register_blueprint(D4_sensors)
# ========= =========#
# ========= LOGIN MANAGER ========
@ -284,7 +276,7 @@ def login():
if request.method == 'POST':
username = request.form.get('username')
password = request.form.get('password')
next_page = request.form.get('next_page')
#next_page = request.form.get('next_page')
if username is not None:
user = User.get(username)
@ -301,16 +293,11 @@ def login():
#if not check_user_role_integrity(user.get_id()):
# error = 'Incorrect User ACL, Please contact your administrator'
# return render_template("login.html", error=error)
if not user.is_in_role('user'):
return render_template("403.html"), 403
login_user(user) ## TODO: use remember me ?
if user.request_password_change():
return redirect(url_for('change_password'))
else:
if next_page and next_page!='None':
return redirect(next_page)
else:
return redirect(url_for('index'))
return redirect(url_for('index'))
# login failed
else:
# set brute force protection
@ -326,13 +313,12 @@ def login():
return 'please provide a valid username'
else:
next_page = request.args.get('next')
#next_page = request.args.get('next')
error = request.args.get('error')
return render_template("login.html" , error=error, next_page=next_page)
return render_template("login.html" , error=error)
@app.route('/change_password', methods=['POST', 'GET'])
@login_required
@login_user_basic
def change_password():
password1 = request.form.get('password1')
password2 = request.form.get('password2')
@ -367,7 +353,7 @@ def logout():
@app.route('/role', methods=['POST', 'GET'])
@login_required
def role():
return render_template("403.html"), 403
return render_template("error/403.html"), 403
@app.route('/')
@login_required
@ -602,8 +588,6 @@ def uuid_management():
"blacklisted_uuid": blacklisted_uuid, "blacklisted_ip_by_uuid": blacklisted_ip_by_uuid,
"first_seen_gmt": first_seen_gmt, "last_seen_gmt": last_seen_gmt, "Error": Error}
data_uuid['is_monitored'] = Sensor.is_sensor_monitored(uuid_sensor)
if redis_server_stream.sismember('active_connection', uuid_sensor):
active_connection = True
else:
@ -1150,4 +1134,4 @@ def get_uuid_stats_history_json():
if __name__ == "__main__":
app.run(host=FLASK_HOST, port=FLASK_PORT, threaded=True, ssl_context=ssl_context)
app.run(host='0.0.0.0', port=FLASK_PORT, threaded=True, ssl_context=ssl_context)

View File

@ -1,76 +0,0 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
'''
Flask functions and routes for all D4 sensors
'''
import os
import re
import sys
import json
import redis
sys.path.append(os.path.join(os.environ['D4_HOME'], 'lib'))
import ConfigLoader
import Sensor
from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response
from flask_login import login_required, current_user
from Role_Manager import login_admin, login_user_basic
# ============ BLUEPRINT ============
D4_sensors = Blueprint('D4_sensors', __name__, template_folder='templates')
# ============ VARIABLES ============
### Config ###
config_loader = ConfigLoader.ConfigLoader()
r_serv_metadata = config_loader.get_redis_conn("Redis_METADATA")
r_serv_db = config_loader.get_redis_conn("Redis_SERV")
config_loader = None
### ###
# ============ FUNCTIONS ============
# ============= ROUTES ==============
@D4_sensors.route("/sensors/monitoring/add", methods=['GET'])
@login_required
@login_user_basic
def add_sensor_to_monitor():
sensor_uuid = request.args.get("uuid")
return render_template("sensors/add_sensor_to_monitor.html",
sensor_uuid=sensor_uuid)
@D4_sensors.route("/sensors/monitoring/add_post", methods=['POST'])
@login_required
@login_user_basic
def add_sensor_to_monitor_post():
sensor_uuid = request.form.get("uuid")
delta_time = request.form.get("delta_time")
res = Sensor.api_add_sensor_to_monitor({'uuid':sensor_uuid, 'delta_time': delta_time})
if res:
Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
return redirect(url_for('uuid_management', uuid=sensor_uuid))
@D4_sensors.route("/sensors/monitoring/delete", methods=['GET'])
@login_required
@login_user_basic
def delete_sensor_to_monitor():
sensor_uuid = request.args.get("uuid")
res = Sensor.api_delete_sensor_to_monitor({'uuid':sensor_uuid})
if res:
Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
return redirect(url_for('uuid_management', uuid=sensor_uuid))
@D4_sensors.route("/sensors/monitoring/errors", methods=['GET'])
@login_required
@login_user_basic
def get_all_sensors_connection_errors():
res = Sensor.api_get_all_sensors_connection_errors()
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]

View File

@ -18,6 +18,7 @@ from flask import Flask, render_template, jsonify, request, Blueprint, redirect,
from flask_login import login_required, current_user
from Role_Manager import login_admin, login_user_basic
from Role_Manager import create_user_db, edit_user_db, delete_user_db, check_password_strength, generate_new_token, gen_password, get_all_role
# ============ BLUEPRINT ============

View File

@ -142,8 +142,8 @@ def is_valid_uuid_v4(header_uuid):
except:
return False
def build_json_response(resp_data, resp_code):
return Response(json.dumps(resp_data, indent=2, sort_keys=True), mimetype='application/json'), resp_code
def one():
return 1
# ============= ROUTES ==============
@ -154,9 +154,3 @@ def add_sensor_register():
data = request.get_json()
res = Sensor.register_sensor(data)
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
@restApi.route("/api/v1/sensors/monitoring/errors", methods=['GET'])
@token_required('user')
def get_all_sensors_connection_errors():
res = Sensor.api_get_all_sensors_connection_errors()
return build_json_response(res[0], res[1])

View File

@ -69,8 +69,7 @@
<form class="form-signin" action="{{ url_for('login')}}" method="post">
<img class="mb-4" src="{{ url_for('static', filename='img/d4-logo.png')}}" width="300">
<h1 class="h3 mb-3 text-secondary">Please sign in</h1>
<input type="text" id="next_page" name="next_page" value="{{next_page}}" hidden>
<label for="inputEmail" class="sr-only">Email address</label>
<label for="inputEmail" class="sr-only">Email address</label>
<input type="email" id="inputEmail" name="username" class="form-control" placeholder="Email address" required autofocus>
<label for="inputPassword" class="sr-only">Password</label>
<input type="password" id="inputPassword" name="password" class="form-control {% if error %}is-invalid{% endif %}" placeholder="Password" required>

View File

@ -1,57 +0,0 @@
<!DOCTYPE html>
<html>
<head>
<title>D4-Project</title>
<link rel="icon" href="{{ url_for('static', filename='img/d4-logo.png')}}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
<!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap.min.js')}}"></script>
</head>
<body>
{% include 'navbar.html' %}
<form action="{{ url_for('D4_sensors.add_sensor_to_monitor_post') }}" method="post" enctype=multipart/form-data>
<div class="d-flex justify-content-center">
<div class="col-sm-6">
<h4 class="my-3">Monitor a Sensor</h4>
<div class="form-group">
<input class="form-control text-center bg-dark text-white" type="text" value="{{sensor_uuid}}" disabled>
<input type="text" name="uuid" id="uuid" value="{{sensor_uuid}}" hidden>
</div>
<div class="input-group mt-2 mb-2">
<div class="input-group-prepend">
<span class="input-group-text bg-light"><i class="fa fa-clock-o"></i>&nbsp;</span>
</div>
<input class="form-control" type="number" id="delta_time" value="3600" min="30" name="delta_time" required>
<div class="input-group-append">
<span class="input-group-text">Maxinum Time (seconds) between two D4 packets</span>
</div>
</div>
<div class="form-group">
<button class="btn btn-primary" type="submit">Monitor Sensor</button>
</div>
</div>
</div>
</form>
{% include 'navfooter.html' %}
</body>
<script>
$(document).ready(function(){
$("#nav-sensor").addClass("active");
});
</script>

View File

@ -101,18 +101,6 @@
</div>
</div>
<div class="d-flex justify-content-center mt-2">
{% if not data_uuid.get('is_monitored', False) %}
<a href="{{ url_for('D4_sensors.add_sensor_to_monitor') }}?uuid={{uuid_sensor}}">
<button type="button" class="btn btn-primary">Monitor Sensor</button>
</a>
{% else %}
<a href="{{ url_for('D4_sensors.delete_sensor_to_monitor') }}?uuid={{uuid_sensor}}">
<button type="button" class="btn btn-danger">Remove Sensor from monitoring</button>
</a>
{% endif %}
</div>
<div class="card-deck justify-content-center ml-0 mr-0">
<div class="card border-dark mt-3" style="max-width: 18rem;">
<div class="card-body text-dark">

View File

@ -47,8 +47,8 @@ mv temp/bootstrap-${BOOTSTRAP_VERSION}-dist/js/bootstrap.min.js ./static/js/
mv temp/bootstrap-${BOOTSTRAP_VERSION}-dist/css/bootstrap.min.css ./static/css/
mv temp/bootstrap-${BOOTSTRAP_VERSION}-dist/css/bootstrap.min.css.map ./static/css/
mv temp/floating-ui-1.14.3/dist/umd/popper.min.js ./static/js/
mv temp/floating-ui-1.14.3/dist/umd/popper.min.js.map ./static/js/
mv temp/popper-core-1.14.3/dist/umd/popper.min.js ./static/js/
mv temp/popper-core-1.14.3/dist/umd/popper.min.js.map ./static/js/
mv temp/Font-Awesome-${FONT_AWESOME_VERSION} temp/font-awesome

View File

@ -85,8 +85,8 @@ if __name__ == "__main__":
os.makedirs(rel_path)
print('---- worker launched, uuid={} session_uuid={} epoch={}'.format(uuid, session_uuid, time.time()))
else:
print('Incorrect Stream, Closing worker: type={} session_uuid={}'.format(type, session_uuid))
sys.exit(1)
print('Incorrect message')
redis_server_stream.sadd('working_session_uuid:{}'.format(type), session_uuid)
#LAUNCH a tcpdump
@ -149,16 +149,16 @@ if __name__ == "__main__":
except subprocess.TimeoutExpired:
process_compressor.kill()
### compress all files ###
date = datetime.datetime.now().strftime("%Y%m%d")
worker_data_directory = os.path.join(full_tcpdump_path, date[0:4], date[4:6], date[6:8])
if os.path.isdir(worker_data_directory):
all_files = os.listdir(worker_data_directory)
all_files.sort()
if all_files:
for file in all_files:
if file.endswith('.cap'):
full_path = os.path.join(worker_data_directory, file)
if redis_server_stream.get('data_in_process:{}'.format(session_uuid)) != full_path:
compress_file(full_path)
all_files = os.listdir(worker_data_directory)
all_files.sort()
if all_files:
for file in all_files:
if file.endswith('.cap'):
full_path = os.path.join(worker_data_directory, file)
if redis_server_stream.get('data_in_process:{}'.format(session_uuid)) != full_path:
compress_file(full_path)
### ###
#print(process.stderr.read())

View File

@ -1,80 +0,0 @@
#!/usr/bin/env python3
from meta_types_modules.MetaTypesDefault import MetaTypesDefault
import hashlib
import time
import os
import datetime
import base64
import shutil
import gzip
class TypeHandler(MetaTypesDefault):
def __init__(self, uuid, json_file):
super().__init__(uuid, json_file)
self.compress = False
self.extension = ''
self.segregate = True
if "compress" in json_file:
self.compress = json_file['compress']
if "extension" in json_file:
self.extension = json_file['extension']
if "segregate" in json_file:
self.segregate = json_file['segregate']
self.set_rotate_file_mode(False)
self.saved_dir = ''
def process_data(self, data):
# Unpack the thing
self.reconstruct_data(data)
# pushing the filepath instead of the file content to the analyzer
def handle_reconstructed_data(self, data):
m = hashlib.sha256()
self.set_last_time_saved(time.time())
self.set_last_saved_date(datetime.datetime.now().strftime("%Y%m%d%H%M%S"))
# Create folder
save_dir = os.path.join(self.get_save_dir(save_by_uuid=self.segregate), 'files')
if not os.path.isdir(save_dir):
os.makedirs(save_dir)
# write file to disk
decodeddata = base64.b64decode(data)
m.update(decodeddata)
path = os.path.join(save_dir, m.hexdigest())
path = '{}.{}'.format(path, self.extension)
with open(path, 'wb') as p:
p.write(decodeddata)
if self.compress:
compressed_filename = '{}.gz'.format(path)
with open(path, 'rb') as f_in:
with gzip.open(compressed_filename, 'wb') as f_out:
shutil.copyfileobj(f_in, f_out)
os.remove(path)
self.send_to_analyzers(compressed_filename)
else:
self.send_to_analyzers(path)
def reconstruct_data(self, data):
# save data in buffer
self.add_to_buffer(data)
data = self.get_buffer()
# end of element found in data
if self.get_file_separator() in data:
# empty buffer
self.reset_buffer()
all_line = data.split(self.get_file_separator())
for reconstructed_data in all_line[:-1]:
if reconstructed_data != b'':
self.handle_reconstructed_data(reconstructed_data)
# save incomplete element in buffer
if all_line[-1] != b'':
self.add_to_buffer(all_line[-1])
def test(self):
print('Class: filewatcher')

View File

@ -1,38 +0,0 @@
#!/usr/bin/env python3
from meta_types_modules.MetaTypesDefault import MetaTypesDefault
import hashlib
import time
import os
import datetime
class TypeHandler(MetaTypesDefault):
def __init__(self, uuid, json_file):
super().__init__(uuid, json_file)
self.set_rotate_file_mode(False)
self.saved_dir = ''
def process_data(self, data):
self.reconstruct_data(data)
# pushing the filepath instead of the file content to the analyzer
def handle_reconstructed_data(self, data):
m = hashlib.sha256()
self.set_last_time_saved(time.time())
self.set_last_saved_date(datetime.datetime.now().strftime("%Y%m%d%H%M%S"))
# Create folder
jsons_save_dir = os.path.join(self.get_save_dir(save_by_uuid=True), 'files')
if not os.path.isdir(jsons_save_dir):
os.makedirs(jsons_save_dir)
# write json file to disk
m.update(data)
jsons_path = os.path.join(jsons_save_dir, m.hexdigest()+'.json')
with open(jsons_path, 'wb') as j:
j.write(data)
# Send data to Analyszer
self.send_to_analyzers(jsons_path)
def test(self):
print('Class: filewatcherjson')

View File

@ -60,8 +60,8 @@ if __name__ == "__main__":
rel_path = os.path.join(dir_path, filename)
print('---- worker launched, uuid={} session_uuid={} epoch={}'.format(uuid, session_uuid, time.time()))
else:
print('Incorrect Stream, Closing worker: type={} session_uuid={}'.format(type, session_uuid))
sys.exit(1)
print('Incorrect message')
time_file = time.time()
rotate_file = False