Compare commits
44 Commits
Author | SHA1 | Date |
---|---|---|
Jean-Louis Huynen | cb3d618ee1 | |
Koen Van Impe | 27aa5b1df9 | |
Alexandre Dulaunoy | 2e8ddd490f | |
DocArmoryTech | 090d0f66bb | |
DocArmoryTech | dfd53c126b | |
DocArmoryTech | 6273a220b2 | |
Jean-Louis Huynen | 81686aa022 | |
Gerard Wagener | 399e659d8f | |
Terrtia | b2f463e8f1 | |
Terrtia | adf0f6008b | |
Terrtia | 39d593364d | |
Terrtia | cbb90c057a | |
Terrtia | b7998d5601 | |
Terrtia | dc3cdcbc1c | |
Jean-Louis Huynen | 6c3c9f9954 | |
Jean-Louis Huynen | a6d5a3d22c | |
Jean-Louis Huynen | 36a771ea2d | |
Jean-Louis Huynen | ef6e87f3c5 | |
Jean-Louis Huynen | 5a3e299332 | |
Jean-Louis Huynen | d74d2fb71a | |
Jean-Louis Huynen | cf64529929 | |
Terrtia | 893631e003 | |
Terrtia | ac301b5360 | |
Terrtia | 04fab82f5e | |
Terrtia | a297cef179 | |
Terrtia | 3edf227cc1 | |
Terrtia | 2d358918c9 | |
Terrtia | 47f82c8879 | |
Terrtia | 6fee7df9fe | |
Terrtia | 4b30072880 | |
Terrtia | 7ce265e477 | |
Terrtia | 168c31a5bf | |
Terrtia | adda78faad | |
Terrtia | df482d6ee3 | |
Terrtia | 609402ebf2 | |
Thirion Aurélien | 98e562bd47 | |
Terrtia | f17f80b21c | |
Jean-Louis Huynen | 00e3ce3437 | |
Terrtia | 82b2944119 | |
Terrtia | 7078f341ae | |
Terrtia | cb1c8c4d65 | |
Terrtia | 091994a34d | |
Terrtia | 209cd0500f | |
Terrtia | 99656658f2 |
46
README.md
|
@ -64,10 +64,31 @@ git submodule init
|
|||
git submodule update
|
||||
~~~~
|
||||
|
||||
Build the d4 client. This will create the `d4` binary.
|
||||
|
||||
~~~~
|
||||
make
|
||||
~~~~
|
||||
|
||||
Then register the sensor with the server. Replace `API_TOKEN`, `VALID_UUID4` (create a random UUID via [UUIDgenerator](https://www.uuidgenerator.net/)) and `VALID_HMAC_KEY`.
|
||||
|
||||
~~~~
|
||||
curl -k https://127.0.0.1:7000/api/v1/add/sensor/register --header "Authorization: API_TOKEN" -H "Content-Type: application/json" --data '{"uuid":"VALID_UUID4","hmac_key":"VALID_HMAC_KEY"}' -X POST
|
||||
~~~~
|
||||
|
||||
If the registration went correctly the UUID is returned. Do not forget to approve the registration in the D4 server web interface.
|
||||
|
||||
Update the configuration file
|
||||
|
||||
~~~~
|
||||
cp -r conf.sample conf
|
||||
echo VALID_UUID4 > conf/uuid
|
||||
echo VALID_HMAC_KEY > conf/key
|
||||
~~~~
|
||||
|
||||
## D4 core server
|
||||
|
||||
D4 core server is a complete server to handle clients (sensors) including the decapsulation of the [D4 protocol](https://github.com/D4-project/architecture/tree/master/format), control of
|
||||
sensor registrations, management of decoding protocols and dispatching to adequate decoders/analysers.
|
||||
D4 core server is a complete server to handle clients (sensors) including the decapsulation of the [D4 protocol](https://github.com/D4-project/architecture/tree/master/format), control of sensor registrations, management of decoding protocols and dispatching to adequate decoders/analysers.
|
||||
|
||||
### Requirements
|
||||
|
||||
|
@ -76,13 +97,26 @@ sensor registrations, management of decoding protocols and dispatching to adequa
|
|||
|
||||
### Installation
|
||||
|
||||
|
||||
- [Install D4 Server](https://github.com/D4-project/d4-core/tree/master/server)
|
||||
|
||||
### Screenshots of D4 core server management
|
||||
### D4 core server Screenshots
|
||||
|
||||
#### Dashboard:
|
||||
![](https://raw.githubusercontent.com/D4-project/d4-core/master/doc/images/main.png)
|
||||
|
||||
#### Connected Sensors:
|
||||
![](https://raw.githubusercontent.com/D4-project/d4-core/master/doc/images/sensor-mgmt.png)
|
||||
![](https://raw.githubusercontent.com/D4-project/d4-core/master/doc/images/server-mgmt.png)
|
||||
|
||||
#### Sensors Status:
|
||||
![](https://raw.githubusercontent.com/D4-project/d4-core/master/doc/images/sensor_status.png)
|
||||
![](https://raw.githubusercontent.com/D4-project/d4-core/master/doc/images/sensor_stat_types.png)
|
||||
![](https://raw.githubusercontent.com/D4-project/d4-core/master/doc/images/sensor_stat_files.png)
|
||||
|
||||
#### Server Management:
|
||||
![](https://raw.githubusercontent.com/D4-project/d4-core/master/doc/images/server-management.png)
|
||||
![](https://raw.githubusercontent.com/D4-project/d4-core/master/doc/images/server-management-types.png)
|
||||
|
||||
#### analyzer Queues:
|
||||
![](https://raw.githubusercontent.com/D4-project/d4-core/master/doc/images/analyzer-queues.png)
|
||||
![](https://raw.githubusercontent.com/D4-project/d4-core/master/doc/images/create_analyzer_queue.png)
|
||||
![](https://raw.githubusercontent.com/D4-project/d4-core/master/doc/images/analyzer-mgmt.png)
|
||||
![](https://raw.githubusercontent.com/D4-project/d4-core/master/doc/images/server-mgmt2.png)
|
||||
|
|
|
@ -32,7 +32,7 @@ clean:
|
|||
- rm -rf *.o hmac
|
||||
|
||||
d4: d4.o sha2.o hmac.o unpack.o unparse.o pack.o gen_uuid.o randutils.o parse.o
|
||||
gcc -Wall -o d4 d4.o hmac.o sha2.o unpack.o pack.o unparse.o gen_uuid.o randutils.o parse.o
|
||||
$(CC) -Wall -o d4 d4.o hmac.o sha2.o unpack.o pack.o unparse.o gen_uuid.o randutils.o parse.o
|
||||
|
||||
d4.o: d4.c
|
||||
gcc -Wall -c d4.c
|
||||
$(CC) -Wall -c d4.c
|
||||
|
|
|
@ -210,7 +210,7 @@ void d4_transfert(d4_t* d4)
|
|||
//In case of errors see block of 0 bytes
|
||||
bzero(buf, d4->snaplen);
|
||||
nread = read(d4->source.fd, buf, d4->snaplen);
|
||||
if ( nread > 0 ) {
|
||||
if ( nread >= 0 ) {
|
||||
d4_update_header(d4, nread);
|
||||
//Do HMAC on header and payload. HMAC field is 0 during computation
|
||||
if (d4->ctx) {
|
||||
|
@ -238,6 +238,11 @@ void d4_transfert(d4_t* d4)
|
|||
fprintf(stderr,"Incomplete header written. abort to let consumer known that the packet is corrupted\n");
|
||||
abort();
|
||||
}
|
||||
// no data - create empty D4 packet
|
||||
if ( nread == 0 ) {
|
||||
//FIXME no data available, sleep, abort, retry
|
||||
break;
|
||||
}
|
||||
} else{
|
||||
//FIXME no data available, sleep, abort, retry
|
||||
break;
|
||||
|
|
After Width: | Height: | Size: 141 KiB |
After Width: | Height: | Size: 117 KiB |
Before Width: | Height: | Size: 88 KiB After Width: | Height: | Size: 243 KiB |
After Width: | Height: | Size: 52 KiB |
After Width: | Height: | Size: 48 KiB |
After Width: | Height: | Size: 66 KiB |
After Width: | Height: | Size: 85 KiB |
After Width: | Height: | Size: 68 KiB |
|
@ -69,6 +69,8 @@ function launching_d4_server {
|
|||
|
||||
screen -S "Server_D4" -X screen -t "Server_D4" bash -c "cd ${D4_HOME}; ./server.py -v 10; read x"
|
||||
sleep 0.1
|
||||
screen -S "Server_D4" -X screen -t "sensors_manager" bash -c "cd ${D4_HOME}; ./sensors_manager.py; read x"
|
||||
sleep 0.1
|
||||
}
|
||||
|
||||
function launching_workers {
|
||||
|
@ -80,7 +82,7 @@ function launching_workers {
|
|||
sleep 0.1
|
||||
screen -S "Workers_D4" -X screen -t "2_workers" bash -c "cd ${D4_HOME}/workers/workers_2; ./workers_manager.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Workers_D4" -X screen -t "2_workers" bash -c "cd ${D4_HOME}/workers/workers_3; ./workers_manager.py; read x"
|
||||
screen -S "Workers_D4" -X screen -t "3_workers" bash -c "cd ${D4_HOME}/workers/workers_3; ./workers_manager.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Workers_D4" -X screen -t "4_workers" bash -c "cd ${D4_HOME}/workers/workers_4; ./workers_manager.py; read x"
|
||||
sleep 0.1
|
||||
|
|
|
@ -15,11 +15,24 @@ sensor registrations, management of decoding protocols and dispatching to adequa
|
|||
### Installation
|
||||
|
||||
###### Install D4 server
|
||||
|
||||
Clone the repository and install necessary packages. Installation requires *sudo* permissions.
|
||||
|
||||
~~~~
|
||||
git clone https://github.com/D4-project/d4-core.git
|
||||
cd d4-core
|
||||
cd server
|
||||
./install_server.sh
|
||||
~~~~
|
||||
Create or add a pem in [d4-core/server](https://github.com/D4-project/d4-core/tree/master/server) :
|
||||
|
||||
When the installation is finished, scroll back to where `+ ./create_default_user.py` is displayed. The next lines contain the default generated user and should resemble the snippet below. Take a temporary note of the password, you are required to **change the password** on first login.
|
||||
~~~~
|
||||
new user created: admin@admin.test
|
||||
password: <redacted>
|
||||
token: <redacted>
|
||||
~~~~
|
||||
|
||||
Then create or add a pem in [d4-core/server](https://github.com/D4-project/d4-core/tree/master/server) :
|
||||
~~~~
|
||||
cd gen_cert
|
||||
./gen_root.sh
|
||||
|
@ -27,7 +40,6 @@ cd gen_cert
|
|||
cd ..
|
||||
~~~~
|
||||
|
||||
|
||||
###### Launch D4 server
|
||||
~~~~
|
||||
./LAUNCH.sh -l
|
||||
|
@ -35,6 +47,14 @@ cd ..
|
|||
|
||||
The web interface is accessible via `http://127.0.0.1:7000/`
|
||||
|
||||
If you cannot access the web interface on localhost (for example because the system is running on a remote host), then stop the server, change the listening host IP and restart the server. In the below example it's changed to `0.0.0.0` (all interfaces). Make sure that the IP is not unintentionally publicly exposed.
|
||||
|
||||
~~~~
|
||||
./LAUNCH.sh -k
|
||||
sed -i '/\[Flask_Server\]/{:a;N;/host = 127\.0\.0\.1/!ba;s/host = 127\.0\.0\.1/host = 0.0.0.0/}' configs/server.conf
|
||||
./LAUNCH.sh -l
|
||||
~~~~
|
||||
|
||||
### Updating web assets
|
||||
To update javascript libs run:
|
||||
~~~~
|
||||
|
@ -46,19 +66,32 @@ cd web
|
|||
|
||||
[API Documentation](https://github.com/D4-project/d4-core/tree/master/server/documentation/README.md)
|
||||
|
||||
|
||||
### Notes
|
||||
|
||||
- All server logs are located in ``d4-core/server/logs/``
|
||||
- Close D4 Server: ``./LAUNCH.sh -k``
|
||||
|
||||
### Screenshots of D4 core server management
|
||||
### D4 core server
|
||||
|
||||
#### Dashboard:
|
||||
![](https://raw.githubusercontent.com/D4-project/d4-core/master/doc/images/main.png)
|
||||
|
||||
#### Connected Sensors:
|
||||
![](https://raw.githubusercontent.com/D4-project/d4-core/master/doc/images/sensor-mgmt.png)
|
||||
![](https://raw.githubusercontent.com/D4-project/d4-core/master/doc/images/server-mgmt.png)
|
||||
|
||||
#### Sensors Status:
|
||||
![](https://raw.githubusercontent.com/D4-project/d4-core/master/doc/images/sensor_status.png)
|
||||
![](https://raw.githubusercontent.com/D4-project/d4-core/master/doc/images/sensor_stat_types.png)
|
||||
![](https://raw.githubusercontent.com/D4-project/d4-core/master/doc/images/sensor_stat_files.png)
|
||||
|
||||
#### Server Management:
|
||||
![](https://raw.githubusercontent.com/D4-project/d4-core/master/doc/images/server-management.png)
|
||||
![](https://raw.githubusercontent.com/D4-project/d4-core/master/doc/images/server-management-types.png)
|
||||
|
||||
#### analyzer Queues:
|
||||
![](https://raw.githubusercontent.com/D4-project/d4-core/master/doc/images/analyzer-queues.png)
|
||||
![](https://raw.githubusercontent.com/D4-project/d4-core/master/doc/images/create_analyzer_queue.png)
|
||||
![](https://raw.githubusercontent.com/D4-project/d4-core/master/doc/images/analyzer-mgmt.png)
|
||||
![](https://raw.githubusercontent.com/D4-project/d4-core/master/doc/images/server-mgmt2.png)
|
||||
|
||||
### Troubleshooting
|
||||
|
||||
|
@ -71,3 +104,7 @@ Run the following command as root:
|
|||
~~~~
|
||||
aa-complain /usr/sbin/tcpdump
|
||||
~~~~
|
||||
|
||||
###### WARNING - Not registered UUID=UUID4, connection closed
|
||||
|
||||
This happens after you have registered a new sensor, but have not approved the registration. In order to approve the sensor, go in the web interface to **Server Management**, and click **Pending Sensors**.
|
|
@ -24,7 +24,7 @@ if __name__ == "__main__":
|
|||
parser.add_argument('-p', '--port',help='server port' , type=int, dest='target_port', required=True)
|
||||
parser.add_argument('-k', '--Keepalive', help='Keepalive in second', type=int, default='15', dest='ka_sec')
|
||||
parser.add_argument('-n', '--newline', help='add new lines', action="store_true")
|
||||
parser.add_argument('-ri', '--redis_ip',help='redis host' , type=str, default='127.0.0.1', dest='host_redis')
|
||||
parser.add_argument('-ri', '--redis_ip',help='redis ip' , type=str, default='127.0.0.1', dest='host_redis')
|
||||
parser.add_argument('-rp', '--redis_port',help='redis port' , type=int, default=6380, dest='port_redis')
|
||||
args = parser.parse_args()
|
||||
|
||||
|
@ -83,4 +83,4 @@ if __name__ == "__main__":
|
|||
print(d4_data)
|
||||
client_socket.sendall(d4_data)
|
||||
|
||||
client_socket.close()
|
||||
client_socket.shutdown(socket.SHUT_RDWR)
|
||||
|
|
|
@ -0,0 +1,101 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
import redis
|
||||
import time
|
||||
import datetime
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import logging.handlers
|
||||
|
||||
|
||||
import socket
|
||||
import ssl
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description='Export d4 data to stdout')
|
||||
parser.add_argument('-t', '--type', help='d4 type or extended type', type=str, dest='type', required=True)
|
||||
parser.add_argument('-u', '--uuid', help='queue uuid', type=str, dest='uuid', required=True)
|
||||
parser.add_argument('-i', '--ip',help='server ip', type=str, default='127.0.0.1', dest='target_ip')
|
||||
parser.add_argument('-p', '--port',help='server port', type=int, dest='target_port', required=True)
|
||||
parser.add_argument('-k', '--Keepalive', help='Keepalive in second', type=int, default='15', dest='ka_sec')
|
||||
parser.add_argument('-n', '--newline', help='add new lines', action="store_true")
|
||||
parser.add_argument('-ri', '--redis_ip', help='redis ip', type=str, default='127.0.0.1', dest='host_redis')
|
||||
parser.add_argument('-rp', '--redis_port', help='redis port', type=int, default=6380, dest='port_redis')
|
||||
parser.add_argument('-v', '--verify_certificate', help='verify server certificate', type=str, default='True', dest='verify_certificate')
|
||||
parser.add_argument('-c', '--ca_certs', help='cert filename' , type=str, default=None, dest='ca_certs')
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.uuid or not args.type or not args.target_port:
|
||||
parser.print_help()
|
||||
sys.exit(0)
|
||||
|
||||
host_redis=args.host_redis
|
||||
port_redis=args.port_redis
|
||||
newLines=args.newline
|
||||
verify_certificate=args.verify_certificate
|
||||
ca_certs=args.ca_certs
|
||||
|
||||
redis_d4= redis.StrictRedis(
|
||||
host=host_redis,
|
||||
port=port_redis,
|
||||
db=2)
|
||||
try:
|
||||
redis_d4.ping()
|
||||
except redis.exceptions.ConnectionError:
|
||||
print('Error: Redis server {}:{}, ConnectionError'.format(host_redis, port_redis))
|
||||
sys.exit(1)
|
||||
|
||||
d4_uuid = args.uuid
|
||||
d4_type = args.type
|
||||
data_queue = 'analyzer:{}:{}'.format(d4_type, d4_uuid)
|
||||
|
||||
target_ip = args.target_ip
|
||||
target_port = args.target_port
|
||||
addr = (target_ip, target_port)
|
||||
|
||||
# default keep alive: 15
|
||||
ka_sec = args.ka_sec
|
||||
|
||||
# Create a TCP socket
|
||||
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
|
||||
# TCP Keepalive
|
||||
s.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
|
||||
s.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPCNT, 1)
|
||||
s.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, ka_sec)
|
||||
s.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, ka_sec)
|
||||
|
||||
# SSL
|
||||
if verify_certificate in ['False', 'false', 'f']:
|
||||
cert_reqs_option = ssl.CERT_NONE
|
||||
else:
|
||||
cert_reqs_option = ssl.CERT_REQUIRED
|
||||
|
||||
if ca_certs:
|
||||
ca_certs = None
|
||||
|
||||
client_socket = ssl.wrap_socket(s, cert_reqs=cert_reqs_option, ca_certs=ca_certs, ssl_version=ssl.PROTOCOL_TLS)
|
||||
|
||||
# TCP connect
|
||||
client_socket.connect(addr)
|
||||
|
||||
newLines=True
|
||||
while True:
|
||||
|
||||
d4_data = redis_d4.rpop(data_queue)
|
||||
if d4_data is None:
|
||||
time.sleep(1)
|
||||
continue
|
||||
|
||||
if newLines:
|
||||
d4_data = d4_data + b'\n'
|
||||
|
||||
print(d4_data)
|
||||
client_socket.send(d4_data)
|
||||
|
||||
client_socket.shutdown(socket.SHUT_RDWR)
|
|
@ -4,12 +4,15 @@ use_default_save_directory = yes
|
|||
save_directory = None
|
||||
|
||||
[D4_Server]
|
||||
server_port=4443
|
||||
# registration or shared-secret
|
||||
server_mode = registration
|
||||
analyzer_queues_max_size = 10000
|
||||
default_hmac_key = private key to change
|
||||
analyzer_queues_max_size = 100000000
|
||||
|
||||
[Flask_Server]
|
||||
# UI port number
|
||||
host = 127.0.0.1
|
||||
port = 7000
|
||||
|
||||
[Redis_STREAM]
|
||||
|
|
|
@ -30,8 +30,8 @@ git checkout 5.0
|
|||
make
|
||||
popd
|
||||
|
||||
# LAUNCH Redis
|
||||
bash ${AIL_BIN}LAUNCH.sh -lrv &
|
||||
# LAUNCH
|
||||
bash LAUNCH.sh -l &
|
||||
wait
|
||||
echo ""
|
||||
|
||||
|
|
|
@ -303,12 +303,12 @@ def add_data_to_queue(sensor_uuid, queue_type, data):
|
|||
r_serv_analyzer.ltrim('analyzer:{}:{}'.format(queue_type, queue_uuid), 0, analyser_queue_max_size)
|
||||
|
||||
|
||||
def flush_queue(queue_uuid, format_type):
|
||||
r_serv_analyzer.delete('analyzer:{}:{}'.format(format_type, queue_uuid))
|
||||
def flush_queue(queue_uuid, queue_type):
|
||||
r_serv_analyzer.delete('analyzer:{}:{}'.format(queue_type, queue_uuid))
|
||||
|
||||
def remove_queues(queue_uuid, format_type, metatype_name=None):
|
||||
def remove_queues(queue_uuid, queue_type, metatype_name=None):
|
||||
try:
|
||||
format_type = int(format_type)
|
||||
queue_type = int(queue_type)
|
||||
except:
|
||||
print('error: Invalid format type')
|
||||
return {'error': 'Invalid format type'}
|
||||
|
@ -321,7 +321,7 @@ def remove_queues(queue_uuid, format_type, metatype_name=None):
|
|||
print('error: unknow queue uuid')
|
||||
return {'error': 'unknow queue uuid'}
|
||||
|
||||
if format_type==254 and not metatype_name:
|
||||
if queue_type==254 and not metatype_name:
|
||||
metatype_name = get_queue_extended_type(queue_uuid)
|
||||
|
||||
# delete metadata
|
||||
|
@ -332,7 +332,7 @@ def remove_queues(queue_uuid, format_type, metatype_name=None):
|
|||
if l_sensors_uuid:
|
||||
r_serv_metadata.delete('analyzer_sensor_group:{}'.format(queue_uuid))
|
||||
|
||||
if format_type == 254:
|
||||
if queue_type == 254:
|
||||
queue_type = metatype_name
|
||||
for sensor_uuid in l_sensors_uuid:
|
||||
r_serv_metadata.srem('sensor:queues:{}:{}'.format(queue_type, sensor_uuid), queue_uuid)
|
||||
|
@ -342,25 +342,25 @@ def remove_queues(queue_uuid, format_type, metatype_name=None):
|
|||
else:
|
||||
analyzer_key_name = 'analyzer'
|
||||
|
||||
r_serv_metadata.srem('all:analyzer:by:format_type:{}'.format(format_type), queue_uuid)
|
||||
if format_type == 254:
|
||||
r_serv_metadata.srem('all:analyzer:by:format_type:{}'.format(queue_type), queue_uuid)
|
||||
if queue_type == 254:
|
||||
r_serv_metadata.srem('{}:254:{}'.format(analyzer_key_name, metatype_name), queue_uuid)
|
||||
r_serv_metadata.srem('all:analyzer:by:extended_type:{}'.format(metatype_name), queue_uuid)
|
||||
else:
|
||||
r_serv_metadata.srem('{}:{}'.format(analyzer_key_name, format_type), queue_uuid)
|
||||
r_serv_metadata.srem('{}:{}'.format(analyzer_key_name, queue_type), queue_uuid)
|
||||
|
||||
r_serv_metadata.srem('all_analyzer_queues', queue_uuid)
|
||||
|
||||
## delete global queue ##
|
||||
if not r_serv_metadata.exists('all:analyzer:by:format_type:{}'.format(format_type)):
|
||||
r_serv_metadata.srem('all:analyzer:format_type', format_type)
|
||||
if format_type ==254:
|
||||
if not r_serv_metadata.exists('all:analyzer:by:format_type:{}'.format(queue_type)):
|
||||
r_serv_metadata.srem('all:analyzer:format_type', queue_type)
|
||||
if queue_type ==254:
|
||||
if not r_serv_metadata.exists('all:analyzer:by:extended_type:{}'.format(metatype_name)):
|
||||
r_serv_metadata.srem('all:analyzer:extended_type', metatype_name)
|
||||
## --- ##
|
||||
|
||||
# delete qeue
|
||||
r_serv_analyzer.delete('analyzer:{}:{}'.format(format_type, queue_uuid))
|
||||
r_serv_analyzer.delete('analyzer:{}:{}'.format(queue_type, queue_uuid))
|
||||
|
||||
def get_sensor_queues(sensor_uuid):
|
||||
pass
|
||||
|
|
|
@ -11,6 +11,7 @@ from flask import escape
|
|||
|
||||
sys.path.append(os.path.join(os.environ['D4_HOME'], 'lib/'))
|
||||
import ConfigLoader
|
||||
import d4_server
|
||||
|
||||
### Config ###
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
|
@ -26,6 +27,13 @@ def is_valid_uuid_v4(UUID):
|
|||
except:
|
||||
return False
|
||||
|
||||
def get_time_sensor_last_seen(sensor_uuid):
|
||||
res = r_serv_db.hget('metadata_uuid:{}'.format(sensor_uuid), 'last_seen')
|
||||
if res:
|
||||
return int(res)
|
||||
else:
|
||||
return 0
|
||||
|
||||
def _get_sensor_type(sensor_uuid, first_seen=True, last_seen=True, time_format='default'):
|
||||
uuid_type = []
|
||||
uuid_all_type = r_serv_db.smembers('all_types_by_uuid:{}'.format(sensor_uuid))
|
||||
|
@ -68,6 +76,8 @@ def _get_sensor_metadata(sensor_uuid, first_seen=True, last_seen=True, time_form
|
|||
meta_sensor['mail'] = r_serv_db.hget('metadata_uuid:{}'.format(sensor_uuid), 'user_mail')
|
||||
return meta_sensor
|
||||
|
||||
### BEGIN - SENSOR REGISTRATION ###
|
||||
|
||||
## TODO: add description
|
||||
def register_sensor(req_dict):
|
||||
sensor_uuid = req_dict.get('uuid', None)
|
||||
|
@ -80,7 +90,7 @@ def register_sensor(req_dict):
|
|||
sensor_uuid = sensor_uuid.replace('-', '')
|
||||
# sensor already exist
|
||||
if r_serv_db.exists('metadata_uuid:{}'.format(sensor_uuid)):
|
||||
return ({"status": "error", "reason": "Sensor already registred"}, 409)
|
||||
return ({"status": "error", "reason": "Sensor already registered"}, 409)
|
||||
|
||||
# hmac key
|
||||
if not hmac_key:
|
||||
|
@ -167,3 +177,99 @@ def delete_registered_sensor(req_dict):
|
|||
def _delete_registered_sensor(sensor_uuid):
|
||||
r_serv_db.srem('registered_uuid', sensor_uuid)
|
||||
return ({'uuid': sensor_uuid}, 200)
|
||||
|
||||
### --- END - SENSOR REGISTRATION --- ###
|
||||
|
||||
|
||||
### BEGIN - SENSOR MONITORING ###
|
||||
def get_sensors_monitoring_last_updated():
|
||||
res = r_serv_db.get('sensors_monitoring:last_updated')
|
||||
if res:
|
||||
return int(res)
|
||||
else:
|
||||
return 0
|
||||
|
||||
def get_all_sensors_to_monitor():
|
||||
return r_serv_db.smembers('to_monitor:sensors')
|
||||
|
||||
def get_to_monitor_delta_time_by_uuid(sensor_uuid):
|
||||
return int(r_serv_db.hget('to_monitor:sensor:{}'.format(sensor_uuid), 'delta_time'))
|
||||
|
||||
def get_all_sensors_to_monitor_dict():
|
||||
dict_to_monitor = {}
|
||||
for sensor_uuid in get_all_sensors_to_monitor():
|
||||
dict_to_monitor[sensor_uuid] = get_to_monitor_delta_time_by_uuid(sensor_uuid)
|
||||
return dict_to_monitor
|
||||
|
||||
def _check_sensor_delta(sensor_uuid, sensor_delta):
|
||||
last_d4_packet = get_time_sensor_last_seen(sensor_uuid)
|
||||
|
||||
# check sensor delta time between two D4 packets + check sensor connection
|
||||
if int(time.time()) - last_d4_packet > sensor_delta or not d4_server.is_sensor_connected(sensor_uuid):
|
||||
r_serv_db.sadd('sensors_monitoring:sensors_error', sensor_uuid)
|
||||
handle_sensor_monitoring_error(sensor_uuid)
|
||||
else:
|
||||
r_serv_db.srem('sensors_monitoring:sensors_error', sensor_uuid)
|
||||
|
||||
def handle_sensor_monitoring_error(sensor_uuid):
|
||||
print('sensor monitoring error: {}'.format(sensor_uuid))
|
||||
## TODO: ##
|
||||
# MAILS
|
||||
# UI Notifications
|
||||
# SNMP
|
||||
# Syslog message
|
||||
## ## ## ##
|
||||
return None
|
||||
|
||||
def is_sensor_monitored(sensor_uuid):
|
||||
return r_serv_db.sismember('to_monitor:sensors', sensor_uuid)
|
||||
|
||||
def get_all_sensors_connection_errors():
|
||||
return r_serv_db.smembers('sensors_monitoring:sensors_error')
|
||||
|
||||
def api_get_all_sensors_connection_errors():
|
||||
return list(get_all_sensors_connection_errors()), 200
|
||||
|
||||
def add_sensor_to_monitor(sensor_uuid, delta_time):
|
||||
r_serv_db.sadd('to_monitor:sensors', sensor_uuid)
|
||||
r_serv_db.hset('to_monitor:sensor:{}'.format(sensor_uuid), 'delta_time', delta_time)
|
||||
r_serv_db.set('sensors_monitoring:last_updated', int(time.time()))
|
||||
r_serv_db.srem('sensors_monitoring:sensors_error', sensor_uuid)
|
||||
|
||||
def delete_sensor_to_monitor(sensor_uuid):
|
||||
r_serv_db.srem('to_monitor:sensors', sensor_uuid)
|
||||
r_serv_db.delete('to_monitor:sensor:{}'.format(sensor_uuid))
|
||||
r_serv_db.set('sensors_monitoring:last_updated', int(time.time()))
|
||||
r_serv_db.srem('sensors_monitoring:sensors_error', sensor_uuid)
|
||||
|
||||
def api_add_sensor_to_monitor(data_dict):
|
||||
sensor_uuid = data_dict.get('uuid', None)
|
||||
delta_time = data_dict.get('delta_time', None)
|
||||
|
||||
if not is_valid_uuid_v4(sensor_uuid):
|
||||
return ({"status": "error", "reason": "Invalid uuid"}, 400)
|
||||
sensor_uuid = sensor_uuid.replace('-', '')
|
||||
|
||||
# hmac key
|
||||
if not delta_time:
|
||||
return ({"status": "error", "reason": "Mandatory parameter(s) not provided"}, 400)
|
||||
else:
|
||||
try:
|
||||
delta_time = int(delta_time)
|
||||
if delta_time < 1:
|
||||
return ({"status": "error", "reason": "Invalid delta_time"}, 400)
|
||||
except Exception:
|
||||
return ({"status": "error", "reason": "Invalid delta_time"}, 400)
|
||||
add_sensor_to_monitor(sensor_uuid, delta_time)
|
||||
|
||||
def api_delete_sensor_to_monitor(data_dict):
|
||||
sensor_uuid = data_dict.get('uuid', None)
|
||||
if not is_valid_uuid_v4(sensor_uuid):
|
||||
return ({"status": "error", "reason": "Invalid uuid"}, 400)
|
||||
sensor_uuid = sensor_uuid.replace('-', '')
|
||||
if not is_sensor_monitored(sensor_uuid):
|
||||
return ({"status": "error", "reason": "Sensor not monitored"}, 400)
|
||||
delete_sensor_to_monitor(sensor_uuid)
|
||||
|
||||
|
||||
### --- END - SENSOR REGISTRATION --- ###
|
||||
|
|
|
@ -0,0 +1,44 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
import redis
|
||||
|
||||
from flask import escape
|
||||
|
||||
sys.path.append(os.path.join(os.environ['D4_HOME'], 'lib/'))
|
||||
import ConfigLoader
|
||||
|
||||
### Config ###
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
r_stream = config_loader.get_redis_conn("Redis_STREAM")
|
||||
config_loader = None
|
||||
### ###
|
||||
|
||||
### BEGIN - SENSOR CONNECTION ###
|
||||
|
||||
def get_all_connected_sensors(r_list=False):
|
||||
res = r_stream.smembers('active_connection')
|
||||
if r_list:
|
||||
if res:
|
||||
return list(res)
|
||||
else:
|
||||
return []
|
||||
else:
|
||||
return res
|
||||
|
||||
def get_all_connected_sensors_by_type(d4_type, d4_extended_type=None):
|
||||
# D4 extended type
|
||||
if d4_type == 254 and d4_extended_type:
|
||||
return r_stream.smembers('active_connection_extended_type:{}'.format(d4_extended_type))
|
||||
# type 1-253
|
||||
else:
|
||||
return r_stream.smembers('active_connection:{}'.format(d4_type))
|
||||
|
||||
def is_sensor_connected(sensor_uuid):
|
||||
return r_stream.sismember('active_connection', sensor_uuid)
|
||||
|
||||
### --- END - SENSOR CONNECTION --- ###
|
|
@ -1,8 +1,9 @@
|
|||
twisted[tls]
|
||||
redis
|
||||
flask
|
||||
flask==2.2.2
|
||||
flask-login
|
||||
bcrypt
|
||||
Werkzeug==2.2.2
|
||||
|
||||
#sudo python3 -m pip install --upgrade service_identity
|
||||
|
||||
|
|
|
@ -0,0 +1,54 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import redis
|
||||
|
||||
sys.path.append(os.path.join(os.environ['D4_HOME'], 'lib/'))
|
||||
import ConfigLoader
|
||||
import Sensor
|
||||
|
||||
### Config ###
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
#redis_server_stream = config_loader.get_redis_conn("Redis_STREAM", decode_responses=False)
|
||||
redis_server_metadata = config_loader.get_redis_conn("Redis_METADATA")
|
||||
config_loader = None
|
||||
### ###
|
||||
|
||||
try:
|
||||
redis_server_metadata.ping()
|
||||
except redis.exceptions.ConnectionError:
|
||||
print('Error: Redis server: Redis_METADATA, ConnectionError')
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def reload_all_sensors_to_monitor_dict(dict_to_monitor, last_updated):
|
||||
if not dict_to_monitor:
|
||||
dict_to_monitor = Sensor.get_all_sensors_to_monitor_dict()
|
||||
else:
|
||||
monitoring_last_updated = Sensor.get_sensors_monitoring_last_updated()
|
||||
if monitoring_last_updated > last_updated:
|
||||
dict_to_monitor = Sensor.get_all_sensors_to_monitor_dict()
|
||||
last_updated = int(time.time())
|
||||
print('updated: List of sensors to monitor')
|
||||
return dict_to_monitor
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
time_refresh = int(time.time())
|
||||
last_updated = time_refresh
|
||||
all_sensors_to_monitor = Sensor.get_all_sensors_to_monitor_dict()
|
||||
|
||||
while True:
|
||||
|
||||
for sensor_uuid in all_sensors_to_monitor:
|
||||
Sensor._check_sensor_delta(sensor_uuid, all_sensors_to_monitor[sensor_uuid])
|
||||
time.sleep(10)
|
||||
|
||||
## reload dict_to_monitor ##
|
||||
curr_time = int(time.time())
|
||||
if curr_time - time_refresh >= 60:
|
||||
time_refresh = curr_time
|
||||
all_sensors_to_monitor = reload_all_sensors_to_monitor_dict(all_sensors_to_monitor, last_updated)
|
||||
##-- --##
|
|
@ -13,6 +13,8 @@ import argparse
|
|||
import logging
|
||||
import logging.handlers
|
||||
|
||||
import configparser
|
||||
|
||||
from twisted.internet import ssl, task, protocol, endpoints, defer
|
||||
from twisted.python import log
|
||||
from twisted.python.modules import getModule
|
||||
|
@ -24,7 +26,6 @@ sys.path.append(os.path.join(os.environ['D4_HOME'], 'lib/'))
|
|||
import ConfigLoader
|
||||
|
||||
hmac_reset = bytearray(32)
|
||||
hmac_key = os.getenv('D4_HMAC_KEY', b'private key to change')
|
||||
|
||||
accepted_type = [1, 2, 4, 8, 254]
|
||||
accepted_extended_type = ['ja3-jl']
|
||||
|
@ -46,7 +47,16 @@ redis_server_stream = config_loader.get_redis_conn("Redis_STREAM", decode_respon
|
|||
redis_server_metadata = config_loader.get_redis_conn("Redis_METADATA", decode_responses=False)
|
||||
|
||||
# get server_mode
|
||||
try:
|
||||
D4server_port = config_loader.get_config_int("D4_Server", "server_port")
|
||||
except configparser.NoOptionError:
|
||||
D4server_port = 4443
|
||||
|
||||
server_mode = config_loader.get_config_str("D4_Server", "server_mode")
|
||||
try:
|
||||
hmac_key = config_loader.get_config_str("D4_Server", "default_hmac_key")
|
||||
except configparser.NoOptionError:
|
||||
hmac_key = 'private key to change'
|
||||
|
||||
config_loader = None
|
||||
### ###
|
||||
|
@ -134,10 +144,10 @@ def extract_ip(ip_string):
|
|||
return ip_string
|
||||
|
||||
def server_mode_registration(header_uuid):
|
||||
# only accept registred uuid
|
||||
# only accept registered uuid
|
||||
if server_mode == 'registration':
|
||||
if not redis_server_metadata.sismember('registered_uuid', header_uuid):
|
||||
error_msg = 'Not registred UUID={}, connection closed'.format(header_uuid)
|
||||
error_msg = 'Not registered UUID={}, connection closed'.format(header_uuid)
|
||||
print(error_msg)
|
||||
logger.warning(error_msg)
|
||||
#redis_server_metadata.hset('metadata_uuid:{}'.format(data_header['uuid_header']), 'Error', 'Error: This UUID is temporarily blacklisted')
|
||||
|
@ -245,8 +255,8 @@ class D4_Server(Protocol, TimeoutMixin):
|
|||
if not redis_server_stream.exists('active_connection_by_uuid:{}'.format(self.uuid)):
|
||||
redis_server_stream.srem('active_connection', self.uuid)
|
||||
|
||||
logger.debug('Connection closed: session_uuid={}'.format(self.session_uuid))
|
||||
dict_all_connection.pop(self.session_uuid)
|
||||
logger.debug('Connection closed: session_uuid={}'.format(self.session_uuid))
|
||||
dict_all_connection.pop(self.session_uuid)
|
||||
|
||||
def unpack_header(self, data):
|
||||
data_header = {}
|
||||
|
@ -259,6 +269,20 @@ class D4_Server(Protocol, TimeoutMixin):
|
|||
data_header['size'] = struct.unpack('I', data[58:62])[0]
|
||||
return data_header
|
||||
|
||||
def check_hmac_key(self, hmac_header, data):
|
||||
if self.hmac_key is None:
|
||||
self.hmac_key = redis_server_metadata.hget('metadata_uuid:{}'.format(self.uuid), 'hmac_key')
|
||||
if self.hmac_key is None:
|
||||
self.hmac_key = redis_server_metadata.get('server:hmac_default_key')
|
||||
|
||||
# set hmac_header to 0
|
||||
data = data.replace(hmac_header, hmac_reset, 1)
|
||||
|
||||
HMAC = hmac.new(self.hmac_key, msg=data, digestmod='sha256')
|
||||
hmac_header = hmac_header.hex()
|
||||
# hmac match
|
||||
return hmac_header == HMAC.hexdigest()
|
||||
|
||||
def check_connection_validity(self, data_header):
|
||||
# blacklist ip by uuid
|
||||
if redis_server_metadata.sismember('blacklist_ip_by_uuid', data_header['uuid_header']):
|
||||
|
@ -335,8 +359,14 @@ class D4_Server(Protocol, TimeoutMixin):
|
|||
self.type = data_header['type']
|
||||
self.uuid = data_header['uuid_header']
|
||||
|
||||
# worker entry point: map type:session_uuid
|
||||
redis_server_stream.sadd('session_uuid:{}'.format(data_header['type']), self.session_uuid.encode())
|
||||
# # check HMAC /!\ incomplete data
|
||||
# if not self.check_hmac_key(data_header['hmac_header'], data):
|
||||
# print('hmac do not match')
|
||||
# print(data)
|
||||
# logger.debug("HMAC don't match, uuid={}, session_uuid={}".format(self.uuid, self.session_uuid))
|
||||
# redis_server_metadata.hset('metadata_uuid:{}'.format(data_header['uuid_header']), 'Error', 'Error: HMAC don\'t match')
|
||||
# self.transport.abortConnection()
|
||||
# return 1
|
||||
|
||||
## save active connection ##
|
||||
#active Connection
|
||||
|
@ -463,15 +493,6 @@ class D4_Server(Protocol, TimeoutMixin):
|
|||
def process_d4_data(self, data, data_header, ip):
|
||||
# empty buffer
|
||||
self.buffer = b''
|
||||
# set hmac_header to 0
|
||||
data = data.replace(data_header['hmac_header'], hmac_reset, 1)
|
||||
if self.hmac_key is None:
|
||||
self.hmac_key = redis_server_metadata.hget('metadata_uuid:{}'.format(data_header['uuid_header']), 'hmac_key')
|
||||
if self.hmac_key is None:
|
||||
self.hmac_key = redis_server_metadata.get('server:hmac_default_key')
|
||||
|
||||
HMAC = hmac.new(self.hmac_key, msg=data, digestmod='sha256')
|
||||
data_header['hmac_header'] = data_header['hmac_header'].hex()
|
||||
|
||||
### Debug ###
|
||||
#print('hexdigest: {}'.format( HMAC.hexdigest() ))
|
||||
|
@ -484,7 +505,7 @@ class D4_Server(Protocol, TimeoutMixin):
|
|||
### ###
|
||||
|
||||
# hmac match
|
||||
if data_header['hmac_header'] == HMAC.hexdigest():
|
||||
if self.check_hmac_key(data_header['hmac_header'], data):
|
||||
if not self.stream_max_size:
|
||||
temp = redis_server_metadata.hget('stream_max_size_by_uuid', data_header['uuid_header'])
|
||||
if temp is not None:
|
||||
|
@ -509,12 +530,16 @@ class D4_Server(Protocol, TimeoutMixin):
|
|||
redis_server_metadata.zincrby('stat_uuid_type:{}:{}'.format(date, data_header['uuid_header']), 1, data_header['type'])
|
||||
|
||||
#
|
||||
d4_packet_rcv_time = int(time.time())
|
||||
if not redis_server_metadata.hexists('metadata_uuid:{}'.format(data_header['uuid_header']), 'first_seen'):
|
||||
redis_server_metadata.hset('metadata_uuid:{}'.format(data_header['uuid_header']), 'first_seen', data_header['timestamp'])
|
||||
redis_server_metadata.hset('metadata_uuid:{}'.format(data_header['uuid_header']), 'last_seen', data_header['timestamp'])
|
||||
redis_server_metadata.hset('metadata_type_by_uuid:{}:{}'.format(data_header['uuid_header'], data_header['type']), 'last_seen', data_header['timestamp'])
|
||||
redis_server_metadata.hset('metadata_uuid:{}'.format(data_header['uuid_header']), 'first_seen', d4_packet_rcv_time)
|
||||
redis_server_metadata.hset('metadata_uuid:{}'.format(data_header['uuid_header']), 'last_seen', d4_packet_rcv_time)
|
||||
redis_server_metadata.hset('metadata_type_by_uuid:{}:{}'.format(data_header['uuid_header'], data_header['type']), 'last_seen', d4_packet_rcv_time)
|
||||
|
||||
if not self.data_saved:
|
||||
# worker entry point: map type:session_uuid
|
||||
redis_server_stream.sadd('session_uuid:{}'.format(data_header['type']), self.session_uuid.encode())
|
||||
|
||||
#UUID IP: ## TODO: use d4 timestamp ?
|
||||
redis_server_metadata.lpush('list_uuid_ip:{}'.format(data_header['uuid_header']), '{}-{}'.format(ip, datetime.datetime.now().strftime("%Y%m%d%H%M%S")))
|
||||
redis_server_metadata.ltrim('list_uuid_ip:{}'.format(data_header['uuid_header']), 0, 15)
|
||||
|
@ -523,7 +548,7 @@ class D4_Server(Protocol, TimeoutMixin):
|
|||
if self.update_stream_type:
|
||||
|
||||
if not redis_server_metadata.hexists('metadata_type_by_uuid:{}:{}'.format(data_header['uuid_header'], data_header['type']), 'first_seen'):
|
||||
redis_server_metadata.hset('metadata_type_by_uuid:{}:{}'.format(data_header['uuid_header'], data_header['type']), 'first_seen', data_header['timestamp'])
|
||||
redis_server_metadata.hset('metadata_type_by_uuid:{}:{}'.format(data_header['uuid_header'], data_header['type']), 'first_seen', d4_packet_rcv_time)
|
||||
self.update_stream_type = False
|
||||
return 0
|
||||
else:
|
||||
|
@ -554,7 +579,7 @@ def main(reactor):
|
|||
certificate = ssl.PrivateCertificate.loadPEM(certData)
|
||||
factory = protocol.Factory.forProtocol(D4_Server)
|
||||
# use interface to support both IPv4 and IPv6
|
||||
reactor.listenSSL(4443, factory, certificate.options(), interface='::')
|
||||
reactor.listenSSL(D4server_port, factory, certificate.options(), interface='::')
|
||||
return defer.Deferred()
|
||||
|
||||
|
||||
|
|
|
@ -35,6 +35,7 @@ import Analyzer_Queue
|
|||
from blueprints.restApi import restApi
|
||||
from blueprints.settings import settings
|
||||
from blueprints.analyzer_queue import analyzer_queue
|
||||
from blueprints.D4_sensors import D4_sensors
|
||||
|
||||
baseUrl = ''
|
||||
if baseUrl != '':
|
||||
|
@ -64,6 +65,12 @@ server_mode = config_loader.get_config_str("D4_Server", "server_mode")
|
|||
if server_mode not in all_server_modes:
|
||||
print('Error: incorrect server_mode')
|
||||
|
||||
try:
|
||||
FLASK_HOST = config_loader.get_config_str("Flask_Server", "host")
|
||||
except Exception as e:
|
||||
print(e)
|
||||
FLASK_HOST = '127.0.0.1'
|
||||
|
||||
try:
|
||||
FLASK_PORT = config_loader.get_config_int("Flask_Server", "port")
|
||||
except Exception:
|
||||
|
@ -96,7 +103,7 @@ app = Flask(__name__, static_url_path=baseUrl+'/static/')
|
|||
app.config['MAX_CONTENT_LENGTH'] = 900 * 1024 * 1024
|
||||
|
||||
# ========= Cookie name ========
|
||||
app.config.update(SESSION_COOKIE_NAME='d4_project_server{}'.format(FLASK_PORT))
|
||||
app.config.update(SESSION_COOKIE_NAME='d4_project_server{}'.format(uuid.uuid4().int))
|
||||
|
||||
# ========= session ========
|
||||
app.secret_key = str(random.getrandbits(256))
|
||||
|
@ -109,6 +116,7 @@ login_manager.init_app(app)
|
|||
app.register_blueprint(restApi)
|
||||
app.register_blueprint(settings)
|
||||
app.register_blueprint(analyzer_queue)
|
||||
app.register_blueprint(D4_sensors)
|
||||
# ========= =========#
|
||||
|
||||
# ========= LOGIN MANAGER ========
|
||||
|
@ -276,7 +284,7 @@ def login():
|
|||
if request.method == 'POST':
|
||||
username = request.form.get('username')
|
||||
password = request.form.get('password')
|
||||
#next_page = request.form.get('next_page')
|
||||
next_page = request.form.get('next_page')
|
||||
|
||||
if username is not None:
|
||||
user = User.get(username)
|
||||
|
@ -293,11 +301,16 @@ def login():
|
|||
#if not check_user_role_integrity(user.get_id()):
|
||||
# error = 'Incorrect User ACL, Please contact your administrator'
|
||||
# return render_template("login.html", error=error)
|
||||
if not user.is_in_role('user'):
|
||||
return render_template("403.html"), 403
|
||||
login_user(user) ## TODO: use remember me ?
|
||||
if user.request_password_change():
|
||||
return redirect(url_for('change_password'))
|
||||
else:
|
||||
return redirect(url_for('index'))
|
||||
if next_page and next_page!='None':
|
||||
return redirect(next_page)
|
||||
else:
|
||||
return redirect(url_for('index'))
|
||||
# login failed
|
||||
else:
|
||||
# set brute force protection
|
||||
|
@ -313,12 +326,13 @@ def login():
|
|||
return 'please provide a valid username'
|
||||
|
||||
else:
|
||||
#next_page = request.args.get('next')
|
||||
next_page = request.args.get('next')
|
||||
error = request.args.get('error')
|
||||
return render_template("login.html" , error=error)
|
||||
return render_template("login.html" , error=error, next_page=next_page)
|
||||
|
||||
@app.route('/change_password', methods=['POST', 'GET'])
|
||||
@login_required
|
||||
@login_user_basic
|
||||
def change_password():
|
||||
password1 = request.form.get('password1')
|
||||
password2 = request.form.get('password2')
|
||||
|
@ -353,7 +367,7 @@ def logout():
|
|||
@app.route('/role', methods=['POST', 'GET'])
|
||||
@login_required
|
||||
def role():
|
||||
return render_template("error/403.html"), 403
|
||||
return render_template("403.html"), 403
|
||||
|
||||
@app.route('/')
|
||||
@login_required
|
||||
|
@ -588,6 +602,8 @@ def uuid_management():
|
|||
"blacklisted_uuid": blacklisted_uuid, "blacklisted_ip_by_uuid": blacklisted_ip_by_uuid,
|
||||
"first_seen_gmt": first_seen_gmt, "last_seen_gmt": last_seen_gmt, "Error": Error}
|
||||
|
||||
data_uuid['is_monitored'] = Sensor.is_sensor_monitored(uuid_sensor)
|
||||
|
||||
if redis_server_stream.sismember('active_connection', uuid_sensor):
|
||||
active_connection = True
|
||||
else:
|
||||
|
@ -1134,4 +1150,4 @@ def get_uuid_stats_history_json():
|
|||
|
||||
|
||||
if __name__ == "__main__":
|
||||
app.run(host='0.0.0.0', port=FLASK_PORT, threaded=True, ssl_context=ssl_context)
|
||||
app.run(host=FLASK_HOST, port=FLASK_PORT, threaded=True, ssl_context=ssl_context)
|
||||
|
|
|
@ -109,7 +109,7 @@ def create_user_db(username_id , password, default=False, role=None, update=Fals
|
|||
|
||||
r_serv_db.hset('user:all', username_id, password_hash)
|
||||
|
||||
def edit_user_db(user_id, role, password=None):
|
||||
def edit_user_db(user_id, password=None, role=None):
|
||||
if password:
|
||||
password_hash = hashing_password(password.encode())
|
||||
r_serv_db.hset('user:all', user_id, password_hash)
|
||||
|
|
|
@ -0,0 +1,76 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
'''
|
||||
Flask functions and routes for all D4 sensors
|
||||
'''
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import json
|
||||
import redis
|
||||
|
||||
sys.path.append(os.path.join(os.environ['D4_HOME'], 'lib'))
|
||||
import ConfigLoader
|
||||
import Sensor
|
||||
|
||||
from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response
|
||||
from flask_login import login_required, current_user
|
||||
|
||||
from Role_Manager import login_admin, login_user_basic
|
||||
|
||||
# ============ BLUEPRINT ============
|
||||
|
||||
D4_sensors = Blueprint('D4_sensors', __name__, template_folder='templates')
|
||||
|
||||
# ============ VARIABLES ============
|
||||
|
||||
### Config ###
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
r_serv_metadata = config_loader.get_redis_conn("Redis_METADATA")
|
||||
r_serv_db = config_loader.get_redis_conn("Redis_SERV")
|
||||
config_loader = None
|
||||
### ###
|
||||
|
||||
# ============ FUNCTIONS ============
|
||||
|
||||
|
||||
# ============= ROUTES ==============
|
||||
|
||||
@D4_sensors.route("/sensors/monitoring/add", methods=['GET'])
|
||||
@login_required
|
||||
@login_user_basic
|
||||
def add_sensor_to_monitor():
|
||||
sensor_uuid = request.args.get("uuid")
|
||||
return render_template("sensors/add_sensor_to_monitor.html",
|
||||
sensor_uuid=sensor_uuid)
|
||||
|
||||
@D4_sensors.route("/sensors/monitoring/add_post", methods=['POST'])
|
||||
@login_required
|
||||
@login_user_basic
|
||||
def add_sensor_to_monitor_post():
|
||||
sensor_uuid = request.form.get("uuid")
|
||||
delta_time = request.form.get("delta_time")
|
||||
res = Sensor.api_add_sensor_to_monitor({'uuid':sensor_uuid, 'delta_time': delta_time})
|
||||
if res:
|
||||
Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||
return redirect(url_for('uuid_management', uuid=sensor_uuid))
|
||||
|
||||
@D4_sensors.route("/sensors/monitoring/delete", methods=['GET'])
|
||||
@login_required
|
||||
@login_user_basic
|
||||
def delete_sensor_to_monitor():
|
||||
sensor_uuid = request.args.get("uuid")
|
||||
res = Sensor.api_delete_sensor_to_monitor({'uuid':sensor_uuid})
|
||||
if res:
|
||||
Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||
return redirect(url_for('uuid_management', uuid=sensor_uuid))
|
||||
|
||||
|
||||
@D4_sensors.route("/sensors/monitoring/errors", methods=['GET'])
|
||||
@login_required
|
||||
@login_user_basic
|
||||
def get_all_sensors_connection_errors():
|
||||
res = Sensor.api_get_all_sensors_connection_errors()
|
||||
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
|
@ -18,7 +18,6 @@ from flask import Flask, render_template, jsonify, request, Blueprint, redirect,
|
|||
from flask_login import login_required, current_user
|
||||
|
||||
from Role_Manager import login_admin, login_user_basic
|
||||
from Role_Manager import create_user_db, edit_user_db, delete_user_db, check_password_strength, generate_new_token, gen_password, get_all_role
|
||||
|
||||
# ============ BLUEPRINT ============
|
||||
|
||||
|
|
|
@ -142,8 +142,8 @@ def is_valid_uuid_v4(header_uuid):
|
|||
except:
|
||||
return False
|
||||
|
||||
def one():
|
||||
return 1
|
||||
def build_json_response(resp_data, resp_code):
|
||||
return Response(json.dumps(resp_data, indent=2, sort_keys=True), mimetype='application/json'), resp_code
|
||||
|
||||
# ============= ROUTES ==============
|
||||
|
||||
|
@ -154,3 +154,9 @@ def add_sensor_register():
|
|||
data = request.get_json()
|
||||
res = Sensor.register_sensor(data)
|
||||
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||
|
||||
@restApi.route("/api/v1/sensors/monitoring/errors", methods=['GET'])
|
||||
@token_required('user')
|
||||
def get_all_sensors_connection_errors():
|
||||
res = Sensor.api_get_all_sensors_connection_errors()
|
||||
return build_json_response(res[0], res[1])
|
||||
|
|
|
@ -25,7 +25,7 @@ if __name__ == "__main__":
|
|||
|
||||
username = 'admin@admin.test'
|
||||
password = gen_password()
|
||||
if r_serv.exists('user_metadata:admin@admin.test'):
|
||||
if r_serv.exists('user_metadata:{}'.format(username)):
|
||||
edit_user_db(username, password=password, role='admin')
|
||||
else:
|
||||
create_user_db(username, password, role='admin', default=True)
|
||||
|
|
|
@ -69,7 +69,8 @@
|
|||
<form class="form-signin" action="{{ url_for('login')}}" method="post">
|
||||
<img class="mb-4" src="{{ url_for('static', filename='img/d4-logo.png')}}" width="300">
|
||||
<h1 class="h3 mb-3 text-secondary">Please sign in</h1>
|
||||
<label for="inputEmail" class="sr-only">Email address</label>
|
||||
<input type="text" id="next_page" name="next_page" value="{{next_page}}" hidden>
|
||||
<label for="inputEmail" class="sr-only">Email address</label>
|
||||
<input type="email" id="inputEmail" name="username" class="form-control" placeholder="Email address" required autofocus>
|
||||
<label for="inputPassword" class="sr-only">Password</label>
|
||||
<input type="password" id="inputPassword" name="password" class="form-control {% if error %}is-invalid{% endif %}" placeholder="Password" required>
|
||||
|
|
|
@ -0,0 +1,57 @@
|
|||
<!DOCTYPE html>
|
||||
|
||||
<html>
|
||||
<head>
|
||||
<title>D4-Project</title>
|
||||
<link rel="icon" href="{{ url_for('static', filename='img/d4-logo.png')}}">
|
||||
<!-- Core CSS -->
|
||||
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
|
||||
|
||||
<!-- JS -->
|
||||
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/bootstrap.min.js')}}"></script>
|
||||
</head>
|
||||
|
||||
<body>
|
||||
|
||||
{% include 'navbar.html' %}
|
||||
|
||||
<form action="{{ url_for('D4_sensors.add_sensor_to_monitor_post') }}" method="post" enctype=multipart/form-data>
|
||||
|
||||
<div class="d-flex justify-content-center">
|
||||
<div class="col-sm-6">
|
||||
<h4 class="my-3">Monitor a Sensor</h4>
|
||||
<div class="form-group">
|
||||
<input class="form-control text-center bg-dark text-white" type="text" value="{{sensor_uuid}}" disabled>
|
||||
<input type="text" name="uuid" id="uuid" value="{{sensor_uuid}}" hidden>
|
||||
</div>
|
||||
|
||||
<div class="input-group mt-2 mb-2">
|
||||
<div class="input-group-prepend">
|
||||
<span class="input-group-text bg-light"><i class="fa fa-clock-o"></i> </span>
|
||||
</div>
|
||||
<input class="form-control" type="number" id="delta_time" value="3600" min="30" name="delta_time" required>
|
||||
<div class="input-group-append">
|
||||
<span class="input-group-text">Maxinum Time (seconds) between two D4 packets</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
<button class="btn btn-primary" type="submit">Monitor Sensor</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</form>
|
||||
|
||||
{% include 'navfooter.html' %}
|
||||
</body>
|
||||
|
||||
<script>
|
||||
$(document).ready(function(){
|
||||
$("#nav-sensor").addClass("active");
|
||||
});
|
||||
|
||||
</script>
|
|
@ -101,6 +101,18 @@
|
|||
</div>
|
||||
</div>
|
||||
|
||||
<div class="d-flex justify-content-center mt-2">
|
||||
{% if not data_uuid.get('is_monitored', False) %}
|
||||
<a href="{{ url_for('D4_sensors.add_sensor_to_monitor') }}?uuid={{uuid_sensor}}">
|
||||
<button type="button" class="btn btn-primary">Monitor Sensor</button>
|
||||
</a>
|
||||
{% else %}
|
||||
<a href="{{ url_for('D4_sensors.delete_sensor_to_monitor') }}?uuid={{uuid_sensor}}">
|
||||
<button type="button" class="btn btn-danger">Remove Sensor from monitoring</button>
|
||||
</a>
|
||||
{% endif %}
|
||||
</div>
|
||||
|
||||
<div class="card-deck justify-content-center ml-0 mr-0">
|
||||
<div class="card border-dark mt-3" style="max-width: 18rem;">
|
||||
<div class="card-body text-dark">
|
||||
|
|
|
@ -47,8 +47,8 @@ mv temp/bootstrap-${BOOTSTRAP_VERSION}-dist/js/bootstrap.min.js ./static/js/
|
|||
mv temp/bootstrap-${BOOTSTRAP_VERSION}-dist/css/bootstrap.min.css ./static/css/
|
||||
mv temp/bootstrap-${BOOTSTRAP_VERSION}-dist/css/bootstrap.min.css.map ./static/css/
|
||||
|
||||
mv temp/popper.js-1.14.3/dist/umd/popper.min.js ./static/js/
|
||||
mv temp/popper.js-1.14.3/dist/umd/popper.min.js.map ./static/js/
|
||||
mv temp/floating-ui-1.14.3/dist/umd/popper.min.js ./static/js/
|
||||
mv temp/floating-ui-1.14.3/dist/umd/popper.min.js.map ./static/js/
|
||||
|
||||
mv temp/Font-Awesome-${FONT_AWESOME_VERSION} temp/font-awesome
|
||||
|
||||
|
|
|
@ -85,8 +85,8 @@ if __name__ == "__main__":
|
|||
os.makedirs(rel_path)
|
||||
print('---- worker launched, uuid={} session_uuid={} epoch={}'.format(uuid, session_uuid, time.time()))
|
||||
else:
|
||||
print('Incorrect Stream, Closing worker: type={} session_uuid={}'.format(type, session_uuid))
|
||||
sys.exit(1)
|
||||
print('Incorrect message')
|
||||
redis_server_stream.sadd('working_session_uuid:{}'.format(type), session_uuid)
|
||||
|
||||
#LAUNCH a tcpdump
|
||||
|
@ -149,16 +149,16 @@ if __name__ == "__main__":
|
|||
except subprocess.TimeoutExpired:
|
||||
process_compressor.kill()
|
||||
### compress all files ###
|
||||
date = datetime.datetime.now().strftime("%Y%m%d")
|
||||
worker_data_directory = os.path.join(full_tcpdump_path, date[0:4], date[4:6], date[6:8])
|
||||
all_files = os.listdir(worker_data_directory)
|
||||
all_files.sort()
|
||||
if all_files:
|
||||
for file in all_files:
|
||||
if file.endswith('.cap'):
|
||||
full_path = os.path.join(worker_data_directory, file)
|
||||
if redis_server_stream.get('data_in_process:{}'.format(session_uuid)) != full_path:
|
||||
compress_file(full_path)
|
||||
if os.path.isdir(worker_data_directory):
|
||||
all_files = os.listdir(worker_data_directory)
|
||||
all_files.sort()
|
||||
if all_files:
|
||||
for file in all_files:
|
||||
if file.endswith('.cap'):
|
||||
full_path = os.path.join(worker_data_directory, file)
|
||||
if redis_server_stream.get('data_in_process:{}'.format(session_uuid)) != full_path:
|
||||
compress_file(full_path)
|
||||
### ###
|
||||
|
||||
#print(process.stderr.read())
|
||||
|
|
|
@ -0,0 +1,80 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
from meta_types_modules.MetaTypesDefault import MetaTypesDefault
|
||||
import hashlib
|
||||
import time
|
||||
import os
|
||||
import datetime
|
||||
import base64
|
||||
import shutil
|
||||
import gzip
|
||||
|
||||
class TypeHandler(MetaTypesDefault):
|
||||
|
||||
def __init__(self, uuid, json_file):
|
||||
super().__init__(uuid, json_file)
|
||||
self.compress = False
|
||||
self.extension = ''
|
||||
self.segregate = True
|
||||
if "compress" in json_file:
|
||||
self.compress = json_file['compress']
|
||||
if "extension" in json_file:
|
||||
self.extension = json_file['extension']
|
||||
if "segregate" in json_file:
|
||||
self.segregate = json_file['segregate']
|
||||
self.set_rotate_file_mode(False)
|
||||
self.saved_dir = ''
|
||||
|
||||
def process_data(self, data):
|
||||
# Unpack the thing
|
||||
self.reconstruct_data(data)
|
||||
|
||||
# pushing the filepath instead of the file content to the analyzer
|
||||
def handle_reconstructed_data(self, data):
|
||||
m = hashlib.sha256()
|
||||
self.set_last_time_saved(time.time())
|
||||
self.set_last_saved_date(datetime.datetime.now().strftime("%Y%m%d%H%M%S"))
|
||||
|
||||
# Create folder
|
||||
save_dir = os.path.join(self.get_save_dir(save_by_uuid=self.segregate), 'files')
|
||||
if not os.path.isdir(save_dir):
|
||||
os.makedirs(save_dir)
|
||||
# write file to disk
|
||||
decodeddata = base64.b64decode(data)
|
||||
|
||||
m.update(decodeddata)
|
||||
path = os.path.join(save_dir, m.hexdigest())
|
||||
path = '{}.{}'.format(path, self.extension)
|
||||
with open(path, 'wb') as p:
|
||||
p.write(decodeddata)
|
||||
if self.compress:
|
||||
compressed_filename = '{}.gz'.format(path)
|
||||
with open(path, 'rb') as f_in:
|
||||
with gzip.open(compressed_filename, 'wb') as f_out:
|
||||
shutil.copyfileobj(f_in, f_out)
|
||||
os.remove(path)
|
||||
self.send_to_analyzers(compressed_filename)
|
||||
else:
|
||||
self.send_to_analyzers(path)
|
||||
|
||||
def reconstruct_data(self, data):
|
||||
# save data in buffer
|
||||
self.add_to_buffer(data)
|
||||
data = self.get_buffer()
|
||||
|
||||
# end of element found in data
|
||||
if self.get_file_separator() in data:
|
||||
# empty buffer
|
||||
self.reset_buffer()
|
||||
all_line = data.split(self.get_file_separator())
|
||||
for reconstructed_data in all_line[:-1]:
|
||||
if reconstructed_data != b'':
|
||||
self.handle_reconstructed_data(reconstructed_data)
|
||||
|
||||
# save incomplete element in buffer
|
||||
if all_line[-1] != b'':
|
||||
self.add_to_buffer(all_line[-1])
|
||||
|
||||
|
||||
def test(self):
|
||||
print('Class: filewatcher')
|
|
@ -0,0 +1,38 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
from meta_types_modules.MetaTypesDefault import MetaTypesDefault
|
||||
import hashlib
|
||||
import time
|
||||
import os
|
||||
import datetime
|
||||
|
||||
class TypeHandler(MetaTypesDefault):
|
||||
|
||||
def __init__(self, uuid, json_file):
|
||||
super().__init__(uuid, json_file)
|
||||
self.set_rotate_file_mode(False)
|
||||
self.saved_dir = ''
|
||||
|
||||
def process_data(self, data):
|
||||
self.reconstruct_data(data)
|
||||
|
||||
# pushing the filepath instead of the file content to the analyzer
|
||||
def handle_reconstructed_data(self, data):
|
||||
m = hashlib.sha256()
|
||||
self.set_last_time_saved(time.time())
|
||||
self.set_last_saved_date(datetime.datetime.now().strftime("%Y%m%d%H%M%S"))
|
||||
|
||||
# Create folder
|
||||
jsons_save_dir = os.path.join(self.get_save_dir(save_by_uuid=True), 'files')
|
||||
if not os.path.isdir(jsons_save_dir):
|
||||
os.makedirs(jsons_save_dir)
|
||||
# write json file to disk
|
||||
m.update(data)
|
||||
jsons_path = os.path.join(jsons_save_dir, m.hexdigest()+'.json')
|
||||
with open(jsons_path, 'wb') as j:
|
||||
j.write(data)
|
||||
# Send data to Analyszer
|
||||
self.send_to_analyzers(jsons_path)
|
||||
|
||||
def test(self):
|
||||
print('Class: filewatcherjson')
|
|
@ -170,7 +170,7 @@ if __name__ == "__main__":
|
|||
redis_server_stream.delete(stream_name)
|
||||
try:
|
||||
if os.path.isfile(save_path):
|
||||
print('save')
|
||||
#print('save')
|
||||
compress_file(save_path)
|
||||
except NameError:
|
||||
pass
|
||||
|
|
|
@ -60,8 +60,8 @@ if __name__ == "__main__":
|
|||
rel_path = os.path.join(dir_path, filename)
|
||||
print('---- worker launched, uuid={} session_uuid={} epoch={}'.format(uuid, session_uuid, time.time()))
|
||||
else:
|
||||
print('Incorrect Stream, Closing worker: type={} session_uuid={}'.format(type, session_uuid))
|
||||
sys.exit(1)
|
||||
print('Incorrect message')
|
||||
|
||||
time_file = time.time()
|
||||
rotate_file = False
|
||||
|
|