Merge pull request #50 from cudeso/master

Contributions to the documentation and small type for "registered"

README.md

@@ -64,10 +64,31 @@ git submodule init
 git submodule update
 ~~~~
 
+Build the d4 client. This will create the `d4` binary.
+
+~~~~
+make
+~~~~
+
+Then register the sensor with the server. Replace `API_TOKEN`, `VALID_UUID4` (create a random UUID via [UUIDgenerator](https://www.uuidgenerator.net/)) and `VALID_HMAC_KEY`.
+
+~~~~
+curl -k https://127.0.0.1:7000/api/v1/add/sensor/register --header "Authorization: API_TOKEN" -H "Content-Type: application/json" --data '{"uuid":"VALID_UUID4","hmac_key":"VALID_HMAC_KEY"}' -X POST
+~~~~
+
+If the registration went correctly the UUID is returned. Do not forget to approve the registration in the D4 server web interface.
+
+Update the configuration file
+
+~~~~
+cp -r conf.sample conf
+echo VALID_UUID4 > conf/uuid
+echo VALID_HMAC_KEY > conf/key
+~~~~
+
 ## D4 core server
 
-D4 core server is a complete server to handle clients (sensors) including the decapsulation of the [D4 protocol](https://github.com/D4-project/architecture/tree/master/format), control of
-sensor registrations, management of decoding protocols and dispatching to adequate decoders/analysers.
+D4 core server is a complete server to handle clients (sensors) including the decapsulation of the [D4 protocol](https://github.com/D4-project/architecture/tree/master/format), control of sensor registrations, management of decoding protocols and dispatching to adequate decoders/analysers.
 
 ### Requirements
 
@@ -76,7 +97,6 @@ sensor registrations, management of decoding protocols and dispatching to adequa
 
 ### Installation
 
-
 - [Install D4 Server](https://github.com/D4-project/d4-core/tree/master/server)
 
 ### D4 core server Screenshots

client/Makefile

@@ -32,7 +32,7 @@ clean:
 	- rm -rf *.o hmac
 
 d4:	d4.o sha2.o hmac.o unpack.o unparse.o pack.o gen_uuid.o randutils.o parse.o
-	gcc -Wall -o d4 d4.o hmac.o sha2.o unpack.o pack.o unparse.o gen_uuid.o randutils.o parse.o
+	$(CC) -Wall -o d4 d4.o hmac.o sha2.o unpack.o pack.o unparse.o gen_uuid.o randutils.o parse.o
 
 d4.o:	d4.c
-	gcc -Wall -c d4.c
+	$(CC) -Wall -c d4.c

server/README.md

@@ -15,11 +15,24 @@ sensor registrations, management of decoding protocols and dispatching to adequa
 ### Installation
 
 ###### Install D4 server
+
+Clone the repository and install necessary packages. Installation requires *sudo* permissions.
+
 ~~~~
+git clone https://github.com/D4-project/d4-core.git
+cd d4-core
 cd server
 ./install_server.sh
 ~~~~
-Create or add a pem in [d4-core/server](https://github.com/D4-project/d4-core/tree/master/server) :
+
+When the installation is finished, scroll back to where `+ ./create_default_user.py` is displayed. The next lines contain the default generated user and should resemble the snippet below. Take a temporary note of the password, you are required to **change the password** on first login.
+~~~~
+new user created: admin@admin.test
+password: <redacted>
+token: <redacted>
+~~~~
+
+Then create or add a pem in [d4-core/server](https://github.com/D4-project/d4-core/tree/master/server) :
 ~~~~
 cd gen_cert
 ./gen_root.sh
@@ -27,7 +40,6 @@ cd gen_cert
 cd ..
 ~~~~
 
-
 ###### Launch D4 server
 ~~~~
 ./LAUNCH.sh -l
@@ -35,6 +47,14 @@ cd ..
 
 The web interface is accessible via `http://127.0.0.1:7000/`
 
+If you cannot access the web interface on localhost (for example because the system is running on a remote host), then stop the server, change the listening host IP and restart the server. In the below example it's changed to `0.0.0.0` (all interfaces). Make sure that the IP is not unintentionally publicly exposed.
+
+~~~~
+./LAUNCH.sh -k
+sed -i '/\[Flask_Server\]/{:a;N;/host = 127\.0\.0\.1/!ba;s/host = 127\.0\.0\.1/host = 0.0.0.0/}' configs/server.conf
+./LAUNCH.sh -l
+~~~~
+
 ### Updating web assets
 To update javascript libs run:
 ~~~~
@@ -46,7 +66,6 @@ cd web
 
 [API Documentation](https://github.com/D4-project/d4-core/tree/master/server/documentation/README.md)
 
-
 ### Notes
 
 - All server logs are located in ``d4-core/server/logs/``
@@ -85,3 +104,7 @@ Run the following command as root:
 ~~~~
 aa-complain /usr/sbin/tcpdump
 ~~~~
+
+###### WARNING - Not registered UUID=UUID4, connection closed
+
+This happens after you have registered a new sensor, but have not approved the registration. In order to approve the sensor, go in the web interface to **Server Management**, and click **Pending Sensors**.

server/install_server.sh

@@ -30,8 +30,8 @@ git checkout 5.0
 make
 popd
 
-# LAUNCH Redis
-bash ${AIL_BIN}LAUNCH.sh -lrv &
+# LAUNCH
+bash LAUNCH.sh -l &
 wait
 echo ""
 

server/lib/Sensor.py

@@ -90,7 +90,7 @@ def register_sensor(req_dict):
     sensor_uuid = sensor_uuid.replace('-', '')
     # sensor already exist
     if r_serv_db.exists('metadata_uuid:{}'.format(sensor_uuid)):
-        return ({"status": "error", "reason": "Sensor already registred"}, 409)
+        return ({"status": "error", "reason": "Sensor already registered"}, 409)
 
     # hmac key
     if not hmac_key:

server/requirement.txt

@@ -1,8 +1,9 @@
 twisted[tls]
 redis
-flask
+flask==2.2.2
 flask-login
 bcrypt
+Werkzeug==2.2.2
 
 #sudo python3 -m pip install --upgrade service_identity
 

server/server.py

@@ -144,10 +144,10 @@ def extract_ip(ip_string):
         return ip_string
 
 def server_mode_registration(header_uuid):
-    # only accept registred uuid
+    # only accept registered uuid
     if server_mode == 'registration':
         if not redis_server_metadata.sismember('registered_uuid', header_uuid):
-            error_msg = 'Not registred UUID={}, connection closed'.format(header_uuid)
+            error_msg = 'Not registered UUID={}, connection closed'.format(header_uuid)
             print(error_msg)
             logger.warning(error_msg)
             #redis_server_metadata.hset('metadata_uuid:{}'.format(data_header['uuid_header']), 'Error', 'Error: This UUID is temporarily blacklisted')
@@ -255,8 +255,8 @@ class D4_Server(Protocol, TimeoutMixin):
                 if not redis_server_stream.exists('active_connection_by_uuid:{}'.format(self.uuid)):
                     redis_server_stream.srem('active_connection', self.uuid)
 
-                logger.debug('Connection closed: session_uuid={}'.format(self.session_uuid))
-                dict_all_connection.pop(self.session_uuid)
+            logger.debug('Connection closed: session_uuid={}'.format(self.session_uuid))
+            dict_all_connection.pop(self.session_uuid)
 
     def unpack_header(self, data):
         data_header = {}
@@ -269,6 +269,20 @@ class D4_Server(Protocol, TimeoutMixin):
             data_header['size'] = struct.unpack('I', data[58:62])[0]
         return data_header
 
+    def check_hmac_key(self, hmac_header, data):
+        if self.hmac_key is None:
+            self.hmac_key = redis_server_metadata.hget('metadata_uuid:{}'.format(self.uuid), 'hmac_key')
+            if self.hmac_key is None:
+                self.hmac_key = redis_server_metadata.get('server:hmac_default_key')
+
+        # set hmac_header to 0
+        data = data.replace(hmac_header, hmac_reset, 1)
+
+        HMAC = hmac.new(self.hmac_key, msg=data, digestmod='sha256')
+        hmac_header = hmac_header.hex()
+        # hmac match
+        return hmac_header == HMAC.hexdigest()
+
     def check_connection_validity(self, data_header):
         # blacklist ip by uuid
         if redis_server_metadata.sismember('blacklist_ip_by_uuid', data_header['uuid_header']):
@@ -345,8 +359,14 @@ class D4_Server(Protocol, TimeoutMixin):
                         self.type = data_header['type']
                         self.uuid = data_header['uuid_header']
 
-                        # worker entry point: map type:session_uuid
-                        redis_server_stream.sadd('session_uuid:{}'.format(data_header['type']), self.session_uuid.encode())
+                        # # check HMAC /!\ incomplete data
+                        # if not self.check_hmac_key(data_header['hmac_header'], data):
+                        #     print('hmac do not match')
+                        #     print(data)
+                        #     logger.debug("HMAC don't match, uuid={}, session_uuid={}".format(self.uuid, self.session_uuid))
+                        #     redis_server_metadata.hset('metadata_uuid:{}'.format(data_header['uuid_header']), 'Error', 'Error: HMAC don\'t match')
+                        #     self.transport.abortConnection()
+                        #     return 1
 
                         ## save active connection ##
                         #active Connection
@@ -473,15 +493,6 @@ class D4_Server(Protocol, TimeoutMixin):
     def process_d4_data(self, data, data_header, ip):
         # empty buffer
         self.buffer = b''
-        # set hmac_header to 0
-        data = data.replace(data_header['hmac_header'], hmac_reset, 1)
-        if self.hmac_key is None:
-            self.hmac_key = redis_server_metadata.hget('metadata_uuid:{}'.format(data_header['uuid_header']), 'hmac_key')
-            if self.hmac_key is None:
-                self.hmac_key = redis_server_metadata.get('server:hmac_default_key')
-
-        HMAC = hmac.new(self.hmac_key, msg=data, digestmod='sha256')
-        data_header['hmac_header'] = data_header['hmac_header'].hex()
 
         ### Debug ###
         #print('hexdigest: {}'.format( HMAC.hexdigest() ))
@@ -494,7 +505,7 @@ class D4_Server(Protocol, TimeoutMixin):
         ###       ###
 
         # hmac match
-        if data_header['hmac_header'] == HMAC.hexdigest():
+        if self.check_hmac_key(data_header['hmac_header'], data):
             if not self.stream_max_size:
                 temp = redis_server_metadata.hget('stream_max_size_by_uuid', data_header['uuid_header'])
                 if temp is not None:
@@ -526,6 +537,9 @@ class D4_Server(Protocol, TimeoutMixin):
                 redis_server_metadata.hset('metadata_type_by_uuid:{}:{}'.format(data_header['uuid_header'], data_header['type']), 'last_seen', d4_packet_rcv_time)
 
                 if not self.data_saved:
+                    # worker entry point: map type:session_uuid
+                    redis_server_stream.sadd('session_uuid:{}'.format(data_header['type']), self.session_uuid.encode())
+
                     #UUID IP:           ## TODO: use d4 timestamp ?
                     redis_server_metadata.lpush('list_uuid_ip:{}'.format(data_header['uuid_header']), '{}-{}'.format(ip, datetime.datetime.now().strftime("%Y%m%d%H%M%S")))
                     redis_server_metadata.ltrim('list_uuid_ip:{}'.format(data_header['uuid_header']), 0, 15)

server/web/update_web.sh

@@ -47,8 +47,8 @@ mv temp/bootstrap-${BOOTSTRAP_VERSION}-dist/js/bootstrap.min.js ./static/js/
 mv temp/bootstrap-${BOOTSTRAP_VERSION}-dist/css/bootstrap.min.css ./static/css/
 mv temp/bootstrap-${BOOTSTRAP_VERSION}-dist/css/bootstrap.min.css.map ./static/css/
 
-mv temp/popper-core-1.14.3/dist/umd/popper.min.js ./static/js/
-mv temp/popper-core-1.14.3/dist/umd/popper.min.js.map ./static/js/
+mv temp/floating-ui-1.14.3/dist/umd/popper.min.js ./static/js/
+mv temp/floating-ui-1.14.3/dist/umd/popper.min.js.map ./static/js/
 
 mv temp/Font-Awesome-${FONT_AWESOME_VERSION} temp/font-awesome
 

server/workers/workers_1/worker.py

@@ -85,8 +85,8 @@ if __name__ == "__main__":
             os.makedirs(rel_path)
         print('----    worker launched, uuid={} session_uuid={} epoch={}'.format(uuid, session_uuid, time.time()))
     else:
+        print('Incorrect Stream, Closing worker: type={} session_uuid={}'.format(type, session_uuid))
         sys.exit(1)
-        print('Incorrect message')
     redis_server_stream.sadd('working_session_uuid:{}'.format(type), session_uuid)
 
     #LAUNCH a tcpdump
@@ -149,16 +149,16 @@ if __name__ == "__main__":
                 except subprocess.TimeoutExpired:
                     process_compressor.kill()
                     ### compress all files ###
-                    date = datetime.datetime.now().strftime("%Y%m%d")
                     worker_data_directory = os.path.join(full_tcpdump_path, date[0:4], date[4:6], date[6:8])
-                    all_files = os.listdir(worker_data_directory)
-                    all_files.sort()
-                    if all_files:
-                        for file in all_files:
-                            if file.endswith('.cap'):
-                                full_path = os.path.join(worker_data_directory, file)
-                                if redis_server_stream.get('data_in_process:{}'.format(session_uuid)) != full_path:
-                                    compress_file(full_path)
+                    if os.path.isdir(worker_data_directory):
+                        all_files = os.listdir(worker_data_directory)
+                        all_files.sort()
+                        if all_files:
+                            for file in all_files:
+                                if file.endswith('.cap'):
+                                    full_path = os.path.join(worker_data_directory, file)
+                                    if redis_server_stream.get('data_in_process:{}'.format(session_uuid)) != full_path:
+                                        compress_file(full_path)
                     ### ###
 
                 #print(process.stderr.read())

server/workers/workers_4/worker.py

@@ -60,8 +60,8 @@ if __name__ == "__main__":
         rel_path = os.path.join(dir_path, filename)
         print('----    worker launched, uuid={} session_uuid={} epoch={}'.format(uuid, session_uuid, time.time()))
     else:
+        print('Incorrect Stream, Closing worker: type={} session_uuid={}'.format(type, session_uuid))
         sys.exit(1)
-        print('Incorrect message')
 
     time_file = time.time()
     rotate_file = False