Merge branch 'master' of github.com:D4-project/d4-core

server/.gitignore

@@ -4,3 +4,6 @@
 *.key
 data/
 logs/
+redis/
+D4ENV/
+dump.rdb

server/LAUNCH.sh

@@ -0,0 +1,279 @@
+#!/bin/bash
+
+GREEN="\\033[1;32m"
+DEFAULT="\\033[0;39m"
+RED="\\033[1;31m"
+ROSE="\\033[1;35m"
+BLUE="\\033[1;34m"
+WHITE="\\033[0;02m"
+YELLOW="\\033[1;33m"
+CYAN="\\033[1;36m"
+
+. ./D4ENV/bin/activate
+
+isredis=`screen -ls | egrep '[0-9]+.Redis_D4' | cut -d. -f1`
+isd4server=`screen -ls | egrep '[0-9]+.Server_D4' | cut -d. -f1`
+isworker=`screen -ls | egrep '[0-9]+.Workers_D4' | cut -d. -f1`
+isflask=`screen -ls | egrep '[0-9]+.Flask_D4' | cut -d. -f1`
+
+function helptext {
+    echo -e $YELLOW"
+         _______   __    __                                                                   __
+        /       \ /  |  /  |                                                                 /  |
+        \$\$\$\$\$\$\$  |\$\$ |  \$\$ |        ______    ______    ______      __   ______    _______  _\$\$ |_
+        \$\$ |  \$\$ |\$\$ |__\$\$ |       /      \  /      \  /      \    /  | /      \  /       |/ \$\$   |
+        \$\$ |  \$\$ |\$\$    \$\$ |      /\$\$\$\$\$\$  |/\$\$\$\$\$\$  |/\$\$\$\$\$\$  |   \$\$/ /\$\$\$\$\$\$  |/\$\$\$\$\$\$\$/ \$\$\$\$\$\$/
+        \$\$ |  \$\$ |\$\$\$\$\$\$\$\$ |      \$\$ |  \$\$ |\$\$ |  \$\$/ \$\$ |  \$\$ |   /  |\$\$    \$\$ |\$\$ |        \$\$ | __
+        \$\$ |__\$\$ |      \$\$ |      \$\$ |__\$\$ |\$\$ |      \$\$ \__\$\$ |   \$\$ |\$\$\$\$\$\$\$\$/ \$\$ \_____   \$\$ |/  |
+        \$\$    \$\$/       \$\$ |      \$\$    \$\$/ \$\$ |      \$\$    \$\$/    \$\$ |\$\$       |\$\$       |  \$\$  \$\$/
+        \$\$\$\$\$\$\$/        \$\$/       \$\$\$\$\$\$\$/  \$\$/        \$\$\$\$\$\$/__   \$\$ | \$\$\$\$\$\$\$/  \$\$\$\$\$\$\$/    \$\$\$\$/
+                                  \$\$ |                       /  \__\$\$ |
+                                  \$\$ |                       \$\$    \$\$/
+                                  \$\$/                         \$\$\$\$\$\$/
+
+    "$DEFAULT"
+    This script launch:    (Inside screen Daemons)"$CYAN"
+      - D4 Twisted server.
+      - All wokers manager.
+      - All Redis in memory servers.
+      - Flak server.
+
+    Usage:    LAUNCH.sh
+                  [-l | --launchAuto]
+                  [-k | --killAll]
+                  [-h | --help]
+    "
+}
+
+function launching_redis {
+    conf_dir="${D4_HOME}/configs/"
+    redis_dir="${D4_HOME}/redis/src/"
+
+    screen -dmS "Redis_D4"
+    sleep 0.1
+    echo -e $GREEN"\t* Launching D4 Redis ervers"$DEFAULT
+    screen -S "Redis_D4" -X screen -t "6379" bash -c $redis_dir'redis-server '$conf_dir'6379.conf ; read x'
+    sleep 0.1
+    screen -S "Redis_D4" -X screen -t "6380" bash -c $redis_dir'redis-server '$conf_dir'6380.conf ; read x'
+    sleep 0.1
+}
+
+function launching_d4_server {
+    screen -dmS "Server_D4"
+    sleep 0.1
+    echo -e $GREEN"\t* Launching D4 Server"$DEFAULT
+
+    screen -S "Server_D4" -X screen -t "Server_D4" bash -c "cd ${D4_HOME}; ./server.py -v 10; read x"
+    sleep 0.1
+}
+
+function launching_workers {
+    screen -dmS "Workers_D4"
+    sleep 0.1
+    echo -e $GREEN"\t* Launching D4 Workers"$DEFAULT
+
+    screen -S "Workers_D4" -X screen -t "1_workers_manager" bash -c "cd ${D4_HOME}/workers/workers_1; ./workers_manager.py; read x"
+    sleep 0.1
+    screen -S "Workers_D4" -X screen -t "4_workers_manager" bash -c "cd ${D4_HOME}/workers/workers_4; ./workers_manager.py; read x"
+    sleep 0.1
+}
+
+function shutting_down_redis {
+    redis_dir=${D4_HOME}/redis/src/
+    bash -c $redis_dir'redis-cli -p 6379 SHUTDOWN'
+    sleep 0.1
+    bash -c $redis_dir'redis-cli -p 6380 SHUTDOWN'
+    sleep 0.1
+}
+
+function checking_redis {
+    flag_redis=0
+    redis_dir=${D4_HOME}/redis/src/
+    bash -c $redis_dir'redis-cli -p 6379 PING | grep "PONG" &> /dev/null'
+    if [ ! $? == 0 ]; then
+       echo -e $RED"\t6379 not ready"$DEFAULT
+       flag_redis=1
+    fi
+    sleep 0.1
+    bash -c $redis_dir'redis-cli -p 6380 PING | grep "PONG" &> /dev/null'
+    if [ ! $? == 0 ]; then
+       echo -e $RED"\t6380 not ready"$DEFAULT
+       flag_redis=1
+    fi
+    sleep 0.1
+
+    return $flag_redis;
+}
+
+function launch_redis {
+    if [[ ! $isredis ]]; then
+        launching_redis;
+    else
+        echo -e $RED"\t* A D4_Redis screen is already launched"$DEFAULT
+    fi
+}
+
+function launch_d4_server {
+    if [[ ! $isd4server ]]; then
+      sleep 1
+        if checking_redis; then
+            launching_d4_server;
+        else
+            echo -e $YELLOW"\tD4 Redis not started, waiting 5 more secondes"$DEFAULT
+            sleep 5
+            if checking_redis; then
+                launching_d4_server;
+            else
+                echo -e $RED"\tError: Redis not started"$DEFAULT
+                exit 1
+            fi;
+        fi;
+    else
+        echo -e $RED"\t* A Server_D4 screen is already launched"$DEFAULT
+    fi
+}
+
+function launch_workers {
+    if [[ ! $isworker ]]; then
+      sleep 1
+        if checking_redis; then
+            launching_workers;
+        else
+            echo -e $YELLOW"\tD4 Redis not started, waiting 5 more secondes"$DEFAULT
+            sleep 5
+            if checking_redis; then
+                launching_workers;
+            else
+                echo -e $RED"\tError: Redis not started"$DEFAULT
+                exit 1
+            fi;
+        fi;
+    else
+        echo -e $RED"\t* A Workers_D4 screen is already launched"$DEFAULT
+    fi
+}
+
+function launch_flask {
+    if [[ ! $isflask ]]; then
+        flask_dir=${D4_HOME}/web
+        screen -dmS "Flask_D4"
+        sleep 0.1
+        echo -e $GREEN"\t* Launching Flask server"$DEFAULT
+        screen -S "Flask_D4" -X screen -t "Flask_server" bash -c "cd $flask_dir; ls; ./Flask_server.py; read x"
+    else
+        echo -e $RED"\t* A Flask_D4 screen is already launched"$DEFAULT
+    fi
+}
+
+function killall {
+
+
+    if [[ $isredis || $isd4server || $isworker || $isflask ]]; then
+        echo -e $GREEN"\t* Gracefully closing D4 Servers ..."$DEFAULT
+        kill -SIGINT $isd4server
+        sleep 3
+        kill $isd4server $isflask
+        echo -e $GREEN"\t* Gracefully closing D4 Workers ..."$DEFAULT
+        kill -SIGINT $isworker
+        sleep 0.5
+        kill $isworker
+        echo -e $GREEN"\t* $isd4server $isworker $isflask killed."$DEFAULT
+        echo -e $GREEN"\t* Gracefully closing redis servers ..."$DEFAULT
+        shutting_down_redis;
+        kill $isredis
+        sleep 0.2
+        echo -e $ROSE`screen -ls`$DEFAULT
+    else
+        echo -e $RED"\t* No screen to kill"$DEFAULT
+    fi
+}
+
+function update_web {
+    echo -e "\t* Updating web..."
+    bash -c "(cd ${D4_HOME}/web; ./update_web.sh)"
+    exitStatus=$?
+    if [ $exitStatus -ge 1 ]; then
+        echo -e $RED"\t* Web not up-to-date"$DEFAULT
+        exit
+    else
+        echo -e $GREEN"\t* Web updated"$DEFAULT
+    fi
+}
+
+function launch_all {
+    helptext;
+    launch_redis;
+    launch_d4_server;
+    launch_workers;
+    launch_flask;
+}
+
+#If no params, display the menu
+[[ $@ ]] || {
+
+    helptext;
+
+    options=("Redis" "D4-Server" "Workers-manager" "Flask" "Killall" "Update-web")
+
+    menu() {
+        echo "What do you want to Launch?:"
+        for i in ${!options[@]}; do
+            printf "%3d%s) %s\n" $((i+1)) "${choices[i]:- }" "${options[i]}"
+        done
+        [[ "$msg" ]] && echo "$msg"; :
+    }
+
+    prompt="Check an option (again to uncheck, ENTER when done): "
+    while menu && read -rp "$prompt" numinput && [[ "$numinput" ]]; do
+        for num in $numinput; do
+            [[ "$num" != *[![:digit:]]* ]] && (( num > 0 && num <= ${#options[@]} )) || {
+                msg="Invalid option: $num"; break
+            }
+            ((num--)); msg="${options[num]} was ${choices[num]:+un}checked"
+            [[ "${choices[num]}" ]] && choices[num]="" || choices[num]="+"
+        done
+    done
+
+    for i in ${!options[@]}; do
+        if [[ "${choices[i]}" ]]; then
+            case ${options[i]} in
+                Redis)
+                    launch_redis
+                    ;;
+                D4-Server)
+                    launch_d4_server;
+                    ;;
+                Workers-manager)
+                    launch_workers;
+                    ;;
+                Flask)
+                    launch_flask;
+                    ;;
+                Killall)
+                    killall;
+                    ;;
+                Update-web)
+                    update_web;
+                    ;;
+            esac
+        fi
+    done
+
+    exit
+}
+
+while [ "$1" != "" ]; do
+    case $1 in
+        -l | --launchAuto )         launch_all;
+                                    ;;
+        -k | --killAll )            helptext;
+                                    killall;
+                                    ;;
+        -h | --help )               helptext;
+                                    exit
+                                    ;;
+        * )                         helptext
+                                    exit 1
+    esac
+    shift
+done

server/configs/6379.conf

@@ -174,7 +174,7 @@ rdbcompression yes
 rdbchecksum yes
 
 # The filename where to dump the DB
-dbfilename dump.rdb
+dbfilename dump6379.rdb
 
 # The working directory.
 #

server/configs/6380.conf

@@ -174,7 +174,7 @@ rdbcompression yes
 rdbchecksum yes
 
 # The filename where to dump the DB
-dbfilename dump.rdb
+dbfilename dump6880.rdb
 
 # The working directory.
 #

server/gen_cert.sh

@@ -1,7 +0,0 @@
-#!/usr/bin/env bash 
-openssl genrsa -out server.key 4096
-openssl req -sha256 -new -key server.key -out server.csr -subj '/CN=localhost'
-openssl x509 -req -sha256 -days 365 -in server.csr -signkey server.key -out server.crt
-openssl x509 -req -in server.csr -CA rootCA.crt -CAkey rootCA.key -CAcreateserial -out server.crt -days 500 -sha256
-cat server.crt server.key > server.pem
-

server/gen_cert/README.md

@@ -0,0 +1,14 @@
+Usage
+=====
+These scripts are useful to generate the server's self-signed certificate.
+## Root Certification Authority ##
+`gen_root.sh`
+This will create a rootCA.crt file that you may copy in the d4-goclient conf folder to verify the server's cert.
+## Server certificate ##
+### Configuration Files ###
+Look into these files if you plan to verify the certificate:
+  * san.cnf holds basic information about the certificate.
+  * ext.3 holds in particular the 'subjectAltNames** option that is for the verification to succeed.
+
+ ### Generation ###
+`gen_root.sh`

server/gen_cert/ext3.cnf

@@ -0,0 +1,5 @@
+keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment
+subjectAltName = @alt_names
+
+[alt_names]
+IP.1 = 127.0.0.1

server/gen_cert/gen_cert.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash 
+# Create Server key
+openssl genrsa -out server.key 4096
+# Create the Server Signing Request - non interactive, config in san.cnf
+openssl req -sha256 -new -key server.key -out server.csr -config san.cnf
+# Create the server certificate by rootCA, with ext3 subjectAltName in ext3.cnf
+openssl x509 -req -in server.csr -CA rootCA.crt -CAkey rootCA.key -CAcreateserial -out server.crt -days 500 -sha256 -extfile ext3.cnf
+# Concat in pem
+cat server.crt server.key > ../server.pem

server/gen_cert/gen_root.sh

@@ -1,3 +1,5 @@
 #!/usr/bin/env bash 
+# Create Root key
 openssl genrsa -out rootCA.key 4096
+# Create and Sign the Root CA Certificate
 openssl req -x509 -new -nodes -key rootCA.key -sha256 -days 1024 -out rootCA.crt

server/gen_cert/san.cnf

@@ -0,0 +1,11 @@
+[req]
+default_bits = 4096
+prompt = no
+default_md = sha256
+distinguished_name = dn
+
+[dn]
+C=LU
+L=Luxembourg
+O=SMILE, CIRCL
+CN = localhost

server/install_server.sh

@@ -3,7 +3,20 @@
 set -e
 set -x
 
-sudo apt-get install python3-pip -y
+sudo apt-get install python3-pip virtualenv screen -y
+
+if [ -z "$VIRTUAL_ENV" ]; then
+    virtualenv -p python3 D4ENV
+    echo export D4_HOME=$(pwd) >> ./D4ENV/bin/activate
+    . ./D4ENV/bin/activate
+fi
+python3 -m pip install -r requirement.txt
+
+pushd web/
+./update_web.sh
+popd
+
+
 python3 -m pip install -r requirement.txt
 
 # REDIS #

server/requirement.txt

@@ -1,5 +1,6 @@
 twisted[tls]
 redis
+flask
 
 #sudo python3 -m pip install --upgrade service_identity
 

server/save_data.py

@@ -1,83 +0,0 @@
-#!/usr/bin/env python3
-
-import os
-import sys
-import redis
-import time
-import gzip
-import datetime
-
-redis_server = redis.StrictRedis(
-                    host="localhost",
-                    port=6379,
-                    db=0)
-
-type = 1
-max_timestamp = 60*5
-
-def gzip_file(filepath):
-    with open(filepath, 'rb') as f:
-        content = f.read()
-    with gzip.open(filepath+'.gz', 'wb') as f2:
-        f2.write(content)
-    os.remove(filepath)
-
-if __name__ == "__main__":
-    stream_name = 'stream:{}'.format(type)
-
-    #group_name = 'group_stream:{}'.format(type)
-    #try:
-    #    redis_server.xgroup_create(stream_name, group_name)
-    #except:
-    #    pass
-
-    while True:
-
-        #print(redis_server.xpending(stream_name, group_name))
-
-        #res = redis_server.xreadgroup(group_name, 'consumername', {stream_name: '>'}, count=1)
-        res = redis_server.xread({stream_name: '0'}, count=1, block=500)
-        if res:
-            id = res[0][1][0][0]
-            data = res[0][1][0][1]
-            if id and data:
-                #print(id.decode())
-                #print(data)
-
-                date = datetime.datetime.now().strftime("%Y/%m/%d")
-                dir_path = os.path.join('data', date, data[b'uuid'].decode())
-                filename = ''
-                data_timestamp = data[b'timestamp'].decode()
-
-                try:
-                    it = os.scandir(dir_path)
-                    for entry in it:
-                        if not entry.name.endswith(".gz") and entry.is_file():
-                            filename = entry.name
-                            break
-                    filepath = os.path.join(dir_path, filename)
-
-                    #if os.path.getsize(filepath) > 500000000: #bytes
-                    #    gzip_file(filepath)
-                    #    filename = data_timestamp
-
-                except FileNotFoundError:
-                    os.makedirs(dir_path)
-                # # TODO:  use contexte manager in python 3.6
-                it = []
-                # #
-
-                if not filename:
-                    filename = data_timestamp
-
-                if int(data_timestamp) - int(filename) > max_timestamp:
-                    gzip_file(filepath)
-                    filename = data_timestamp
-
-                with open(os.path.join(dir_path, filename), 'ab') as f:
-                    f.write(data[b'message'])
-
-                #redis_server.xack(stream_name, group_name, id)
-                redis_server.xdel(stream_name, id)
-        else:
-            time.sleep(10)

server/server.py

@@ -29,7 +29,7 @@ timeout_time = 30
 
 header_size = 62
 
-data_default_size_limit = 100000
+data_default_size_limit = 1000000
 default_max_entries_by_stream = 10000
 
 host_redis_stream = "localhost"
@@ -113,7 +113,14 @@ class Echo(Protocol, TimeoutMixin):
             # check default size limit
             if data_header['size'] > data_default_size_limit:
                 self.transport.abortConnection()
-                logger.warning('Incorrect data size: the server received more data than expected by default, expected={}, received={} , uuid={}, session_uuid={}'.format(data_default_size_limit, data_header['size'] ,data_header['uuid_header'], self.session_uuid))
+                logger.warning('Incorrect header data size: the server received more data than expected by default, expected={}, received={} , uuid={}, session_uuid={}'.format(data_default_size_limit, data_header['size'] ,data_header['uuid_header'], self.session_uuid))
+
+            # Worker: Incorrect type
+            if redis_server_stream.sismember('Error:IncorrectType:{}'.format(data_header['type']), self.session_uuid):
+                self.transport.abortConnection()
+                redis_server_stream.delete(stream_name)
+                redis_server_stream.srem('Error:IncorrectType:{}'.format(data_header['type']), self.session_uuid)
+                logger.warning('Incorrect type={} detected by worker, uuid={}, session_uuid={}'.format(data_header['type'] ,data_header['uuid_header'], self.session_uuid))
 
         return data_header
 

server/test_ctypes.py

@@ -1,31 +0,0 @@
-#!/usr/bin/env python3
-# -*-coding:UTF-8 -*
-
-from ctypes import *
-
-class D4Header(Structure):
-    _fields_ = [
-        ("version", c_uint8),
-        ("type", c_uint8),
-        #("uuid", c_uint128),
-        ("uuid", c_uint64),
-        ("timestamp", c_uint64),
-        #("hmac", c_uint256),
-        ("hmac", c_uint64),
-        ("size", c_uint32),
-        ]
-
-def pack(ctype_instance):
-    return string_at(byref(ctype_instance), sizeof(ctype_instance))
-
-def unpack(ctype, buffer):
-    c_str = create_string_buffer(buffer)
-    return cast(pointer(c_str), POINTER(ctype)).contents
-
-if __name__ == '__main__':
-    d4h = D4Header(1,1,88888,1543398024,233243445342,342)
-    print(d4h.version)
-    buffer = pack(d4h)
-    print(buffer)
-    d = unpack(D4Header, buffer)
-    assert(d4h.version == d.version)

server/web/templates/index.html

@@ -23,7 +23,7 @@
     }
     #chart{
     width:600px;
-    height:1000px;
+    height:500px;
     }
     .bar{
     fill:#eaeaea;
@@ -70,7 +70,7 @@ var setup = function(targetID){
 	//Set size of svg element and chart
 	var margin = {top: 0, right: 0, bottom: 0, left: 0},
 		width = 600 - margin.left - margin.right,
-		height = 1000 - margin.top - margin.bottom,
+		height = 500 - margin.top - margin.bottom,
 		categoryIndent = 4*15 + 5,
 		defaultBarWidth = 2000;
 

server/workers/workers_1/worker.py

@@ -8,18 +8,36 @@ import subprocess
 
 import datetime
 
-def data_incorrect_format(session_uuid):
+def data_incorrect_format(stream_name, session_uuid, uuid):
+    redis_server_stream.sadd('Error:IncorrectType:{}'.format(type), session_uuid)
+    redis_server_metadata.hset('metadata_uuid:{}'.format(uuid), 'Error', 'Error: Type={}, Incorrect file format'.format(type))
+    clean_stream(stream_name, session_uuid)
     print('Incorrect format')
     sys.exit(1)
 
+def clean_stream(stream_name, session_uuid):
+    redis_server_stream.srem('ended_session', session_uuid)
+    redis_server_stream.srem('session_uuid:{}'.format(type), session_uuid)
+    redis_server_stream.srem('working_session_uuid:{}'.format(type), session_uuid)
+    redis_server_stream.hdel('map-type:session_uuid-uuid:{}'.format(type), session_uuid)
+    redis_server_stream.delete(stream_name)
+
 host_redis_stream = "localhost"
 port_redis_stream = 6379
 
+host_redis_metadata = "localhost"
+port_redis_metadata = 6380
+
 redis_server_stream = redis.StrictRedis(
                     host=host_redis_stream,
                     port=port_redis_stream,
                     db=0)
 
+redis_server_metadata = redis.StrictRedis(
+                    host=host_redis_metadata,
+                    port=port_redis_metadata,
+                    db=0)
+
 type = 1
 tcp_dump_cycle = '300'
 stream_buffer = 100
@@ -51,7 +69,7 @@ if __name__ == "__main__":
     redis_server_stream.sadd('working_session_uuid:{}'.format(type), session_uuid)
 
     #LAUNCH a tcpdump
-    process = subprocess.Popen(["tcpdump", '-n', '-r', '-', '-G', tcp_dump_cycle, '-w', '{}/%Y/%m/%d/{}-%Y-%m-%d-%H%M%S.cap'.format(tcpdump_path, uuid)], stdin=subprocess.PIPE)
+    process = subprocess.Popen(["tcpdump", '-n', '-r', '-', '-G', tcp_dump_cycle, '-w', '{}/%Y/%m/%d/{}-%Y-%m-%d-%H%M%S.cap'.format(tcpdump_path, uuid)], stdin=subprocess.PIPE, stderr=subprocess.PIPE)
     nb_save = 0
 
     while True:
@@ -78,7 +96,7 @@ if __name__ == "__main__":
                     except:
                         Error_message = process.stderr.read()
                         if Error_message == b'tcpdump: unknown file format\n':
-                            data_incorrect_format(session_uuid)
+                            data_incorrect_format(stream_name, session_uuid, uuid)
 
                         #print(process.stdout.read())
                     nb_save += 1
@@ -95,7 +113,7 @@ if __name__ == "__main__":
                 out, err = process.communicate(timeout= 0.5)
                 #print(out)
                 if err == b'tcpdump: unknown file format\n':
-                    data_incorrect_format(session_uuid)
+                    data_incorrect_format(stream_name, session_uuid, uuid)
                 elif err:
                     print(err)