diff --git a/server/.gitignore b/server/.gitignore index 832729a..7d17db7 100644 --- a/server/.gitignore +++ b/server/.gitignore @@ -4,3 +4,6 @@ *.key data/ logs/ +redis/ +D4ENV/ +dump.rdb diff --git a/server/LAUNCH.sh b/server/LAUNCH.sh new file mode 100755 index 0000000..75fa2fc --- /dev/null +++ b/server/LAUNCH.sh @@ -0,0 +1,278 @@ +#!/bin/bash + +GREEN="\\033[1;32m" +DEFAULT="\\033[0;39m" +RED="\\033[1;31m" +ROSE="\\033[1;35m" +BLUE="\\033[1;34m" +WHITE="\\033[0;02m" +YELLOW="\\033[1;33m" +CYAN="\\033[1;36m" + +. ./D4ENV/bin/activate + +isredis=`screen -ls | egrep '[0-9]+.Redis_D4' | cut -d. -f1` +isd4server=`screen -ls | egrep '[0-9]+.Server_D4' | cut -d. -f1` +isworker=`screen -ls | egrep '[0-9]+.Workers_D4' | cut -d. -f1` +isflask=`screen -ls | egrep '[0-9]+.Flask_D4' | cut -d. -f1` + +function helptext { + echo -e $YELLOW" + _______ __ __ __ + / \ / | / | / | + \$\$\$\$\$\$\$ |\$\$ | \$\$ | ______ ______ ______ __ ______ _______ _\$\$ |_ + \$\$ | \$\$ |\$\$ |__\$\$ | / \ / \ / \ / | / \ / |/ \$\$ | + \$\$ | \$\$ |\$\$ \$\$ | /\$\$\$\$\$\$ |/\$\$\$\$\$\$ |/\$\$\$\$\$\$ | \$\$/ /\$\$\$\$\$\$ |/\$\$\$\$\$\$\$/ \$\$\$\$\$\$/ + \$\$ | \$\$ |\$\$\$\$\$\$\$\$ | \$\$ | \$\$ |\$\$ | \$\$/ \$\$ | \$\$ | / |\$\$ \$\$ |\$\$ | \$\$ | __ + \$\$ |__\$\$ | \$\$ | \$\$ |__\$\$ |\$\$ | \$\$ \__\$\$ | \$\$ |\$\$\$\$\$\$\$\$/ \$\$ \_____ \$\$ |/ | + \$\$ \$\$/ \$\$ | \$\$ \$\$/ \$\$ | \$\$ \$\$/ \$\$ |\$\$ |\$\$ | \$\$ \$\$/ + \$\$\$\$\$\$\$/ \$\$/ \$\$\$\$\$\$\$/ \$\$/ \$\$\$\$\$\$/__ \$\$ | \$\$\$\$\$\$\$/ \$\$\$\$\$\$\$/ \$\$\$\$/ + \$\$ | / \__\$\$ | + \$\$ | \$\$ \$\$/ + \$\$/ \$\$\$\$\$\$/ + + "$DEFAULT" + This script launch: (Inside screen Daemons)"$CYAN" + - D4 Twisted server. + - All wokers manager. + - All Redis in memory servers. + - Flak server. + + Usage: LAUNCH.sh + [-l | --launchAuto] + [-k | --killAll] + [-h | --help] + " +} + +function launching_redis { + conf_dir="${D4_HOME}/configs/" + redis_dir="${D4_HOME}/redis/src/" + + screen -dmS "Redis_D4" + sleep 0.1 + echo -e $GREEN"\t* Launching D4 Redis ervers"$DEFAULT + screen -S "Redis_D4" -X screen -t "6379" bash -c $redis_dir'redis-server '$conf_dir'6379.conf ; read x' + sleep 0.1 + screen -S "Redis_D4" -X screen -t "6380" bash -c $redis_dir'redis-server '$conf_dir'6380.conf ; read x' + sleep 0.1 +} + +function launching_d4_server { + screen -dmS "Server_D4" + sleep 0.1 + echo -e $GREEN"\t* Launching D4 Server"$DEFAULT + + screen -S "Server_D4" -X screen -t "Server_D4" bash -c "cd ${D4_HOME}; ./server.py -v 10; read x" + sleep 0.1 +} + +function launching_workers { + screen -dmS "Workers_D4" + sleep 0.1 + echo -e $GREEN"\t* Launching D4 Workers"$DEFAULT + + screen -S "Workers_D4" -X screen -t "1_workers_manager" bash -c "cd ${D4_HOME}/workers/workers_1; ./workers_manager.py; read x" + sleep 0.1 + screen -S "Workers_D4" -X screen -t "2_workers_manager" bash -c "cd ${D4_HOME}/workers/workers_4; ./workers_manager.py; read x" + sleep 0.1 +} + +function shutting_down_redis { + redis_dir=${D4_HOME}/redis/src/ + bash -c $redis_dir'redis-cli -p 6379 SHUTDOWN' + sleep 0.1 + bash -c $redis_dir'redis-cli -p 6380 SHUTDOWN' + sleep 0.1 +} + +function checking_redis { + flag_redis=0 + redis_dir=${D4_HOME}/redis/src/ + bash -c $redis_dir'redis-cli -p 6379 PING | grep "PONG" &> /dev/null' + if [ ! $? == 0 ]; then + echo -e $RED"\t6379 not ready"$DEFAULT + flag_redis=1 + fi + sleep 0.1 + bash -c $redis_dir'redis-cli -p 6380 PING | grep "PONG" &> /dev/null' + if [ ! $? == 0 ]; then + echo -e $RED"\t6380 not ready"$DEFAULT + flag_redis=1 + fi + sleep 0.1 + + return $flag_redis; +} + +function launch_redis { + if [[ ! $isredis ]]; then + launching_redis; + else + echo -e $RED"\t* A D4_Redis screen is already launched"$DEFAULT + fi +} + +function launch_d4_server { + if [[ ! $isd4server ]]; then + sleep 1 + if checking_redis; then + launching_d4_server; + else + echo -e $YELLOW"\tD4 Redis not started, waiting 5 more secondes"$DEFAULT + sleep 5 + if checking_redis; then + launching_d4_server; + else + echo -e $RED"\tError: Redis not started"$DEFAULT + exit 1 + fi; + fi; + else + echo -e $RED"\t* A Server_D4 screen is already launched"$DEFAULT + fi +} + +function launch_workers { + if [[ ! $isworker ]]; then + sleep 1 + if checking_redis; then + launching_workers; + else + echo -e $YELLOW"\tD4 Redis not started, waiting 5 more secondes"$DEFAULT + sleep 5 + if checking_redis; then + launching_workers; + else + echo -e $RED"\tError: Redis not started"$DEFAULT + exit 1 + fi; + fi; + else + echo -e $RED"\t* A Workers_D4 screen is already launched"$DEFAULT + fi +} + +function launch_flask { + if [[ ! $isflask ]]; then + flask_dir=${D4_HOME}/web + screen -dmS "Flask_D4" + sleep 0.1 + echo -e $GREEN"\t* Launching Flask server"$DEFAULT + screen -S "Flask_D4" -X screen -t "Flask_server" bash -c "cd $flask_dir; ls; ./Flask_server.py; read x" + else + echo -e $RED"\t* A Flask_D4 screen is already launched"$DEFAULT + fi +} + +function killall { + + + if [[ $isredis || $isd4server || $isworker || $isflask ]]; then + echo -e $GREEN"\t* Gracefully closing D4 Servers ..."$DEFAULT + kill -SIGINT $isd4server + sleep 3 + kill $isd4server $isflask + echo -e $GREEN"\t* Gracefully closing D4 Workers ..."$DEFAULT + kill -SIGINT $isworker + sleep 0.5 + kill $isworker + echo -e $GREEN"\t* $isd4server $isworker $isflask killed."$DEFAULT + echo -e $GREEN"\t* Gracefully closing redis servers ..."$DEFAULT + shutting_down_redis; + kill $isredis + sleep 0.2 + echo -e $ROSE`screen -ls`$DEFAULT + else + echo -e $RED"\t* No screen to kill"$DEFAULT + fi +} + +function update_web { + echo -e "\t* Updating web..." + bash -c "(cd ${D4_HOME}/web; ./update_web.sh)" + exitStatus=$? + if [ $exitStatus -ge 1 ]; then + echo -e $RED"\t* Web not up-to-date"$DEFAULT + exit + else + echo -e $GREEN"\t* Web updated"$DEFAULT + fi +} + +function launch_all { + helptext; + launch_redis; + launch_d4_server; + launch_workers; + launch_flask; +} + +#If no params, display the menu +[[ $@ ]] || { + + helptext; + + options=("Redis" "D4-Server" "Workers-manager" "Flask" "Killall" "Update-web") + + menu() { + echo "What do you want to Launch?:" + for i in ${!options[@]}; do + printf "%3d%s) %s\n" $((i+1)) "${choices[i]:- }" "${options[i]}" + done + [[ "$msg" ]] && echo "$msg"; : + } + + prompt="Check an option (again to uncheck, ENTER when done): " + while menu && read -rp "$prompt" numinput && [[ "$numinput" ]]; do + for num in $numinput; do + [[ "$num" != *[![:digit:]]* ]] && (( num > 0 && num <= ${#options[@]} )) || { + msg="Invalid option: $num"; break + } + ((num--)); msg="${options[num]} was ${choices[num]:+un}checked" + [[ "${choices[num]}" ]] && choices[num]="" || choices[num]="+" + done + done + + for i in ${!options[@]}; do + if [[ "${choices[i]}" ]]; then + case ${options[i]} in + Redis) + launch_redis + ;; + D4-Server) + launch_d4_server; + ;; + Workers-manager) + launch_workers; + ;; + Flask) + launch_flask; + ;; + Killall) + killall; + ;; + Update-web) + update_web; + ;; + esac + fi + done + + exit +} + +while [ "$1" != "" ]; do + case $1 in + -l | --launchAuto ) launch_all; + ;; + -k | --killAll ) killall; + ;; + -h | --help ) helptext; + exit + ;; + * ) helptext + exit 1 + esac + shift +done diff --git a/server/6379.conf b/server/configs/6379.conf similarity index 100% rename from server/6379.conf rename to server/configs/6379.conf diff --git a/server/6380.conf b/server/configs/6380.conf similarity index 100% rename from server/6380.conf rename to server/configs/6380.conf diff --git a/server/install_server.sh b/server/install_server.sh index 39553f0..bf04a6f 100755 --- a/server/install_server.sh +++ b/server/install_server.sh @@ -3,7 +3,20 @@ set -e set -x -sudo apt-get install python3-pip -y +sudo apt-get install python3-pip virtualenv -y + +if [ -z "$VIRTUAL_ENV" ]; then + virtualenv -p python3 D4ENV + echo export D4_HOME=$(pwd) >> ./D4ENV/bin/activate + . ./D4ENV/bin/activate +fi +python3 -m pip install -r requirement.txt + +pushd web/ +./update_web.sh +popd + + python3 -m pip install -r requirement.txt # REDIS # diff --git a/server/requirement.txt b/server/requirement.txt index 6662368..68a556b 100644 --- a/server/requirement.txt +++ b/server/requirement.txt @@ -1,5 +1,6 @@ twisted[tls] redis +flask #sudo python3 -m pip install --upgrade service_identity diff --git a/server/save_data.py b/server/save_data.py deleted file mode 100755 index 9108293..0000000 --- a/server/save_data.py +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env python3 - -import os -import sys -import redis -import time -import gzip -import datetime - -redis_server = redis.StrictRedis( - host="localhost", - port=6379, - db=0) - -type = 1 -max_timestamp = 60*5 - -def gzip_file(filepath): - with open(filepath, 'rb') as f: - content = f.read() - with gzip.open(filepath+'.gz', 'wb') as f2: - f2.write(content) - os.remove(filepath) - -if __name__ == "__main__": - stream_name = 'stream:{}'.format(type) - - #group_name = 'group_stream:{}'.format(type) - #try: - # redis_server.xgroup_create(stream_name, group_name) - #except: - # pass - - while True: - - #print(redis_server.xpending(stream_name, group_name)) - - #res = redis_server.xreadgroup(group_name, 'consumername', {stream_name: '>'}, count=1) - res = redis_server.xread({stream_name: '0'}, count=1, block=500) - if res: - id = res[0][1][0][0] - data = res[0][1][0][1] - if id and data: - #print(id.decode()) - #print(data) - - date = datetime.datetime.now().strftime("%Y/%m/%d") - dir_path = os.path.join('data', date, data[b'uuid'].decode()) - filename = '' - data_timestamp = data[b'timestamp'].decode() - - try: - it = os.scandir(dir_path) - for entry in it: - if not entry.name.endswith(".gz") and entry.is_file(): - filename = entry.name - break - filepath = os.path.join(dir_path, filename) - - #if os.path.getsize(filepath) > 500000000: #bytes - # gzip_file(filepath) - # filename = data_timestamp - - except FileNotFoundError: - os.makedirs(dir_path) - # # TODO: use contexte manager in python 3.6 - it = [] - # # - - if not filename: - filename = data_timestamp - - if int(data_timestamp) - int(filename) > max_timestamp: - gzip_file(filepath) - filename = data_timestamp - - with open(os.path.join(dir_path, filename), 'ab') as f: - f.write(data[b'message']) - - #redis_server.xack(stream_name, group_name, id) - redis_server.xdel(stream_name, id) - else: - time.sleep(10) diff --git a/server/test_ctypes.py b/server/test_ctypes.py deleted file mode 100755 index 20f2bd9..0000000 --- a/server/test_ctypes.py +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* - -from ctypes import * - -class D4Header(Structure): - _fields_ = [ - ("version", c_uint8), - ("type", c_uint8), - #("uuid", c_uint128), - ("uuid", c_uint64), - ("timestamp", c_uint64), - #("hmac", c_uint256), - ("hmac", c_uint64), - ("size", c_uint32), - ] - -def pack(ctype_instance): - return string_at(byref(ctype_instance), sizeof(ctype_instance)) - -def unpack(ctype, buffer): - c_str = create_string_buffer(buffer) - return cast(pointer(c_str), POINTER(ctype)).contents - -if __name__ == '__main__': - d4h = D4Header(1,1,88888,1543398024,233243445342,342) - print(d4h.version) - buffer = pack(d4h) - print(buffer) - d = unpack(D4Header, buffer) - assert(d4h.version == d.version)