mirror of https://github.com/D4-project/d4-core
chg: [server launcher] add launcher
Terrtia
parent
7760d37084
commit
653a1073b8
|
|
@ -4,3 +4,6 @@
|
|||
*.key
|
||||
data/
|
||||
logs/
|
||||
redis/
|
||||
D4ENV/
|
||||
dump.rdb
|
||||
|
|
|
|||
|
|
@ -0,0 +1,278 @@
|
|||
#!/bin/bash
|
||||
|
||||
GREEN="\\033[1;32m"
|
||||
DEFAULT="\\033[0;39m"
|
||||
RED="\\033[1;31m"
|
||||
ROSE="\\033[1;35m"
|
||||
BLUE="\\033[1;34m"
|
||||
WHITE="\\033[0;02m"
|
||||
YELLOW="\\033[1;33m"
|
||||
CYAN="\\033[1;36m"
|
||||
|
||||
. ./D4ENV/bin/activate
|
||||
|
||||
isredis=`screen -ls | egrep '[0-9]+.Redis_D4' | cut -d. -f1`
|
||||
isd4server=`screen -ls | egrep '[0-9]+.Server_D4' | cut -d. -f1`
|
||||
isworker=`screen -ls | egrep '[0-9]+.Workers_D4' | cut -d. -f1`
|
||||
isflask=`screen -ls | egrep '[0-9]+.Flask_D4' | cut -d. -f1`
|
||||
|
||||
function helptext {
|
||||
echo -e $YELLOW"
|
||||
_______ __ __ __
|
||||
/ \ / | / | / |
|
||||
\$\$\$\$\$\$\$ |\$\$ | \$\$ | ______ ______ ______ __ ______ _______ _\$\$ |_
|
||||
\$\$ | \$\$ |\$\$ |__\$\$ | / \ / \ / \ / | / \ / |/ \$\$ |
|
||||
\$\$ | \$\$ |\$\$ \$\$ | /\$\$\$\$\$\$ |/\$\$\$\$\$\$ |/\$\$\$\$\$\$ | \$\$/ /\$\$\$\$\$\$ |/\$\$\$\$\$\$\$/ \$\$\$\$\$\$/
|
||||
\$\$ | \$\$ |\$\$\$\$\$\$\$\$ | \$\$ | \$\$ |\$\$ | \$\$/ \$\$ | \$\$ | / |\$\$ \$\$ |\$\$ | \$\$ | __
|
||||
\$\$ |__\$\$ | \$\$ | \$\$ |__\$\$ |\$\$ | \$\$ \__\$\$ | \$\$ |\$\$\$\$\$\$\$\$/ \$\$ \_____ \$\$ |/ |
|
||||
\$\$ \$\$/ \$\$ | \$\$ \$\$/ \$\$ | \$\$ \$\$/ \$\$ |\$\$ |\$\$ | \$\$ \$\$/
|
||||
\$\$\$\$\$\$\$/ \$\$/ \$\$\$\$\$\$\$/ \$\$/ \$\$\$\$\$\$/__ \$\$ | \$\$\$\$\$\$\$/ \$\$\$\$\$\$\$/ \$\$\$\$/
|
||||
\$\$ | / \__\$\$ |
|
||||
\$\$ | \$\$ \$\$/
|
||||
\$\$/ \$\$\$\$\$\$/
|
||||
|
||||
"$DEFAULT"
|
||||
This script launch: (Inside screen Daemons)"$CYAN"
|
||||
- D4 Twisted server.
|
||||
- All wokers manager.
|
||||
- All Redis in memory servers.
|
||||
- Flak server.
|
||||
|
||||
Usage: LAUNCH.sh
|
||||
[-l | --launchAuto]
|
||||
[-k | --killAll]
|
||||
[-h | --help]
|
||||
"
|
||||
}
|
||||
|
||||
function launching_redis {
|
||||
conf_dir="${D4_HOME}/configs/"
|
||||
redis_dir="${D4_HOME}/redis/src/"
|
||||
|
||||
screen -dmS "Redis_D4"
|
||||
sleep 0.1
|
||||
echo -e $GREEN"\t* Launching D4 Redis ervers"$DEFAULT
|
||||
screen -S "Redis_D4" -X screen -t "6379" bash -c $redis_dir'redis-server '$conf_dir'6379.conf ; read x'
|
||||
sleep 0.1
|
||||
screen -S "Redis_D4" -X screen -t "6380" bash -c $redis_dir'redis-server '$conf_dir'6380.conf ; read x'
|
||||
sleep 0.1
|
||||
}
|
||||
|
||||
function launching_d4_server {
|
||||
screen -dmS "Server_D4"
|
||||
sleep 0.1
|
||||
echo -e $GREEN"\t* Launching D4 Server"$DEFAULT
|
||||
|
||||
screen -S "Server_D4" -X screen -t "Server_D4" bash -c "cd ${D4_HOME}; ./server.py -v 10; read x"
|
||||
sleep 0.1
|
||||
}
|
||||
|
||||
function launching_workers {
|
||||
screen -dmS "Workers_D4"
|
||||
sleep 0.1
|
||||
echo -e $GREEN"\t* Launching D4 Workers"$DEFAULT
|
||||
|
||||
screen -S "Workers_D4" -X screen -t "1_workers_manager" bash -c "cd ${D4_HOME}/workers/workers_1; ./workers_manager.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Workers_D4" -X screen -t "2_workers_manager" bash -c "cd ${D4_HOME}/workers/workers_4; ./workers_manager.py; read x"
|
||||
sleep 0.1
|
||||
}
|
||||
|
||||
function shutting_down_redis {
|
||||
redis_dir=${D4_HOME}/redis/src/
|
||||
bash -c $redis_dir'redis-cli -p 6379 SHUTDOWN'
|
||||
sleep 0.1
|
||||
bash -c $redis_dir'redis-cli -p 6380 SHUTDOWN'
|
||||
sleep 0.1
|
||||
}
|
||||
|
||||
function checking_redis {
|
||||
flag_redis=0
|
||||
redis_dir=${D4_HOME}/redis/src/
|
||||
bash -c $redis_dir'redis-cli -p 6379 PING | grep "PONG" &> /dev/null'
|
||||
if [ ! $? == 0 ]; then
|
||||
echo -e $RED"\t6379 not ready"$DEFAULT
|
||||
flag_redis=1
|
||||
fi
|
||||
sleep 0.1
|
||||
bash -c $redis_dir'redis-cli -p 6380 PING | grep "PONG" &> /dev/null'
|
||||
if [ ! $? == 0 ]; then
|
||||
echo -e $RED"\t6380 not ready"$DEFAULT
|
||||
flag_redis=1
|
||||
fi
|
||||
sleep 0.1
|
||||
|
||||
return $flag_redis;
|
||||
}
|
||||
|
||||
function launch_redis {
|
||||
if [[ ! $isredis ]]; then
|
||||
launching_redis;
|
||||
else
|
||||
echo -e $RED"\t* A D4_Redis screen is already launched"$DEFAULT
|
||||
fi
|
||||
}
|
||||
|
||||
function launch_d4_server {
|
||||
if [[ ! $isd4server ]]; then
|
||||
sleep 1
|
||||
if checking_redis; then
|
||||
launching_d4_server;
|
||||
else
|
||||
echo -e $YELLOW"\tD4 Redis not started, waiting 5 more secondes"$DEFAULT
|
||||
sleep 5
|
||||
if checking_redis; then
|
||||
launching_d4_server;
|
||||
else
|
||||
echo -e $RED"\tError: Redis not started"$DEFAULT
|
||||
exit 1
|
||||
fi;
|
||||
fi;
|
||||
else
|
||||
echo -e $RED"\t* A Server_D4 screen is already launched"$DEFAULT
|
||||
fi
|
||||
}
|
||||
|
||||
function launch_workers {
|
||||
if [[ ! $isworker ]]; then
|
||||
sleep 1
|
||||
if checking_redis; then
|
||||
launching_workers;
|
||||
else
|
||||
echo -e $YELLOW"\tD4 Redis not started, waiting 5 more secondes"$DEFAULT
|
||||
sleep 5
|
||||
if checking_redis; then
|
||||
launching_workers;
|
||||
else
|
||||
echo -e $RED"\tError: Redis not started"$DEFAULT
|
||||
exit 1
|
||||
fi;
|
||||
fi;
|
||||
else
|
||||
echo -e $RED"\t* A Workers_D4 screen is already launched"$DEFAULT
|
||||
fi
|
||||
}
|
||||
|
||||
function launch_flask {
|
||||
if [[ ! $isflask ]]; then
|
||||
flask_dir=${D4_HOME}/web
|
||||
screen -dmS "Flask_D4"
|
||||
sleep 0.1
|
||||
echo -e $GREEN"\t* Launching Flask server"$DEFAULT
|
||||
screen -S "Flask_D4" -X screen -t "Flask_server" bash -c "cd $flask_dir; ls; ./Flask_server.py; read x"
|
||||
else
|
||||
echo -e $RED"\t* A Flask_D4 screen is already launched"$DEFAULT
|
||||
fi
|
||||
}
|
||||
|
||||
function killall {
|
||||
|
||||
|
||||
if [[ $isredis || $isd4server || $isworker || $isflask ]]; then
|
||||
echo -e $GREEN"\t* Gracefully closing D4 Servers ..."$DEFAULT
|
||||
kill -SIGINT $isd4server
|
||||
sleep 3
|
||||
kill $isd4server $isflask
|
||||
echo -e $GREEN"\t* Gracefully closing D4 Workers ..."$DEFAULT
|
||||
kill -SIGINT $isworker
|
||||
sleep 0.5
|
||||
kill $isworker
|
||||
echo -e $GREEN"\t* $isd4server $isworker $isflask killed."$DEFAULT
|
||||
echo -e $GREEN"\t* Gracefully closing redis servers ..."$DEFAULT
|
||||
shutting_down_redis;
|
||||
kill $isredis
|
||||
sleep 0.2
|
||||
echo -e $ROSE`screen -ls`$DEFAULT
|
||||
else
|
||||
echo -e $RED"\t* No screen to kill"$DEFAULT
|
||||
fi
|
||||
}
|
||||
|
||||
function update_web {
|
||||
echo -e "\t* Updating web..."
|
||||
bash -c "(cd ${D4_HOME}/web; ./update_web.sh)"
|
||||
exitStatus=$?
|
||||
if [ $exitStatus -ge 1 ]; then
|
||||
echo -e $RED"\t* Web not up-to-date"$DEFAULT
|
||||
exit
|
||||
else
|
||||
echo -e $GREEN"\t* Web updated"$DEFAULT
|
||||
fi
|
||||
}
|
||||
|
||||
function launch_all {
|
||||
helptext;
|
||||
launch_redis;
|
||||
launch_d4_server;
|
||||
launch_workers;
|
||||
launch_flask;
|
||||
}
|
||||
|
||||
#If no params, display the menu
|
||||
[[ $@ ]] || {
|
||||
|
||||
helptext;
|
||||
|
||||
options=("Redis" "D4-Server" "Workers-manager" "Flask" "Killall" "Update-web")
|
||||
|
||||
menu() {
|
||||
echo "What do you want to Launch?:"
|
||||
for i in ${!options[@]}; do
|
||||
printf "%3d%s) %s\n" $((i+1)) "${choices[i]:- }" "${options[i]}"
|
||||
done
|
||||
[[ "$msg" ]] && echo "$msg"; :
|
||||
}
|
||||
|
||||
prompt="Check an option (again to uncheck, ENTER when done): "
|
||||
while menu && read -rp "$prompt" numinput && [[ "$numinput" ]]; do
|
||||
for num in $numinput; do
|
||||
[[ "$num" != *[![:digit:]]* ]] && (( num > 0 && num <= ${#options[@]} )) || {
|
||||
msg="Invalid option: $num"; break
|
||||
}
|
||||
((num--)); msg="${options[num]} was ${choices[num]:+un}checked"
|
||||
[[ "${choices[num]}" ]] && choices[num]="" || choices[num]="+"
|
||||
done
|
||||
done
|
||||
|
||||
for i in ${!options[@]}; do
|
||||
if [[ "${choices[i]}" ]]; then
|
||||
case ${options[i]} in
|
||||
Redis)
|
||||
launch_redis
|
||||
;;
|
||||
D4-Server)
|
||||
launch_d4_server;
|
||||
;;
|
||||
Workers-manager)
|
||||
launch_workers;
|
||||
;;
|
||||
Flask)
|
||||
launch_flask;
|
||||
;;
|
||||
Killall)
|
||||
killall;
|
||||
;;
|
||||
Update-web)
|
||||
update_web;
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
done
|
||||
|
||||
exit
|
||||
}
|
||||
|
||||
while [ "$1" != "" ]; do
|
||||
case $1 in
|
||||
-l | --launchAuto ) launch_all;
|
||||
;;
|
||||
-k | --killAll ) killall;
|
||||
;;
|
||||
-h | --help ) helptext;
|
||||
exit
|
||||
;;
|
||||
* ) helptext
|
||||
exit 1
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
|
@ -3,7 +3,20 @@
|
|||
set -e
|
||||
set -x
|
||||
|
||||
sudo apt-get install python3-pip -y
|
||||
sudo apt-get install python3-pip virtualenv -y
|
||||
|
||||
if [ -z "$VIRTUAL_ENV" ]; then
|
||||
virtualenv -p python3 D4ENV
|
||||
echo export D4_HOME=$(pwd) >> ./D4ENV/bin/activate
|
||||
. ./D4ENV/bin/activate
|
||||
fi
|
||||
python3 -m pip install -r requirement.txt
|
||||
|
||||
pushd web/
|
||||
./update_web.sh
|
||||
popd
|
||||
|
||||
|
||||
python3 -m pip install -r requirement.txt
|
||||
|
||||
# REDIS #
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
twisted[tls]
|
||||
redis
|
||||
flask
|
||||
|
||||
#sudo python3 -m pip install --upgrade service_identity
|
||||
|
||||
|
|
|
|||
|
|
@ -1,83 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
import sys
|
||||
import redis
|
||||
import time
|
||||
import gzip
|
||||
import datetime
|
||||
|
||||
redis_server = redis.StrictRedis(
|
||||
host="localhost",
|
||||
port=6379,
|
||||
db=0)
|
||||
|
||||
type = 1
|
||||
max_timestamp = 60*5
|
||||
|
||||
def gzip_file(filepath):
|
||||
with open(filepath, 'rb') as f:
|
||||
content = f.read()
|
||||
with gzip.open(filepath+'.gz', 'wb') as f2:
|
||||
f2.write(content)
|
||||
os.remove(filepath)
|
||||
|
||||
if __name__ == "__main__":
|
||||
stream_name = 'stream:{}'.format(type)
|
||||
|
||||
#group_name = 'group_stream:{}'.format(type)
|
||||
#try:
|
||||
# redis_server.xgroup_create(stream_name, group_name)
|
||||
#except:
|
||||
# pass
|
||||
|
||||
while True:
|
||||
|
||||
#print(redis_server.xpending(stream_name, group_name))
|
||||
|
||||
#res = redis_server.xreadgroup(group_name, 'consumername', {stream_name: '>'}, count=1)
|
||||
res = redis_server.xread({stream_name: '0'}, count=1, block=500)
|
||||
if res:
|
||||
id = res[0][1][0][0]
|
||||
data = res[0][1][0][1]
|
||||
if id and data:
|
||||
#print(id.decode())
|
||||
#print(data)
|
||||
|
||||
date = datetime.datetime.now().strftime("%Y/%m/%d")
|
||||
dir_path = os.path.join('data', date, data[b'uuid'].decode())
|
||||
filename = ''
|
||||
data_timestamp = data[b'timestamp'].decode()
|
||||
|
||||
try:
|
||||
it = os.scandir(dir_path)
|
||||
for entry in it:
|
||||
if not entry.name.endswith(".gz") and entry.is_file():
|
||||
filename = entry.name
|
||||
break
|
||||
filepath = os.path.join(dir_path, filename)
|
||||
|
||||
#if os.path.getsize(filepath) > 500000000: #bytes
|
||||
# gzip_file(filepath)
|
||||
# filename = data_timestamp
|
||||
|
||||
except FileNotFoundError:
|
||||
os.makedirs(dir_path)
|
||||
# # TODO: use contexte manager in python 3.6
|
||||
it = []
|
||||
# #
|
||||
|
||||
if not filename:
|
||||
filename = data_timestamp
|
||||
|
||||
if int(data_timestamp) - int(filename) > max_timestamp:
|
||||
gzip_file(filepath)
|
||||
filename = data_timestamp
|
||||
|
||||
with open(os.path.join(dir_path, filename), 'ab') as f:
|
||||
f.write(data[b'message'])
|
||||
|
||||
#redis_server.xack(stream_name, group_name, id)
|
||||
redis_server.xdel(stream_name, id)
|
||||
else:
|
||||
time.sleep(10)
|
||||
|
|
@ -1,31 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
from ctypes import *
|
||||
|
||||
class D4Header(Structure):
|
||||
_fields_ = [
|
||||
("version", c_uint8),
|
||||
("type", c_uint8),
|
||||
#("uuid", c_uint128),
|
||||
("uuid", c_uint64),
|
||||
("timestamp", c_uint64),
|
||||
#("hmac", c_uint256),
|
||||
("hmac", c_uint64),
|
||||
("size", c_uint32),
|
||||
]
|
||||
|
||||
def pack(ctype_instance):
|
||||
return string_at(byref(ctype_instance), sizeof(ctype_instance))
|
||||
|
||||
def unpack(ctype, buffer):
|
||||
c_str = create_string_buffer(buffer)
|
||||
return cast(pointer(c_str), POINTER(ctype)).contents
|
||||
|
||||
if __name__ == '__main__':
|
||||
d4h = D4Header(1,1,88888,1543398024,233243445342,342)
|
||||
print(d4h.version)
|
||||
buffer = pack(d4h)
|
||||
print(buffer)
|
||||
d = unpack(D4Header, buffer)
|
||||
assert(d4h.version == d.version)
|
||||
Loading…
Reference in New Issue