mirror of https://github.com/CIRCL/AIL-framework
1271 lines
45 KiB
Python
Executable File
1271 lines
45 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
# -*-coding:UTF-8 -*
|
|
|
|
import os
|
|
import json
|
|
import secrets
|
|
import re
|
|
import sys
|
|
import time
|
|
import uuid
|
|
|
|
import subprocess
|
|
|
|
from flask import escape
|
|
from pubsublogger import publisher
|
|
|
|
sys.path.append(os.environ['AIL_BIN'])
|
|
##################################
|
|
# Import Project packages
|
|
##################################
|
|
from lib import ConfigLoader
|
|
from lib.objects.Items import Item
|
|
# from lib import Tag
|
|
from core import screen
|
|
|
|
config_loader = ConfigLoader.ConfigLoader()
|
|
r_cache = config_loader.get_redis_conn("Redis_Cache")
|
|
r_serv_db = config_loader.get_db_conn("Kvrocks_DB")
|
|
r_serv_sync = config_loader.get_db_conn("Kvrocks_DB")
|
|
config_loader = None
|
|
|
|
WEBSOCKETS_CLOSE_CODES = {
|
|
1000: 'Normal Closure',
|
|
1001: 'Going Away',
|
|
1002: 'Protocol Error',
|
|
1003: 'Unsupported Data',
|
|
1005: 'No Status Received',
|
|
1006: 'Abnormal Closure',
|
|
1007: 'Invalid frame payload data',
|
|
1008: 'Policy Violation',
|
|
1009: 'Message too big',
|
|
1010: 'Missing Extension',
|
|
1011: 'Internal Error',
|
|
1012: 'Service Restart',
|
|
1013: 'Try Again Later',
|
|
1014: 'Bad Gateway',
|
|
1015: 'TLS Handshake',
|
|
}
|
|
|
|
#### LOGS ####
|
|
# redis_logger = publisher
|
|
# redis_logger.port = 6380
|
|
# redis_logger.channel = 'Sync'
|
|
|
|
def get_websockets_close_message(code):
|
|
if code in WEBSOCKETS_CLOSE_CODES:
|
|
msg = f'{code} {WEBSOCKETS_CLOSE_CODES[code]}'
|
|
else:
|
|
msg = f'{code} Unknow websockets code'
|
|
return msg
|
|
|
|
##-- LOGS --##
|
|
|
|
def is_valid_uuid_v4(UUID):
|
|
if not UUID:
|
|
return False
|
|
UUID = UUID.replace('-', '')
|
|
try:
|
|
uuid_test = uuid.UUID(hex=UUID, version=4)
|
|
return uuid_test.hex == UUID
|
|
except:
|
|
return False
|
|
|
|
def sanityze_uuid(UUID):
|
|
sanityzed_uuid = uuid.UUID(hex=UUID, version=4)
|
|
return str(sanityzed_uuid)
|
|
|
|
def generate_uuid():
|
|
return str(uuid.uuid4()).replace('-', '')
|
|
|
|
def generate_sync_api_key():
|
|
return secrets.token_urlsafe(42)
|
|
|
|
def get_ail_uuid():
|
|
return r_serv_db.get('ail:uuid')
|
|
|
|
def get_sync_server_version():
|
|
return '0.1'
|
|
|
|
def is_valid_websocket_url(websocket_url):
|
|
regex_websocket_url = r'^(wss:\/\/)([0-9]{1,3}(?:\.[0-9]{1,3}){3}|(?=[^\/]{1,254}(?![^\/]))(?:(?=[a-zA-Z0-9-]{1,63}\.?)(?:xn--+)?[a-zA-Z0-9]+(?:-[a-zA-Z0-9]+)*\.?)+[a-zA-Z]{2,63}):([0-9]{1,5})$'
|
|
if re.match(regex_websocket_url, websocket_url):
|
|
return True
|
|
return False
|
|
|
|
def is_valid_websocket_key(ail_key):
|
|
regex_key = r'^[A-Za-z0-9-_]{56}$'
|
|
if re.match(regex_key, ail_key):
|
|
return True
|
|
return False
|
|
|
|
#### HANDLE CONFIG UPDATE ####
|
|
|
|
def get_last_updated_sync_config():
|
|
epoch = r_serv_sync.get(f'ail:instance:queue:last_updated_sync_config')
|
|
if not epoch:
|
|
epoch = 0
|
|
return float(epoch)
|
|
|
|
def set_last_updated_sync_config():
|
|
epoch = int(time.time())
|
|
r_serv_sync.set(f'ail:instance:queue:last_updated_sync_config', epoch)
|
|
return epoch
|
|
|
|
# # TODO: get connection status
|
|
# # TODO: get connection METADATA
|
|
# # TODO: client => reconnect on fails (with timeout)
|
|
|
|
# # TODO: API KEY change => trigger kill connection
|
|
#############################
|
|
# #
|
|
#### SYNC CLIENT MANAGER ####
|
|
|
|
#### ail servers connected clients ####
|
|
#
|
|
# ail_uuid => WEBSOCKETS => ail_uuid
|
|
# AIL: clients:
|
|
# - 1 push
|
|
# - 1 pull
|
|
# - N api
|
|
|
|
def get_server_all_connected_clients():
|
|
return r_cache.smembers('ail_2_ail:server:all_clients')
|
|
|
|
def add_server_connected_client(ail_uuid, sync_mode):
|
|
r_cache.sadd('ail_2_ail:server:all_clients', ail_uuid)
|
|
r_cache.hset(f'ail_2_ail:server:client:{ail_uuid}', sync_mode, True)
|
|
|
|
def remove_server_connected_client(ail_uuid, sync_mode=None, is_connected=False):
|
|
if sync_mode:
|
|
r_cache.hdel(f'ail_2_ail:server:client:{ail_uuid}', sync_mode)
|
|
if not is_connected:
|
|
r_cache.srem('ail_2_ail:server:all_clients', ail_uuid)
|
|
|
|
def is_server_client_sync_mode_connected(ail_uuid, sync_mode):
|
|
res = r_cache.hexists(f'ail_2_ail:server:client:{ail_uuid}', sync_mode)
|
|
return res == 1
|
|
|
|
def is_server_client_connected(ail_uuid):
|
|
return r_cache.sismember('ail_2_ail:server:all_clients', ail_uuid)
|
|
|
|
def clear_server_connected_clients():
|
|
for ail_uuid in get_server_all_connected_clients():
|
|
r_cache.delete(f'ail_2_ail:server:client:{ail_uuid}')
|
|
r_cache.delete('ail_2_ail:server:all_clients')
|
|
|
|
def get_server_controller_command():
|
|
res = r_cache.spop('ail_2_ail:server_controller:command')
|
|
if res:
|
|
return json.loads(res)
|
|
else:
|
|
return None
|
|
|
|
# command: -kill
|
|
# -killall or shutdown / restart ?
|
|
## TODO: ADD command
|
|
def send_command_to_server_controller(command, ail_uuid=None):
|
|
dict_action = {'command': command, 'ail_uuid': ail_uuid}
|
|
if ail_uuid:
|
|
dict_action['ail_uuid'] = ail_uuid
|
|
str_command = json.dumps(dict_action)
|
|
r_cache.sadd('ail_2_ail:server_controller:command', str_command)
|
|
|
|
##-- --##
|
|
def get_new_sync_client_id():
|
|
for new_id in range(120000, 100000, -1):
|
|
new_id = str(new_id)
|
|
if not r_cache.exists(f'ail_2_ail:sync_client:{new_id}'):
|
|
return str(new_id)
|
|
|
|
def get_all_sync_clients(r_set=False):
|
|
res = r_cache.smembers('ail_2_ail:all_sync_clients')
|
|
if r_set:
|
|
return set(res)
|
|
else:
|
|
return res
|
|
|
|
def get_sync_client_ail_uuid(client_id):
|
|
return r_cache.hget(f'ail_2_ail:sync_client:{client_id}', 'ail_uuid')
|
|
|
|
def get_sync_client_sync_mode(client_id):
|
|
return r_cache.hget(f'ail_2_ail:sync_client:{client_id}', 'sync_mode')
|
|
|
|
def set_sync_client_sync_mode(client_id, sync_mode):
|
|
r_cache.hset(f'ail_2_ail:sync_client:{client_id}', 'sync_mode', sync_mode)
|
|
|
|
def create_sync_client_cache(ail_uuid, sync_mode, client_id=None):
|
|
if client_id is None:
|
|
client_id = get_new_sync_client_id()
|
|
# save sync client status
|
|
r_cache.hset(f'ail_2_ail:sync_client:{client_id}', 'ail_uuid', ail_uuid)
|
|
r_cache.hset(f'ail_2_ail:sync_client:{client_id}', 'launch_time', int(time.time()))
|
|
set_sync_client_sync_mode(client_id, sync_mode)
|
|
|
|
r_cache.sadd('ail_2_ail:all_sync_clients', client_id)
|
|
|
|
# create map ail_uuid/queue_uuid
|
|
r_cache.sadd(f'ail_2_ail:ail_uuid:{ail_uuid}', client_id)
|
|
return client_id
|
|
|
|
# current: only one push registred
|
|
def get_client_id_by_ail_uuid(ail_uuid, filter_push=True):
|
|
res = r_cache.smembers(f'ail_2_ail:ail_uuid:{ail_uuid}')
|
|
if not filter_push:
|
|
return res
|
|
else:
|
|
clients_id = []
|
|
for client_id in res:
|
|
client_id = int(client_id)
|
|
if client_id <= 100000:
|
|
clients_id.append(client_id)
|
|
return clients_id
|
|
|
|
def get_all_running_sync_servers():
|
|
running_ail_servers= []
|
|
for client_id in get_all_sync_clients():
|
|
ail_uuid = get_sync_client_ail_uuid(client_id)
|
|
running_ail_servers.append(ail_uuid)
|
|
return running_ail_servers
|
|
|
|
def get_ail_instance_all_running_sync_mode(ail_uuid):
|
|
clients_id = get_client_id_by_ail_uuid(ail_uuid, filter_push=False)
|
|
running_sync_mode = {'api': False, 'pull': False, 'push': False}
|
|
for client_id in clients_id:
|
|
sync_mode = get_sync_client_sync_mode(client_id)
|
|
running_sync_mode[sync_mode] = True
|
|
return running_sync_mode
|
|
|
|
def delete_sync_client_cache(client_id):
|
|
ail_uuid = get_sync_client_ail_uuid(client_id)
|
|
# map ail_uuid
|
|
r_cache.srem(f'ail_2_ail:ail_uuid:{ail_uuid}', client_id)
|
|
|
|
r_cache.delete(f'ail_2_ail:sync_client:{client_id}')
|
|
r_cache.srem('ail_2_ail:all_sync_clients', client_id)
|
|
|
|
def delete_all_sync_clients_cache():
|
|
for client_id in get_all_sync_clients():
|
|
delete_sync_client_cache(client_id)
|
|
r_cache.delete('ail_2_ail:all_sync_clients')
|
|
|
|
# command: -launch
|
|
# -kill
|
|
# -relaunch
|
|
## TODO: check command
|
|
def send_command_to_manager(command, client_id=-1, ail_uuid=None):
|
|
dict_action = {'command': command, 'client_id': client_id}
|
|
if ail_uuid:
|
|
dict_action['ail_uuid'] = ail_uuid
|
|
str_command = json.dumps(dict_action)
|
|
r_cache.sadd('ail_2_ail:client_manager:command', str_command)
|
|
|
|
def refresh_ail_instance_connection(ail_uuid):
|
|
clients_id = get_client_id_by_ail_uuid(ail_uuid)
|
|
if clients_id:
|
|
client_id = clients_id[0]
|
|
else:
|
|
client_id = None
|
|
launch_required = is_ail_instance_push_enabled(ail_uuid) and is_ail_instance_linked_to_sync_queue(ail_uuid)
|
|
|
|
# relaunch
|
|
if client_id and launch_required:
|
|
send_command_to_manager('relaunch', client_id=client_id)
|
|
# kill
|
|
elif client_id:
|
|
send_command_to_manager('kill', client_id=client_id)
|
|
# launch
|
|
elif launch_required:
|
|
send_command_to_manager('launch', ail_uuid=ail_uuid)
|
|
|
|
|
|
class AIL2AILClientManager(object):
|
|
"""AIL2AILClientManager."""
|
|
|
|
SCREEN_NAME = 'AIL_2_AIL'
|
|
SCRIPT_NAME = 'ail_2_ail_client.py'
|
|
SCRIPT_DIR = os.path.join(os.environ['AIL_BIN'], 'core')
|
|
|
|
def __init__(self):
|
|
# dict client_id: AIL2AILCLIENT or websocket
|
|
self.clients = {}
|
|
# launch all sync clients
|
|
self.relaunch_all_sync_clients()
|
|
|
|
def get_all_clients(self):
|
|
return self.clients
|
|
|
|
# return new client id
|
|
def get_new_sync_client_id(self):
|
|
for new_id in range(1, 100000):
|
|
new_id = str(new_id)
|
|
if new_id not in self.clients:
|
|
return str(new_id)
|
|
|
|
def get_sync_client_ail_uuid(self, client_id):
|
|
return self.clients[client_id]['ail_uuid']
|
|
|
|
# def get_sync_client_queue_uuid(self, client_id):
|
|
# return self.clients[client_id]['queue_uuid']
|
|
|
|
def get_all_sync_clients_to_launch(self):
|
|
ail_instances_to_launch = []
|
|
for ail_uuid in get_all_ail_instance():
|
|
if is_ail_instance_push_enabled(ail_uuid) and is_ail_instance_linked_to_sync_queue(ail_uuid):
|
|
ail_instances_to_launch.append(ail_uuid)
|
|
return ail_instances_to_launch
|
|
|
|
def relaunch_all_sync_clients(self):
|
|
delete_all_sync_clients_cache()
|
|
self.clients = {}
|
|
for ail_uuid in self.get_all_sync_clients_to_launch():
|
|
self.launch_sync_client(ail_uuid)
|
|
|
|
def launch_sync_client(self, ail_uuid):
|
|
dir_project = os.environ['AIL_HOME']
|
|
sync_mode = 'push'
|
|
client_id = self.get_new_sync_client_id()
|
|
script_options = f'-u {ail_uuid} -m push -i {client_id}'
|
|
screen.create_screen(AIL2AILClientManager.SCREEN_NAME)
|
|
screen.launch_uniq_windows_script(AIL2AILClientManager.SCREEN_NAME,
|
|
client_id, dir_project,
|
|
AIL2AILClientManager.SCRIPT_DIR,
|
|
AIL2AILClientManager.SCRIPT_NAME,
|
|
script_options=script_options, kill_previous_windows=True)
|
|
# save sync client status
|
|
create_sync_client_cache(ail_uuid, sync_mode, client_id=client_id)
|
|
|
|
self.clients[client_id] = {'ail_uuid': ail_uuid}
|
|
|
|
# # TODO: FORCE KILL ????????????
|
|
# # TODO: check if exists
|
|
def kill_sync_client(self, client_id):
|
|
if not screen.kill_screen_window('AIL_2_AIL',client_id):
|
|
# # TODO: log kill error
|
|
pass
|
|
|
|
delete_sync_client_cache(client_id)
|
|
self.clients.pop(client_id)
|
|
|
|
## COMMANDS ##
|
|
|
|
def get_manager_command(self):
|
|
res = r_cache.spop('ail_2_ail:client_manager:command')
|
|
if res:
|
|
return json.loads(res)
|
|
else:
|
|
return None
|
|
|
|
def execute_manager_command(self, command_dict):
|
|
command = command_dict.get('command')
|
|
if command == 'launch':
|
|
ail_uuid = command_dict.get('ail_uuid')
|
|
self.launch_sync_client(ail_uuid)
|
|
elif command == 'relaunch_all':
|
|
self.relaunch_all_sync_clients()
|
|
else:
|
|
# only one sync client
|
|
client_id = int(command_dict.get('client_id'))
|
|
if client_id < 1:
|
|
print('Invalid client id')
|
|
return None
|
|
client_id = str(client_id)
|
|
if command == 'kill':
|
|
self.kill_sync_client(client_id)
|
|
elif command == 'relaunch':
|
|
ail_uuid = self.get_sync_client_ail_uuid(client_id)
|
|
self.kill_sync_client(client_id)
|
|
self.launch_sync_client(ail_uuid)
|
|
|
|
########################################
|
|
########################################
|
|
########################################
|
|
|
|
# # TODO: ADD METADATA
|
|
def get_sync_client_status(client_id):
|
|
dict_client = {'id': client_id}
|
|
dict_client['ail_uuid'] = get_sync_client_ail_uuid(client_id)
|
|
return dict_client
|
|
|
|
def get_all_sync_client_status():
|
|
sync_clients = []
|
|
all_sync_clients = r_cache.smembers('ail_2_ail:all_sync_clients')
|
|
for client_id in all_sync_clients:
|
|
sync_clients.append(get_sync_client_status(client_id))
|
|
return sync_clients
|
|
|
|
######################
|
|
# #
|
|
#### AIL INSTANCE ####
|
|
|
|
## AIL KEYS ##
|
|
|
|
def get_all_ail_instance_keys():
|
|
return r_serv_sync.smembers(f'ail:instance:key:all')
|
|
|
|
def is_allowed_ail_instance_key(key):
|
|
return r_serv_sync.sismember(f'ail:instance:key:all', key)
|
|
|
|
def get_ail_instance_key(ail_uuid):
|
|
return r_serv_sync.hget(f'ail:instance:{ail_uuid}', 'api_key')
|
|
|
|
def get_ail_instance_by_key(key):
|
|
return r_serv_sync.get(f'ail:instance:key:{key}')
|
|
|
|
# def check_acl_sync_queue_ail(ail_uuid, queue_uuid, key):
|
|
# return is_ail_instance_queue(ail_uuid, queue_uuid)
|
|
|
|
# def update_ail_instance_key(ail_uuid, new_key):
|
|
# old_key = get_ail_instance_key(ail_uuid)
|
|
# r_serv_sync.srem(f'ail:instance:key:all', old_key)
|
|
# r_serv_sync.delete(f'ail:instance:key:{old_key}')
|
|
#
|
|
# r_serv_sync.sadd(f'ail:instance:key:all', new_key)
|
|
# r_serv_sync.delete(f'ail:instance:key:{new_key}', ail_uuid)
|
|
# r_serv_sync.hset(f'ail:instance:{ail_uuid}', 'api_key', new_key)
|
|
|
|
#- AIL KEYS -#
|
|
|
|
def get_all_ail_instance():
|
|
return r_serv_sync.smembers('ail:instance:all')
|
|
|
|
def get_ail_instance_all_sync_queue(ail_uuid):
|
|
return r_serv_sync.smembers(f'ail:instance:sync_queue:{ail_uuid}')
|
|
|
|
def is_ail_instance_queue(ail_uuid, queue_uuid):
|
|
return r_serv_sync.sismember(f'ail:instance:sync_queue:{ail_uuid}', queue_uuid)
|
|
|
|
def exists_ail_instance(ail_uuid):
|
|
return r_serv_sync.exists(f'ail:instance:{ail_uuid}')
|
|
|
|
def get_ail_instance_url(ail_uuid):
|
|
return r_serv_sync.hget(f'ail:instance:{ail_uuid}', 'url')
|
|
|
|
def get_ail_instance_description(ail_uuid):
|
|
return r_serv_sync.hget(f'ail:instance:{ail_uuid}', 'description')
|
|
|
|
def exists_ail_instance(ail_uuid):
|
|
return r_serv_sync.sismember('ail:instance:all', ail_uuid)
|
|
|
|
def is_ail_instance_push_enabled(ail_uuid):
|
|
res = r_serv_sync.hget(f'ail:instance:{ail_uuid}', 'push')
|
|
return res == 'True'
|
|
|
|
def is_ail_instance_pull_enabled(ail_uuid):
|
|
res = r_serv_sync.hget(f'ail:instance:{ail_uuid}', 'pull')
|
|
return res == 'True'
|
|
|
|
def is_ail_instance_sync_enabled(ail_uuid, sync_mode=None):
|
|
if sync_mode is None:
|
|
return is_ail_instance_push_enabled(ail_uuid) or is_ail_instance_pull_enabled(ail_uuid)
|
|
elif sync_mode == 'pull':
|
|
return is_ail_instance_pull_enabled(ail_uuid)
|
|
elif sync_mode == 'push':
|
|
return is_ail_instance_push_enabled(ail_uuid)
|
|
else:
|
|
return False
|
|
|
|
def is_ail_instance_linked_to_sync_queue(ail_uuid):
|
|
return r_serv_sync.exists(f'ail:instance:sync_queue:{ail_uuid}')
|
|
|
|
def change_pull_push_state(ail_uuid, pull=None, push=None):
|
|
edited = False
|
|
curr_pull = is_ail_instance_pull_enabled(ail_uuid)
|
|
curr_push = is_ail_instance_push_enabled(ail_uuid)
|
|
if pull is not None:
|
|
# sanityze pull
|
|
if pull:
|
|
pull = True
|
|
else:
|
|
pull = False
|
|
if curr_pull != pull:
|
|
#print('pull hset')
|
|
r_serv_sync.hset(f'ail:instance:{ail_uuid}', 'pull', pull)
|
|
edited = True
|
|
if push is not None:
|
|
# sanityze push
|
|
if push:
|
|
push = True
|
|
else:
|
|
push = False
|
|
if curr_push != push:
|
|
#print('push hset')
|
|
r_serv_sync.hset(f'ail:instance:{ail_uuid}', 'push', push)
|
|
edited = True
|
|
if edited:
|
|
set_last_updated_sync_config()
|
|
refresh_ail_instance_connection(ail_uuid)
|
|
|
|
def get_ail_server_version(ail_uuid):
|
|
return r_serv_sync.hget(f'ail:instance:{ail_uuid}', 'version')
|
|
|
|
def get_ail_server_ping(ail_uuid):
|
|
res = r_serv_sync.hget(f'ail:instance:{ail_uuid}', 'ping')
|
|
return res == 'True'
|
|
|
|
def get_ail_server_error(ail_uuid):
|
|
return r_cache.hget(f'ail_2_ail:all_servers:metadata:{ail_uuid}', 'error')
|
|
|
|
# # TODO: HIDE ADD GLOBAL FILTER (ON BOTH SIDE)
|
|
def get_ail_instance_metadata(ail_uuid, client_sync_mode=False, server_sync_mode=False, sync_queues=False):
|
|
dict_meta = {}
|
|
dict_meta['uuid'] = ail_uuid
|
|
dict_meta['url'] = get_ail_instance_url(ail_uuid)
|
|
dict_meta['description'] = get_ail_instance_description(ail_uuid)
|
|
dict_meta['pull'] = is_ail_instance_pull_enabled(ail_uuid)
|
|
dict_meta['push'] = is_ail_instance_push_enabled(ail_uuid)
|
|
dict_meta['ping'] = get_ail_server_ping(ail_uuid)
|
|
dict_meta['version'] = get_ail_server_version(ail_uuid)
|
|
dict_meta['error'] = get_ail_server_error(ail_uuid)
|
|
|
|
# # TODO: HIDE
|
|
dict_meta['api_key'] = get_ail_instance_key(ail_uuid)
|
|
|
|
if sync_queues:
|
|
dict_meta['sync_queues'] = get_ail_instance_all_sync_queue(ail_uuid)
|
|
|
|
if client_sync_mode:
|
|
dict_meta['client_sync_mode'] = {}
|
|
dict_meta['client_sync_mode']['pull'] = is_server_client_sync_mode_connected(ail_uuid, 'pull')
|
|
dict_meta['client_sync_mode']['push'] = is_server_client_sync_mode_connected(ail_uuid, 'push')
|
|
dict_meta['client_sync_mode']['api'] = is_server_client_sync_mode_connected(ail_uuid, 'api')
|
|
|
|
if server_sync_mode:
|
|
dict_meta['server_sync_mode'] = get_ail_instance_all_running_sync_mode(ail_uuid)
|
|
|
|
return dict_meta
|
|
|
|
def get_all_ail_instances_metadata():
|
|
l_servers = []
|
|
for ail_uuid in get_all_ail_instance():
|
|
l_servers.append(get_ail_instance_metadata(ail_uuid, sync_queues=True))
|
|
return l_servers
|
|
|
|
def get_ail_instances_metadata(l_ail_servers, sync_queues=True, client_sync_mode=False, server_sync_mode=False):
|
|
l_servers = []
|
|
for ail_uuid in l_ail_servers:
|
|
server_metadata = get_ail_instance_metadata(ail_uuid, sync_queues=sync_queues,
|
|
client_sync_mode=client_sync_mode, server_sync_mode=server_sync_mode)
|
|
l_servers.append(server_metadata)
|
|
return l_servers
|
|
|
|
def edit_ail_instance_key(ail_uuid, new_key):
|
|
key = get_ail_instance_key(ail_uuid)
|
|
if new_key and key != new_key:
|
|
r_serv_sync.hset(f'ail:instance:{ail_uuid}', 'api_key', new_key)
|
|
r_serv_sync.srem('ail:instance:key:all', key)
|
|
r_serv_sync.srem('ail:instance:key:all', new_key)
|
|
r_serv_sync.delete(f'ail:instance:key:{key}')
|
|
r_serv_sync.set(f'ail:instance:key:{new_key}', ail_uuid)
|
|
refresh_ail_instance_connection(ail_uuid)
|
|
|
|
def edit_ail_instance_url(ail_uuid, new_url):
|
|
url = get_ail_instance_url(ail_uuid)
|
|
if new_url and new_url != url:
|
|
r_serv_sync.hset(f'ail:instance:{ail_uuid}', 'url', new_url)
|
|
refresh_ail_instance_connection(ail_uuid)
|
|
|
|
def edit_ail_instance_pull_push(ail_uuid, new_pull, new_push):
|
|
pull = is_ail_instance_pull_enabled(ail_uuid)
|
|
push = is_ail_instance_push_enabled(ail_uuid)
|
|
if new_pull == pull:
|
|
new_pull = None
|
|
if new_push == push:
|
|
new_push = None
|
|
change_pull_push_state(ail_uuid, pull=new_pull, push=new_push)
|
|
|
|
def edit_ail_instance_description(ail_uuid, new_description):
|
|
description = get_ail_instance_description(ail_uuid)
|
|
if new_description is not None and description != new_description:
|
|
if not new_description:
|
|
r_serv_sync.hdel(f'ail:instance:{ail_uuid}', 'description')
|
|
else:
|
|
r_serv_sync.hset(f'ail:instance:{ail_uuid}', 'description', new_description)
|
|
|
|
# # TODO: VALIDATE URL
|
|
# API KEY
|
|
def create_ail_instance(ail_uuid, url, api_key=None, description=None, pull=True, push=True):
|
|
r_serv_sync.sadd('ail:instance:all', ail_uuid)
|
|
r_serv_sync.hset(f'ail:instance:{ail_uuid}', 'url', url)
|
|
## API KEY ##
|
|
if not api_key:
|
|
api_key = generate_sync_api_key()
|
|
r_serv_sync.hset(f'ail:instance:{ail_uuid}', 'api_key', api_key)
|
|
r_serv_sync.sadd('ail:instance:key:all', api_key)
|
|
r_serv_sync.set(f'ail:instance:key:{api_key}', ail_uuid)
|
|
#- API KEY -#
|
|
if description:
|
|
r_serv_sync.hset(f'ail:instance:{ail_uuid}', 'description', description)
|
|
change_pull_push_state(ail_uuid, pull=pull, push=push)
|
|
set_last_updated_sync_config()
|
|
refresh_ail_instance_connection(ail_uuid)
|
|
return ail_uuid
|
|
|
|
def delete_ail_instance(ail_uuid):
|
|
for queue_uuid in get_ail_instance_all_sync_queue(ail_uuid):
|
|
unregister_ail_to_sync_queue(ail_uuid, queue_uuid)
|
|
r_serv_sync.delete(f'ail:instance:sync_queue:{ail_uuid}')
|
|
key = get_ail_instance_by_key(ail_uuid)
|
|
r_serv_sync.delete(f'ail:instance:{ail_uuid}')
|
|
r_serv_sync.srem('ail:instance:key:all', ail_uuid)
|
|
r_serv_sync.delete(f'ail:instance:key:{key}', ail_uuid)
|
|
r_serv_sync.srem('ail:instance:all', ail_uuid)
|
|
set_last_updated_sync_config()
|
|
refresh_ail_instance_connection(ail_uuid)
|
|
clear_save_ail_server_error(ail_uuid)
|
|
return ail_uuid
|
|
|
|
## WEBSOCKET API - ERRORS ##
|
|
|
|
def set_ail_server_version(ail_uuid, version):
|
|
r_serv_sync.hset(f'ail:instance:{ail_uuid}', 'version', version)
|
|
|
|
def set_ail_server_ping(ail_uuid, pong):
|
|
r_serv_sync.hset(f'ail:instance:{ail_uuid}', 'ping', bool(pong))
|
|
|
|
def save_ail_server_error(ail_uuid, error_message):
|
|
r_cache.hset(f'ail_2_ail:all_servers:metadata:{ail_uuid}', 'error', error_message)
|
|
|
|
def clear_save_ail_server_error(ail_uuid):
|
|
r_cache.hdel(f'ail_2_ail:all_servers:metadata:{ail_uuid}', 'error')
|
|
|
|
def _get_remote_ail_server_response(ail_uuid, api_request):
|
|
websocket_client = os.path.join(os.environ['AIL_BIN'], 'core', 'ail_2_ail_client.py')
|
|
l_command = ['python', websocket_client, '-u', ail_uuid, '-m', 'api', '-a', api_request]
|
|
process = subprocess.Popen(l_command, stdout=subprocess.PIPE)
|
|
while process.poll() is None:
|
|
time.sleep(1)
|
|
|
|
if process.returncode == 0:
|
|
# Scrapy-Splash ERRORS
|
|
if process.stderr:
|
|
stderr = process.stderr.read().decode()
|
|
if stderr:
|
|
print(f'stderr: {stderr}')
|
|
|
|
if process.stdout:
|
|
output = process.stdout.read().decode()
|
|
#print(output)
|
|
if output:
|
|
try:
|
|
message = json.loads(output)
|
|
return message
|
|
except Exception as e:
|
|
print(e)
|
|
error = f'Error: {e}'
|
|
save_ail_server_error(ail_uuid, error)
|
|
return
|
|
# ERROR
|
|
else:
|
|
if process.stderr:
|
|
stderr = process.stderr.read().decode()
|
|
else:
|
|
stderr = ''
|
|
if process.stdout:
|
|
stdout = process.stdout.read().decode()
|
|
else:
|
|
stdout =''
|
|
if stderr or stdout:
|
|
error = f'-stderr-\n{stderr}\n-stdout-\n{stdout}'
|
|
print(error)
|
|
save_ail_server_error(ail_uuid, error)
|
|
return
|
|
|
|
def get_remote_ail_server_version(ail_uuid):
|
|
response = _get_remote_ail_server_response(ail_uuid, 'version')
|
|
if response:
|
|
version = response.get('version')
|
|
if version:
|
|
version = float(version)
|
|
if version >= 0.1:
|
|
set_ail_server_version(ail_uuid, version)
|
|
return version
|
|
|
|
# # TODO: CATCH WEBSOCKETS RESPONSE CODE
|
|
def ping_remote_ail_server(ail_uuid):
|
|
response = _get_remote_ail_server_response(ail_uuid, 'ping')
|
|
if response:
|
|
response = response.get('message', False)
|
|
pong = response == 'pong'
|
|
else:
|
|
pong = False
|
|
set_ail_server_ping(ail_uuid, pong)
|
|
return pong
|
|
|
|
## API ##
|
|
|
|
def api_ping_remote_ail_server(json_dict):
|
|
ail_uuid = json_dict.get('uuid').replace(' ', '')
|
|
if not is_valid_uuid_v4(ail_uuid):
|
|
return {"status": "error", "reason": "Invalid ail uuid"}, 400
|
|
ail_uuid = sanityze_uuid(ail_uuid)
|
|
if not exists_ail_instance(ail_uuid):
|
|
return {"status": "error", "reason": "AIL server not found"}, 404
|
|
|
|
res = ping_remote_ail_server(ail_uuid)
|
|
return res, 200
|
|
|
|
def api_get_remote_ail_server_version(json_dict):
|
|
ail_uuid = json_dict.get('uuid').replace(' ', '')
|
|
if not is_valid_uuid_v4(ail_uuid):
|
|
return {"status": "error", "reason": "Invalid ail uuid"}, 400
|
|
ail_uuid = sanityze_uuid(ail_uuid)
|
|
if not exists_ail_instance(ail_uuid):
|
|
return {"status": "error", "reason": "AIL server not found"}, 404
|
|
|
|
res = get_remote_ail_server_version(ail_uuid)
|
|
return res, 200
|
|
|
|
def api_kill_server_connected_clients(json_dict):
|
|
ail_uuid = json_dict.get('uuid').replace(' ', '')
|
|
if not is_valid_uuid_v4(ail_uuid):
|
|
return {"status": "error", "reason": "Invalid ail uuid"}, 400
|
|
ail_uuid = sanityze_uuid(ail_uuid)
|
|
if not exists_ail_instance(ail_uuid):
|
|
return {"status": "error", "reason": "AIL server not found"}, 404
|
|
if not is_server_client_connected(ail_uuid):
|
|
return {"status": "error", "reason": "Client not connected"}, 400
|
|
|
|
res = send_command_to_server_controller('kill', ail_uuid=ail_uuid)
|
|
return res, 200
|
|
|
|
def api_kill_sync_client(json_dict):
|
|
ail_uuid = json_dict.get('uuid').replace(' ', '')
|
|
if not is_valid_uuid_v4(ail_uuid):
|
|
return {"status": "error", "reason": "Invalid ail uuid"}, 400
|
|
ail_uuid = sanityze_uuid(ail_uuid)
|
|
if not exists_ail_instance(ail_uuid):
|
|
return {"status": "error", "reason": "AIL server not found"}, 404
|
|
|
|
clients_id = get_client_id_by_ail_uuid(ail_uuid)
|
|
if not clients_id:
|
|
return {"status": "error", "reason": "Client not connected"}, 400
|
|
|
|
for client_id in clients_id:
|
|
res = send_command_to_manager('kill', client_id=client_id, ail_uuid=ail_uuid)
|
|
return res, 200
|
|
|
|
def api_launch_sync_client(json_dict):
|
|
ail_uuid = json_dict.get('uuid').replace(' ', '')
|
|
if not is_valid_uuid_v4(ail_uuid):
|
|
return {"status": "error", "reason": "Invalid ail uuid"}, 400
|
|
ail_uuid = sanityze_uuid(ail_uuid)
|
|
if not exists_ail_instance(ail_uuid):
|
|
return {"status": "error", "reason": "AIL server not found"}, 404
|
|
|
|
clients_id = get_client_id_by_ail_uuid(ail_uuid)
|
|
if clients_id:
|
|
return {"status": "error", "reason": "Client already connected"}, 400
|
|
|
|
res = send_command_to_manager('launch', ail_uuid=ail_uuid)
|
|
return res, 200
|
|
|
|
def api_relaunch_sync_client(json_dict):
|
|
ail_uuid = json_dict.get('uuid').replace(' ', '')
|
|
if not is_valid_uuid_v4(ail_uuid):
|
|
return {"status": "error", "reason": "Invalid ail uuid"}, 400
|
|
ail_uuid = sanityze_uuid(ail_uuid)
|
|
if not exists_ail_instance(ail_uuid):
|
|
return {"status": "error", "reason": "AIL server not found"}, 404
|
|
|
|
clients_id = get_client_id_by_ail_uuid(ail_uuid)
|
|
if not clients_id:
|
|
return {"status": "error", "reason": "Client not connected"}, 400
|
|
for client_id in clients_id:
|
|
res = send_command_to_manager('relaunch', client_id=client_id, ail_uuid=ail_uuid)
|
|
return res, 200
|
|
|
|
def api_create_ail_instance(json_dict):
|
|
ail_uuid = json_dict.get('uuid').replace(' ', '')
|
|
if not is_valid_uuid_v4(ail_uuid):
|
|
return {"status": "error", "reason": "Invalid ail uuid"}, 400
|
|
ail_uuid = sanityze_uuid(ail_uuid)
|
|
if exists_ail_instance(ail_uuid):
|
|
return {"status": "error", "reason": "AIL uuid already exists"}, 400
|
|
|
|
if json_dict.get('pull'):
|
|
pull = True
|
|
else:
|
|
pull = False
|
|
if json_dict.get('push'):
|
|
push = True
|
|
else:
|
|
push = False
|
|
description = json_dict.get('description')
|
|
|
|
ail_url = json_dict.get('url').replace(' ', '')
|
|
if not is_valid_websocket_url(ail_url):
|
|
return {"status": "error", "reason": "Invalid websocket url"}, 400
|
|
|
|
ail_key = json_dict.get('key')
|
|
if ail_key:
|
|
ail_key = ail_key.replace(' ', '')
|
|
if not is_valid_websocket_key(ail_key):
|
|
return {"status": "error", "reason": "Invalid websocket key"}, 400
|
|
|
|
res = create_ail_instance(ail_uuid, ail_url, api_key=ail_key, description=description,
|
|
pull=pull, push=push)
|
|
return res, 200
|
|
|
|
def api_edit_ail_instance(json_dict):
|
|
ail_uuid = json_dict.get('uuid').replace(' ', '')
|
|
if not is_valid_uuid_v4(ail_uuid):
|
|
return {"status": "error", "reason": "Invalid ail uuid"}, 400
|
|
ail_uuid = sanityze_uuid(ail_uuid)
|
|
if not exists_ail_instance(ail_uuid):
|
|
return {"status": "error", "reason": "AIL server not found"}, 404
|
|
|
|
pull = json_dict.get('pull')
|
|
push = json_dict.get('push')
|
|
if pull is not None:
|
|
if pull:
|
|
pull = True
|
|
else:
|
|
pull = False
|
|
if push is not None:
|
|
if push:
|
|
push = True
|
|
else:
|
|
push = False
|
|
edit_ail_instance_pull_push(ail_uuid, pull, push)
|
|
|
|
description = json_dict.get('description')
|
|
edit_ail_instance_description(ail_uuid, description)
|
|
|
|
ail_url = json_dict.get('url')
|
|
if ail_url:
|
|
ail_url = ail_url.replace(' ', '')
|
|
if not is_valid_websocket_url(ail_url):
|
|
return {"status": "error", "reason": "Invalid websocket url"}, 400
|
|
edit_ail_instance_url(ail_uuid, ail_url)
|
|
|
|
ail_key = json_dict.get('key')
|
|
if ail_key:
|
|
ail_key = ail_key.replace(' ', '')
|
|
if not is_valid_websocket_key(ail_key):
|
|
return {"status": "error", "reason": "Invalid websocket key"}, 400
|
|
edit_ail_instance_key(ail_uuid, ail_key)
|
|
|
|
return ail_uuid, 200
|
|
|
|
def api_delete_ail_instance(json_dict):
|
|
ail_uuid = json_dict.get('uuid', '').replace(' ', '')
|
|
if not is_valid_uuid_v4(ail_uuid):
|
|
return {"status": "error", "reason": "Invalid AIL uuid"}, 400
|
|
ail_uuid = sanityze_uuid(ail_uuid)
|
|
if not exists_ail_instance(ail_uuid):
|
|
return {"status": "error", "reason": "AIL server not found"}, 404
|
|
|
|
res = delete_ail_instance(ail_uuid)
|
|
return res, 200
|
|
|
|
####################
|
|
# #
|
|
#### SYNC QUEUE ####
|
|
|
|
class Sync_Queue(object): # # TODO: use for edit
|
|
"""Sync_Queue."""
|
|
|
|
def __init__(self, uuid):
|
|
self.uuid = uuid
|
|
|
|
def get_all_sync_queue():
|
|
return r_serv_sync.smembers('ail2ail:sync_queue:all')
|
|
|
|
def get_sync_queue_all_ail_instance(queue_uuid):
|
|
return r_serv_sync.smembers(f'ail2ail:sync_queue:ail_instance:{queue_uuid}')
|
|
|
|
def exists_sync_queue(queue_uuid):
|
|
return r_serv_sync.exists(f'ail2ail:sync_queue:{queue_uuid}')
|
|
|
|
# # TODO: check if push or pull enabled ?
|
|
def is_queue_used_by_ail_instance(queue_uuid):
|
|
return r_serv_sync.exists(f'ail2ail:sync_queue:ail_instance:{queue_uuid}')
|
|
|
|
# # TODO: add others filter
|
|
def get_sync_queue_filter(queue_uuid):
|
|
return r_serv_sync.smembers(f'ail2ail:sync_queue:filter:tags:{queue_uuid}')
|
|
|
|
def get_sync_queue_name(queue_uuid):
|
|
return r_serv_sync.hget(f'ail2ail:sync_queue:{queue_uuid}', 'name')
|
|
|
|
def get_sync_queue_description(queue_uuid):
|
|
return r_serv_sync.hget(f'ail2ail:sync_queue:{queue_uuid}', 'description')
|
|
|
|
def get_sync_queue_max_size(queue_uuid):
|
|
return r_serv_sync.hget(f'ail2ail:sync_queue:{queue_uuid}', 'max_size')
|
|
|
|
# # TODO: ADD FILTER
|
|
def get_sync_queue_metadata(queue_uuid):
|
|
dict_meta = {}
|
|
dict_meta['uuid'] = queue_uuid
|
|
dict_meta['name'] = get_sync_queue_name(queue_uuid)
|
|
dict_meta['description'] = get_sync_queue_description(queue_uuid)
|
|
dict_meta['max_size'] = get_sync_queue_max_size(queue_uuid)
|
|
dict_meta['tags'] = get_sync_queue_filter(queue_uuid)
|
|
|
|
# # TODO: TO ADD:
|
|
# - get uuid instance
|
|
|
|
return dict_meta
|
|
|
|
def get_all_queues_metadata():
|
|
l_queues = []
|
|
for queue_uuid in get_all_sync_queue():
|
|
l_queues.append(get_sync_queue_metadata(queue_uuid))
|
|
return l_queues
|
|
|
|
def get_queues_metadata(l_queues_uuid):
|
|
l_queues = []
|
|
for queue_uuid in l_queues_uuid:
|
|
l_queues.append(get_sync_queue_metadata(queue_uuid))
|
|
return l_queues
|
|
|
|
#####################################################
|
|
def get_all_sync_queue_dict():
|
|
dict_sync_queues = {}
|
|
for queue_uuid in get_all_sync_queue():
|
|
if is_queue_used_by_ail_instance(queue_uuid):
|
|
dict_queue = {}
|
|
dict_queue['filter'] = get_sync_queue_filter(queue_uuid)
|
|
|
|
dict_queue['ail_instances'] = [] ############ USE DICT ?????????
|
|
for ail_uuid in get_sync_queue_all_ail_instance(queue_uuid):
|
|
dict_ail = {'ail_uuid': ail_uuid,
|
|
'pull': is_ail_instance_pull_enabled(ail_uuid),
|
|
'push': is_ail_instance_push_enabled(ail_uuid)}
|
|
if dict_ail['pull'] or dict_ail['push']:
|
|
dict_queue['ail_instances'].append(dict_ail)
|
|
if dict_queue['ail_instances']:
|
|
dict_sync_queues[queue_uuid] = dict_queue
|
|
return dict_sync_queues
|
|
|
|
def is_queue_registred_by_ail_instance(queue_uuid, ail_uuid):
|
|
return r_serv_sync.sismember(f'ail:instance:sync_queue:{ail_uuid}', queue_uuid)
|
|
|
|
def register_ail_to_sync_queue(ail_uuid, queue_uuid):
|
|
is_linked = is_ail_instance_linked_to_sync_queue(ail_uuid)
|
|
r_serv_sync.sadd(f'ail2ail:sync_queue:ail_instance:{queue_uuid}', ail_uuid)
|
|
r_serv_sync.sadd(f'ail:instance:sync_queue:{ail_uuid}', queue_uuid)
|
|
set_last_updated_sync_config()
|
|
if not is_linked:
|
|
refresh_ail_instance_connection(ail_uuid)
|
|
|
|
# # # FIXME: TODO: delete sync queue ????????????????????????????????????????????????????
|
|
def unregister_ail_to_sync_queue(ail_uuid, queue_uuid):
|
|
r_serv_sync.srem(f'ail2ail:sync_queue:ail_instance:{queue_uuid}', ail_uuid)
|
|
r_serv_sync.srem(f'ail:instance:sync_queue:{ail_uuid}', queue_uuid)
|
|
set_last_updated_sync_config()
|
|
is_linked = is_ail_instance_linked_to_sync_queue(ail_uuid)
|
|
if not is_linked:
|
|
refresh_ail_instance_connection(ail_uuid)
|
|
|
|
def get_all_unregistred_queue_by_ail_instance(ail_uuid):
|
|
return r_serv_sync.sdiff('ail2ail:sync_queue:all', f'ail:instance:sync_queue:{ail_uuid}')
|
|
|
|
def edit_sync_queue_name(queue_uuid, new_name):
|
|
name = get_sync_queue_name(queue_uuid)
|
|
if new_name and new_name != name:
|
|
r_serv_sync.hset(f'ail2ail:sync_queue:{queue_uuid}', 'name', new_name)
|
|
|
|
def edit_sync_queue_description(queue_uuid, new_description):
|
|
description = get_sync_queue_description(queue_uuid)
|
|
if new_description is not None and new_description != description:
|
|
r_serv_sync.hset(f'ail2ail:sync_queue:{queue_uuid}', 'description', new_description)
|
|
|
|
# # TODO: trigger update
|
|
def edit_sync_queue_max_size(queue_uuid, new_max_size):
|
|
max_size = get_sync_queue_max_size(queue_uuid)
|
|
if new_max_size > 0 and new_max_size != max_size:
|
|
r_serv_sync.hset(f'ail2ail:sync_queue:{queue_uuid}', 'max_size', new_max_size)
|
|
|
|
def edit_sync_queue_filter_tags(queue_uuid, new_tags):
|
|
tags = set(get_sync_queue_filter(queue_uuid))
|
|
new_tags = set(new_tags)
|
|
if new_tags and new_tags != tags:
|
|
r_serv_sync.delete(f'ail2ail:sync_queue:filter:tags:{queue_uuid}')
|
|
for tag in new_tags:
|
|
r_serv_sync.sadd(f'ail2ail:sync_queue:filter:tags:{queue_uuid}', tag)
|
|
set_last_updated_sync_config()
|
|
|
|
# # TODO: optionnal name ???
|
|
# # TODO: SANITYZE TAGS
|
|
# # TODO: SANITYZE queue_uuid
|
|
def create_sync_queue(name, tags=[], description=None, max_size=100, _queue_uuid=None):
|
|
if _queue_uuid:
|
|
queue_uuid = sanityze_uuid(_queue_uuid).replace('-', '')
|
|
else:
|
|
queue_uuid = generate_uuid()
|
|
r_serv_sync.sadd('ail2ail:sync_queue:all', queue_uuid)
|
|
|
|
r_serv_sync.hset(f'ail2ail:sync_queue:{queue_uuid}', 'name', name)
|
|
if description:
|
|
r_serv_sync.hset(f'ail2ail:sync_queue:{queue_uuid}', 'description', description)
|
|
r_serv_sync.hset(f'ail2ail:sync_queue:{queue_uuid}', 'max_size', max_size)
|
|
|
|
for tag in tags:
|
|
r_serv_sync.sadd(f'ail2ail:sync_queue:filter:tags:{queue_uuid}', tag)
|
|
|
|
set_last_updated_sync_config()
|
|
return queue_uuid
|
|
|
|
def delete_sync_queue(queue_uuid):
|
|
for ail_uuid in get_sync_queue_all_ail_instance(queue_uuid):
|
|
unregister_ail_to_sync_queue(ail_uuid, queue_uuid)
|
|
r_serv_sync.delete(f'ail2ail:sync_queue:{queue_uuid}')
|
|
r_serv_sync.delete(f'ail2ail:sync_queue:filter:tags:{queue_uuid}')
|
|
r_serv_sync.srem('ail2ail:sync_queue:all', queue_uuid)
|
|
set_last_updated_sync_config()
|
|
return queue_uuid
|
|
|
|
## API ##
|
|
|
|
# # TODO: sanityze queue_name
|
|
def api_create_sync_queue(json_dict):
|
|
description = json_dict.get('description')
|
|
description = escape(description)
|
|
queue_name = json_dict.get('name')
|
|
if queue_name: #################################################
|
|
queue_name = escape(queue_name)
|
|
|
|
tags = json_dict.get('tags')
|
|
if not tags:
|
|
return {"status": "error", "reason": "no tags provided"}, 400
|
|
# FIXME: add custom tags
|
|
# if not Tag.are_enabled_tags(tags):
|
|
# return {"status": "error", "reason": "Invalid/Disabled tags"}, 400
|
|
|
|
max_size = json_dict.get('max_size')
|
|
if not max_size:
|
|
max_size = 100
|
|
try:
|
|
max_size = int(max_size)
|
|
except ValueError:
|
|
return {"status": "error", "reason": "Invalid queue size value"}, 400
|
|
if not max_size > 0:
|
|
return {"status": "error", "reason": "Invalid queue size value"}, 400
|
|
|
|
queue_uuid = create_sync_queue(queue_name, tags=tags, description=description,
|
|
max_size=max_size)
|
|
return queue_uuid, 200
|
|
|
|
def api_edit_sync_queue(json_dict):
|
|
queue_uuid = json_dict.get('uuid', '').replace(' ', '').replace('-', '')
|
|
if not is_valid_uuid_v4(queue_uuid):
|
|
return {"status": "error", "reason": "Invalid Queue uuid"}, 400
|
|
if not exists_sync_queue(queue_uuid):
|
|
return {"status": "error", "reason": "Queue Sync not found"}, 404
|
|
|
|
description = json_dict.get('description')
|
|
description = escape(description)
|
|
if description is not None:
|
|
edit_sync_queue_description(queue_uuid, description)
|
|
|
|
queue_name = json_dict.get('name')
|
|
if queue_name:
|
|
queue_name = escape(queue_name)
|
|
edit_sync_queue_name(queue_uuid, queue_name)
|
|
|
|
tags = json_dict.get('tags')
|
|
if tags:
|
|
# FIXME: add custom tags
|
|
# if not Tag.are_enabled_tags(tags):
|
|
# return {"status": "error", "reason": "Invalid/Disabled tags"}, 400
|
|
edit_sync_queue_filter_tags(queue_uuid, tags)
|
|
|
|
max_size = json_dict.get('max_size')
|
|
if max_size:
|
|
try:
|
|
max_size = int(max_size)
|
|
except ValueError:
|
|
return {"status": "error", "reason": "Invalid queue size value"}, 400
|
|
if not max_size > 0:
|
|
return {"status": "error", "reason": "Invalid queue size value"}, 400
|
|
edit_sync_queue_max_size(queue_uuid, max_size)
|
|
|
|
return queue_uuid, 200
|
|
|
|
def api_delete_sync_queue(json_dict):
|
|
queue_uuid = json_dict.get('uuid', '').replace(' ', '').replace('-', '')
|
|
if not is_valid_uuid_v4(queue_uuid):
|
|
return {"status": "error", "reason": "Invalid Queue uuid"}, 400
|
|
if not exists_sync_queue(queue_uuid):
|
|
return {"status": "error", "reason": "Queue Sync not found"}, 404
|
|
|
|
res = delete_sync_queue(queue_uuid)
|
|
return res, 200
|
|
|
|
def api_register_ail_to_sync_queue(json_dict):
|
|
ail_uuid = json_dict.get('ail_uuid', '').replace(' ', '')
|
|
if not is_valid_uuid_v4(ail_uuid):
|
|
return {"status": "error", "reason": "Invalid AIL uuid"}, 400
|
|
ail_uuid = sanityze_uuid(ail_uuid)
|
|
queue_uuid = json_dict.get('queue_uuid', '').replace(' ', '').replace('-', '')
|
|
if not is_valid_uuid_v4(queue_uuid):
|
|
return {"status": "error", "reason": "Invalid Queue uuid"}, 400
|
|
|
|
if not exists_ail_instance(ail_uuid):
|
|
return {"status": "error", "reason": "AIL server not found"}, 404
|
|
if not exists_sync_queue(queue_uuid):
|
|
return {"status": "error", "reason": "Queue Sync not found"}, 404
|
|
if is_queue_registred_by_ail_instance(queue_uuid, ail_uuid):
|
|
return {"status": "error", "reason": "Queue already registred"}, 400
|
|
|
|
res = register_ail_to_sync_queue(ail_uuid, queue_uuid)
|
|
return res, 200
|
|
|
|
def api_unregister_ail_to_sync_queue(json_dict):
|
|
ail_uuid = json_dict.get('ail_uuid', '').replace(' ', '')
|
|
if not is_valid_uuid_v4(ail_uuid):
|
|
return {"status": "error", "reason": "Invalid ail uuid"}, 400
|
|
ail_uuid = sanityze_uuid(ail_uuid)
|
|
queue_uuid = json_dict.get('queue_uuid', '').replace(' ', '').replace('-', '')
|
|
if not is_valid_uuid_v4(queue_uuid):
|
|
return {"status": "error", "reason": "Invalid ail uuid"}, 400
|
|
|
|
if not exists_ail_instance(ail_uuid):
|
|
return {"status": "error", "reason": "AIL server not found"}, 404
|
|
if not exists_sync_queue(queue_uuid):
|
|
return {"status": "error", "reason": "Queue Sync not found"}, 404
|
|
if not is_queue_registred_by_ail_instance(queue_uuid, ail_uuid):
|
|
return {"status": "error", "reason": "Queue not registred"}, 400
|
|
|
|
res = unregister_ail_to_sync_queue(ail_uuid, queue_uuid)
|
|
return res, 200
|
|
|
|
#############################
|
|
# #
|
|
#### SYNC REDIS QUEUE #######
|
|
|
|
def get_sync_queue_object_and_queue_uuid(ail_uuid, push=True):
|
|
for queue_uuid in get_ail_instance_all_sync_queue(ail_uuid):
|
|
obj_dict = get_sync_queue_object_by_queue_uuid(queue_uuid, ail_uuid, push=push)
|
|
if obj_dict:
|
|
return obj_dict, queue_uuid
|
|
return None, None
|
|
|
|
def get_sync_queue_object(ail_uuid, push=True):
|
|
obj_dict, queue_uuid = get_sync_queue_object_and_queue_uuid(ail_uuid, push=push)[0]
|
|
return obj_dict
|
|
|
|
def get_sync_queue_object_by_queue_uuid(queue_uuid, ail_uuid, push=True):
|
|
if push:
|
|
sync_mode = 'push'
|
|
else:
|
|
sync_mode = 'pull'
|
|
obj_dict = r_serv_sync.lpop(f'sync:queue:{sync_mode}:{queue_uuid}:{ail_uuid}')
|
|
if obj_dict:
|
|
obj_dict = json.loads(obj_dict)
|
|
# # REVIEW: # TODO: create by obj type
|
|
return Item(obj_dict['id'])
|
|
|
|
def get_sync_queue_objects_by_queue_uuid(queue_uuid, ail_uuid, push=True):
|
|
if push:
|
|
sync_mode = 'push'
|
|
else:
|
|
sync_mode = 'pull'
|
|
return r_serv_sync.lrange(f'sync:queue:{sync_mode}:{queue_uuid}:{ail_uuid}', 0, -1)
|
|
|
|
# # TODO: use queue max_size
|
|
def add_object_to_sync_queue(queue_uuid, ail_uuid, obj_dict, push=True, pull=True, json_obj=True):
|
|
if json_obj:
|
|
obj = json.dumps(obj_dict)
|
|
else:
|
|
obj = obj_dict
|
|
|
|
# # TODO: # FIXME: USE CACHE ??????
|
|
if push:
|
|
r_serv_sync.lpush(f'sync:queue:push:{queue_uuid}:{ail_uuid}', obj)
|
|
r_serv_sync.ltrim(f'sync:queue:push:{queue_uuid}:{ail_uuid}', 0, 200)
|
|
|
|
if pull:
|
|
r_serv_sync.lpush(f'sync:queue:pull:{queue_uuid}:{ail_uuid}', obj)
|
|
r_serv_sync.ltrim(f'sync:queue:pull:{queue_uuid}:{ail_uuid}', 0, 200)
|
|
|
|
def resend_object_to_sync_queue(ail_uuid, queue_uuid, Obj, push=True):
|
|
if queue_uuid is not None and Obj is not None:
|
|
obj_dict = Obj.get_default_meta()
|
|
if push:
|
|
pull = False
|
|
else:
|
|
pull = True
|
|
add_object_to_sync_queue(queue_uuid, ail_uuid, obj_dict, push=push, pull=pull)
|
|
|
|
# # TODO: # REVIEW: USE CACHE ????? USE QUEUE FACTORY ?????
|
|
def get_sync_importer_ail_stream():
|
|
return r_serv_sync.spop('sync:queue:importer')
|
|
|
|
def add_ail_stream_to_sync_importer(ail_stream):
|
|
ail_stream = json.dumps(ail_stream)
|
|
r_serv_sync.sadd('sync:queue:importer', ail_stream)
|
|
|
|
#############################
|
|
# #
|
|
#### AIL EXCHANGE FORMAT ####
|
|
|
|
# TODO
|
|
def is_valid_ail_exchange_format_json(json_obj):
|
|
try:
|
|
ail_stream = json.dumps(json_obj)
|
|
except ValueError:
|
|
return False
|
|
return is_valid_ail_exchange_format(ail_stream)
|
|
|
|
# TODO
|
|
def is_valid_ail_exchange_format(ail_stream):
|
|
pass
|
|
|
|
def create_ail_stream(Object):
|
|
ail_stream = {'format': 'ail',
|
|
'version': 1,
|
|
'type': Object.get_type()}
|
|
|
|
# OBJECT META
|
|
ail_stream['meta'] = {'ail:mime-type': 'text/plain'}
|
|
ail_stream['meta']['compress'] = 'gzip'
|
|
ail_stream['meta']['encoding'] = 'base64'
|
|
ail_stream['meta']['ail:id'] = Object.get_id()
|
|
ail_stream['meta']['tags'] = Object.get_tags()
|
|
# GLOBAL META
|
|
ail_stream['meta']['uuid_org'] = get_ail_uuid()
|
|
|
|
# OBJECT PAYLOAD
|
|
ail_stream['payload'] = Object.get_ail_2_ail_payload()
|
|
|
|
return ail_stream
|
|
|
|
if __name__ == '__main__':
|
|
|
|
ail_uuid = '03c51929-eeab-4d47-9dc0-c667f94c7d2d'
|
|
url = "wss://localhost:4443"
|
|
api_key = 'secret'
|
|
#description = 'first test instance'
|
|
queue_uuid = '79bcafc0a6d644deb2c75fb5a83d7caa'
|
|
tags = ['infoleak:submission="manual"']
|
|
name = 'submitted queue'
|
|
description = 'first test queue, all submitted items'
|
|
#queue_uuid = ''
|
|
|
|
#res = create_ail_instance(ail_uuid, url, api_key=api_key, description=description)
|
|
|
|
#res = create_sync_queue(name, tags=tags, description=description, max_size=100)
|
|
#res = delete_sync_queue(queue_uuid)
|
|
|
|
#res = register_ail_to_sync_queue(ail_uuid, queue_uuid)
|
|
#res = change_pull_push_state(ail_uuid, push=True, pull=True)
|
|
|
|
# print(get_ail_instance_all_sync_queue(ail_uuid))
|
|
# print(get_all_sync_queue())
|
|
# res = get_all_unregistred_queue_by_ail_instance(ail_uuid)
|
|
|
|
ail_uuid = 'c3c2f3ef-ca53-4ff6-8317-51169b73f731'
|
|
#ail_uuid = '2dfeff47-777d-4e70-8c30-07c059307e6a'
|
|
|
|
# res = ping_remote_ail_server(ail_uuid)
|
|
# print(res)
|
|
#
|
|
res = send_command_to_server_controller('kill', ail_uuid=ail_uuid)
|
|
|
|
#res = _get_remote_ail_server_response(ail_uuid, 'pin')
|
|
print(res)
|