mirror of https://github.com/CIRCL/AIL-framework
Prepared moduleInfo to handle module click with curses (will come back later).
Mokaddem
parent
d37b243e1d
commit
73938cf41b
|
|
@ -25,9 +25,9 @@ import json
|
||||||
from terminaltables import AsciiTable
|
from terminaltables import AsciiTable
|
||||||
import textwrap
|
import textwrap
|
||||||
from colorama import Fore, Back, Style, init
|
from colorama import Fore, Back, Style, init
|
||||||
|
import curses
|
||||||
|
|
||||||
# CONFIG VARIABLES
|
# CONFIG VARIABLES
|
||||||
threshold_stucked_module = 60*10*1 #1 hour
|
|
||||||
kill_retry_threshold = 60 #1m
|
kill_retry_threshold = 60 #1m
|
||||||
log_filename = "../logs/moduleInfo.log"
|
log_filename = "../logs/moduleInfo.log"
|
||||||
command_search_pid = "ps a -o pid,cmd | grep {}"
|
command_search_pid = "ps a -o pid,cmd | grep {}"
|
||||||
|
|
@ -39,6 +39,15 @@ printarrayGlob = [None]*14
|
||||||
printarrayGlob.insert(0, ["Time", "Module", "PID", "Action"])
|
printarrayGlob.insert(0, ["Time", "Module", "PID", "Action"])
|
||||||
lastTimeKillCommand = {}
|
lastTimeKillCommand = {}
|
||||||
|
|
||||||
|
#Curses init
|
||||||
|
#stdscr = curses.initscr()
|
||||||
|
#curses.cbreak()
|
||||||
|
#stdscr.keypad(1)
|
||||||
|
|
||||||
|
# GLOBAL
|
||||||
|
last_refresh = 0
|
||||||
|
|
||||||
|
|
||||||
def getPid(module):
|
def getPid(module):
|
||||||
p = Popen([command_search_pid.format(module+".py")], stdin=PIPE, stdout=PIPE, bufsize=1, shell=True)
|
p = Popen([command_search_pid.format(module+".py")], stdin=PIPE, stdout=PIPE, bufsize=1, shell=True)
|
||||||
for line in p.stdout:
|
for line in p.stdout:
|
||||||
|
|
@ -145,23 +154,33 @@ def get_color(time, idle):
|
||||||
temp = time.split(':')
|
temp = time.split(':')
|
||||||
time = int(temp[0])*3600 + int(temp[1])*60 + int(temp[2])
|
time = int(temp[0])*3600 + int(temp[1])*60 + int(temp[2])
|
||||||
|
|
||||||
if time >= threshold_stucked_module:
|
if time >= args.treshold:
|
||||||
if not idle:
|
if not idle:
|
||||||
return Back.RED + Style.BRIGHT
|
return Back.RED + Style.BRIGHT
|
||||||
else:
|
else:
|
||||||
return Back.MAGENTA + Style.BRIGHT
|
return Back.MAGENTA + Style.BRIGHT
|
||||||
elif time > threshold_stucked_module/2:
|
elif time > args.treshold/2:
|
||||||
return Back.YELLOW + Style.BRIGHT
|
return Back.YELLOW + Style.BRIGHT
|
||||||
else:
|
else:
|
||||||
return Back.GREEN + Style.BRIGHT
|
return Back.GREEN + Style.BRIGHT
|
||||||
else:
|
else:
|
||||||
return Style.RESET_ALL
|
return Style.RESET_ALL
|
||||||
|
|
||||||
|
def waiting_refresh():
|
||||||
|
global last_refresh
|
||||||
|
if time.time() - last_refresh < args.refresh:
|
||||||
|
return False
|
||||||
|
else:
|
||||||
|
last_refresh = time.time()
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|
||||||
parser = argparse.ArgumentParser(description='Show info concerning running modules and log suspected stucked modules. May be use to automatically kill and restart stucked one.')
|
parser = argparse.ArgumentParser(description='Show info concerning running modules and log suspected stucked modules. May be use to automatically kill and restart stucked one.')
|
||||||
parser.add_argument('-r', '--refresh', type=int, required=False, default=1, help='Refresh rate')
|
parser.add_argument('-r', '--refresh', type=int, required=False, default=1, help='Refresh rate')
|
||||||
|
parser.add_argument('-t', '--treshold', type=int, required=False, default=60*10*1, help='Refresh rate')
|
||||||
parser.add_argument('-k', '--autokill', type=int, required=False, default=0, help='Enable auto kill option (1 for TRUE, anything else for FALSE)')
|
parser.add_argument('-k', '--autokill', type=int, required=False, default=0, help='Enable auto kill option (1 for TRUE, anything else for FALSE)')
|
||||||
parser.add_argument('-c', '--clear', type=int, required=False, default=0, help='Clear the current module information (Used to clear data from old launched modules)')
|
parser.add_argument('-c', '--clear', type=int, required=False, default=0, help='Clear the current module information (Used to clear data from old launched modules)')
|
||||||
|
|
||||||
|
|
@ -176,8 +195,6 @@ if __name__ == "__main__":
|
||||||
cfg = ConfigParser.ConfigParser()
|
cfg = ConfigParser.ConfigParser()
|
||||||
cfg.read(configfile)
|
cfg.read(configfile)
|
||||||
|
|
||||||
threshold_stucked_module = cfg.getint("Module_ModuleInformation", "threshold_stucked_module")
|
|
||||||
|
|
||||||
# REDIS #
|
# REDIS #
|
||||||
server = redis.StrictRedis(
|
server = redis.StrictRedis(
|
||||||
host=cfg.get("Redis_Queues", "host"),
|
host=cfg.get("Redis_Queues", "host"),
|
||||||
|
|
@ -199,130 +216,138 @@ if __name__ == "__main__":
|
||||||
cleanRedis()
|
cleanRedis()
|
||||||
|
|
||||||
while True:
|
while True:
|
||||||
|
if waiting_refresh():
|
||||||
|
|
||||||
all_queue = set()
|
#key = ''
|
||||||
printarray1 = []
|
#while key != 'q':
|
||||||
printarray2 = []
|
# key = stdsrc.getch()
|
||||||
printarray3 = []
|
# stdscr.refresh()
|
||||||
for queue, card in server.hgetall("queues").iteritems():
|
|
||||||
all_queue.add(queue)
|
all_queue = set()
|
||||||
key = "MODULE_" + queue + "_"
|
printarray1 = []
|
||||||
keySet = "MODULE_TYPE_" + queue
|
printarray2 = []
|
||||||
array_module_type = []
|
printarray3 = []
|
||||||
|
for queue, card in server.hgetall("queues").iteritems():
|
||||||
for moduleNum in server.smembers(keySet):
|
all_queue.add(queue)
|
||||||
value = server.get(key + str(moduleNum))
|
key = "MODULE_" + queue + "_"
|
||||||
if value is not None:
|
keySet = "MODULE_TYPE_" + queue
|
||||||
timestamp, path = value.split(", ")
|
array_module_type = []
|
||||||
if timestamp is not None and path is not None:
|
|
||||||
startTime_readable = datetime.datetime.fromtimestamp(int(timestamp))
|
for moduleNum in server.smembers(keySet):
|
||||||
processed_time_readable = str((datetime.datetime.now() - startTime_readable)).split('.')[0]
|
value = server.get(key + str(moduleNum))
|
||||||
|
if value is not None:
|
||||||
if int(card) > 0:
|
timestamp, path = value.split(", ")
|
||||||
if int((datetime.datetime.now() - startTime_readable).total_seconds()) > threshold_stucked_module:
|
if timestamp is not None and path is not None:
|
||||||
log = open(log_filename, 'a')
|
startTime_readable = datetime.datetime.fromtimestamp(int(timestamp))
|
||||||
log.write(json.dumps([queue, card, str(startTime_readable), str(processed_time_readable), path]) + "\n")
|
processed_time_readable = str((datetime.datetime.now() - startTime_readable)).split('.')[0]
|
||||||
try:
|
|
||||||
last_kill_try = time.time() - lastTimeKillCommand[moduleNum]
|
if int(card) > 0:
|
||||||
except KeyError:
|
if int((datetime.datetime.now() - startTime_readable).total_seconds()) > args.treshold:
|
||||||
last_kill_try = kill_retry_threshold+1
|
log = open(log_filename, 'a')
|
||||||
if args.autokill == 1 and last_kill_try > kill_retry_threshold :
|
log.write(json.dumps([queue, card, str(startTime_readable), str(processed_time_readable), path]) + "\n")
|
||||||
kill_module(queue, int(moduleNum))
|
try:
|
||||||
|
last_kill_try = time.time() - lastTimeKillCommand[moduleNum]
|
||||||
array_module_type.append([get_color(processed_time_readable, False) + str(queue), str(moduleNum), str(card), str(startTime_readable), str(processed_time_readable), str(path) + get_color(None, False)])
|
except KeyError:
|
||||||
|
last_kill_try = kill_retry_threshold+1
|
||||||
|
if args.autokill == 1 and last_kill_try > kill_retry_threshold :
|
||||||
|
kill_module(queue, int(moduleNum))
|
||||||
|
|
||||||
|
array_module_type.append([get_color(processed_time_readable, False) + str(queue), str(moduleNum), str(card), str(startTime_readable), str(processed_time_readable), str(path) + get_color(None, False)])
|
||||||
|
|
||||||
|
else:
|
||||||
|
printarray2.append([get_color(processed_time_readable, True) + str(queue), str(moduleNum), str(card), str(startTime_readable), str(processed_time_readable), str(path) + get_color(None, True)])
|
||||||
|
array_module_type.sort(lambda x,y: cmp(x[4], y[4]), reverse=True)
|
||||||
|
for e in array_module_type:
|
||||||
|
printarray1.append(e)
|
||||||
|
|
||||||
|
for curr_queue in module_file_array:
|
||||||
|
if curr_queue not in all_queue:
|
||||||
|
printarray3.append([curr_queue, "Not running"])
|
||||||
|
else:
|
||||||
|
if len(list(server.smembers('MODULE_TYPE_'+curr_queue))) == 0:
|
||||||
|
if curr_queue not in no_info_modules:
|
||||||
|
no_info_modules[curr_queue] = int(time.time())
|
||||||
|
printarray3.append([curr_queue, "No data"])
|
||||||
else:
|
else:
|
||||||
printarray2.append([get_color(processed_time_readable, True) + str(queue), str(moduleNum), str(card), str(startTime_readable), str(processed_time_readable), str(path) + get_color(None, True)])
|
#If no info since long time, try to kill
|
||||||
array_module_type.sort(lambda x,y: cmp(x[4], y[4]), reverse=True)
|
if args.autokill == 1:
|
||||||
for e in array_module_type:
|
if int(time.time()) - no_info_modules[curr_queue] > args.treshold:
|
||||||
printarray1.append(e)
|
kill_module(curr_queue, None)
|
||||||
|
no_info_modules[curr_queue] = int(time.time())
|
||||||
for curr_queue in module_file_array:
|
printarray3.append([curr_queue, "Stuck or idle, restarting in " + str(abs(args.treshold - (int(time.time()) - no_info_modules[curr_queue]))) + "s"])
|
||||||
if curr_queue not in all_queue:
|
else:
|
||||||
printarray3.append([curr_queue, "Not running"])
|
printarray3.append([curr_queue, "Stuck or idle, restarting disabled"])
|
||||||
else:
|
|
||||||
if len(list(server.smembers('MODULE_TYPE_'+curr_queue))) == 0:
|
## FIXME To add:
|
||||||
if curr_queue not in no_info_modules:
|
## Button KILL Process using Curses
|
||||||
no_info_modules[curr_queue] = int(time.time())
|
|
||||||
printarray3.append([curr_queue, "No data"])
|
printarray1.sort(key=lambda x: x[0][9:], reverse=False)
|
||||||
else:
|
printarray2.sort(key=lambda x: x[0][9:], reverse=False)
|
||||||
#If no info since long time, try to kill
|
printarray1.insert(0,["Queue", "PID", "Amount", "Paste start time", "Processing time for current paste (H:M:S)", "Paste hash"])
|
||||||
if args.autokill == 1:
|
printarray2.insert(0,["Queue", "PID","Amount", "Paste start time", "Time since idle (H:M:S)", "Last paste hash"])
|
||||||
if int(time.time()) - no_info_modules[curr_queue] > threshold_stucked_module:
|
printarray3.insert(0,["Queue", "State"])
|
||||||
kill_module(curr_queue, None)
|
|
||||||
no_info_modules[curr_queue] = int(time.time())
|
os.system('clear')
|
||||||
printarray3.append([curr_queue, "Stuck or idle, restarting in " + str(abs(threshold_stucked_module - (int(time.time()) - no_info_modules[curr_queue]))) + "s"])
|
t1 = AsciiTable(printarray1, title="Working queues")
|
||||||
else:
|
t1.column_max_width(1)
|
||||||
printarray3.append([curr_queue, "Stuck or idle, restarting disabled"])
|
if not t1.ok:
|
||||||
|
longest_col = t1.column_widths.index(max(t1.column_widths))
|
||||||
|
max_length_col = t1.column_max_width(longest_col)
|
||||||
printarray1.sort(key=lambda x: x[0][9:], reverse=False)
|
if max_length_col > 0:
|
||||||
printarray2.sort(key=lambda x: x[0][9:], reverse=False)
|
for i, content in enumerate(t1.table_data):
|
||||||
printarray1.insert(0,["Queue", "PID", "Amount", "Paste start time", "Processing time for current paste (H:M:S)", "Paste hash"])
|
if len(content[longest_col]) > max_length_col:
|
||||||
printarray2.insert(0,["Queue", "PID","Amount", "Paste start time", "Time since idle (H:M:S)", "Last paste hash"])
|
temp = ''
|
||||||
printarray3.insert(0,["Queue", "State"])
|
for l in content[longest_col].splitlines():
|
||||||
|
if len(l) > max_length_col:
|
||||||
os.system('clear')
|
temp += '\n'.join(textwrap.wrap(l, max_length_col)) + '\n'
|
||||||
t1 = AsciiTable(printarray1, title="Working queues")
|
else:
|
||||||
t1.column_max_width(1)
|
temp += l + '\n'
|
||||||
if not t1.ok:
|
content[longest_col] = temp.strip()
|
||||||
longest_col = t1.column_widths.index(max(t1.column_widths))
|
t1.table_data[i] = content
|
||||||
max_length_col = t1.column_max_width(longest_col)
|
|
||||||
if max_length_col > 0:
|
t2 = AsciiTable(printarray2, title="Idling queues")
|
||||||
for i, content in enumerate(t1.table_data):
|
t2.column_max_width(1)
|
||||||
if len(content[longest_col]) > max_length_col:
|
if not t2.ok:
|
||||||
temp = ''
|
longest_col = t2.column_widths.index(max(t2.column_widths))
|
||||||
for l in content[longest_col].splitlines():
|
max_length_col = t2.column_max_width(longest_col)
|
||||||
if len(l) > max_length_col:
|
if max_length_col > 0:
|
||||||
temp += '\n'.join(textwrap.wrap(l, max_length_col)) + '\n'
|
for i, content in enumerate(t2.table_data):
|
||||||
else:
|
if len(content[longest_col]) > max_length_col:
|
||||||
temp += l + '\n'
|
temp = ''
|
||||||
content[longest_col] = temp.strip()
|
for l in content[longest_col].splitlines():
|
||||||
t1.table_data[i] = content
|
if len(l) > max_length_col:
|
||||||
|
temp += '\n'.join(textwrap.wrap(l, max_length_col)) + '\n'
|
||||||
t2 = AsciiTable(printarray2, title="Idling queues")
|
else:
|
||||||
t2.column_max_width(1)
|
temp += l + '\n'
|
||||||
if not t2.ok:
|
content[longest_col] = temp.strip()
|
||||||
longest_col = t2.column_widths.index(max(t2.column_widths))
|
t2.table_data[i] = content
|
||||||
max_length_col = t2.column_max_width(longest_col)
|
|
||||||
if max_length_col > 0:
|
t3 = AsciiTable(printarray3, title="Not running queues")
|
||||||
for i, content in enumerate(t2.table_data):
|
t3.column_max_width(1)
|
||||||
if len(content[longest_col]) > max_length_col:
|
|
||||||
temp = ''
|
printarray4 = []
|
||||||
for l in content[longest_col].splitlines():
|
for elem in printarrayGlob:
|
||||||
if len(l) > max_length_col:
|
if elem is not None:
|
||||||
temp += '\n'.join(textwrap.wrap(l, max_length_col)) + '\n'
|
printarray4.append(elem)
|
||||||
else:
|
|
||||||
temp += l + '\n'
|
t4 = AsciiTable(printarray4, title="Last actions")
|
||||||
content[longest_col] = temp.strip()
|
t4.column_max_width(1)
|
||||||
t2.table_data[i] = content
|
|
||||||
|
legend_array = [["Color", "Meaning"], [Back.RED+Style.BRIGHT+" "*10+Style.RESET_ALL, "Time >=" +str(args.treshold)+Style.RESET_ALL], [Back.MAGENTA+Style.BRIGHT+" "*10+Style.RESET_ALL, "Time >=" +str(args.treshold)+" while idle"+Style.RESET_ALL], [Back.YELLOW+Style.BRIGHT+" "*10+Style.RESET_ALL, "Time >=" +str(args.treshold/2)+Style.RESET_ALL], [Back.GREEN+Style.BRIGHT+" "*10+Style.RESET_ALL, "Time <" +str(args.treshold)]]
|
||||||
t3 = AsciiTable(printarray3, title="Not running queues")
|
legend = AsciiTable(legend_array, title="Legend")
|
||||||
t3.column_max_width(1)
|
legend.column_max_width(1)
|
||||||
|
|
||||||
printarray4 = []
|
print legend.table
|
||||||
for elem in printarrayGlob:
|
print '\n'
|
||||||
if elem is not None:
|
print t1.table
|
||||||
printarray4.append(elem)
|
print '\n'
|
||||||
|
print t2.table
|
||||||
t4 = AsciiTable(printarray4, title="Last actions")
|
print '\n'
|
||||||
t4.column_max_width(1)
|
print t3.table
|
||||||
|
print '\n'
|
||||||
legend_array = [["Color", "Meaning"], [Back.RED+Style.BRIGHT+" "*10+Style.RESET_ALL, "Time >=" +str(threshold_stucked_module)+Style.RESET_ALL], [Back.MAGENTA+Style.BRIGHT+" "*10+Style.RESET_ALL, "Time >=" +str(threshold_stucked_module)+" while idle"+Style.RESET_ALL], [Back.YELLOW+Style.BRIGHT+" "*10+Style.RESET_ALL, "Time >=" +str(threshold_stucked_module/2)+Style.RESET_ALL], [Back.GREEN+Style.BRIGHT+" "*10+Style.RESET_ALL, "Time <" +str(threshold_stucked_module)]]
|
print t4.table
|
||||||
legend = AsciiTable(legend_array, title="Legend")
|
|
||||||
legend.column_max_width(1)
|
if (datetime.datetime.now() - lastTime).total_seconds() > args.refresh*5:
|
||||||
|
lastTime = datetime.datetime.now()
|
||||||
print legend.table
|
cleanRedis()
|
||||||
print '\n'
|
#time.sleep(args.refresh)
|
||||||
print t1.table
|
|
||||||
print '\n'
|
|
||||||
print t2.table
|
|
||||||
print '\n'
|
|
||||||
print t3.table
|
|
||||||
print '\n'
|
|
||||||
print t4.table
|
|
||||||
|
|
||||||
if (datetime.datetime.now() - lastTime).total_seconds() > args.refresh*5:
|
|
||||||
lastTime = datetime.datetime.now()
|
|
||||||
cleanRedis()
|
|
||||||
time.sleep(args.refresh)
|
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue