AIL-framework/bin/ModulesInformationV2.py

853 lines
34 KiB
Python
Raw Normal View History

2018-05-04 13:53:29 +02:00
#!/usr/bin/env python3
2017-01-16 12:18:23 +01:00
# -*-coding:UTF-8 -*
from asciimatics.widgets import Frame, ListBox, Layout, Divider, Text, \
Button, Label
2017-01-16 12:18:23 +01:00
from asciimatics.scene import Scene
from asciimatics.screen import Screen
from asciimatics.exceptions import ResizeScreenError, NextScene, StopApplication
from asciimatics.event import Event
from asciimatics.event import KeyboardEvent, MouseEvent
2017-01-16 12:18:23 +01:00
import sys, os
import time, datetime
import argparse
2017-01-16 12:18:23 +01:00
import json
import redis
import psutil
from subprocess import PIPE, Popen
from lib.objects.Items import Item
2017-01-16 12:18:23 +01:00
sys.path.append(os.environ['AIL_BIN'])
from lib import ConfigLoader
2017-01-16 12:18:23 +01:00
# CONFIG VARIABLES
kill_retry_threshold = 60 #1m
log_filename = "../logs/moduleInfo.log"
command_search_pid = "ps a -o pid,cmd | grep {}"
command_search_name = "ps a -o pid,cmd | grep {}"
command_restart_module = "screen -S \"Script\" -X screen -t \"{}\" bash -c \"./{}.py; read x\""
2017-02-15 10:57:41 +01:00
printarrayLog = [None]*14
2017-01-16 12:18:23 +01:00
lastTimeKillCommand = {}
2017-02-15 10:04:51 +01:00
# Used to pass information through scenes
current_selected_value = 0
current_selected_queue = ""
current_selected_action = ""
current_selected_amount = 0
2017-02-15 10:04:51 +01:00
# Map PID to Queue name (For restart and killing)
PID_NAME_DICO = {}
2017-02-15 10:04:51 +01:00
# Tables containing info for the dashboad
TABLES = {"running": [], "idle": [], "notRunning": [], "logs": [("No events recorded yet", 0)]}
TABLES_TITLES = {"running": "", "idle": "", "notRunning": "", "logs": ""}
TABLES_PADDING = {"running": [12, 23, 8, 8, 23, 10, 55, 11, 11, 12], "idle": [9, 23, 8, 12, 50], "notRunning": [9, 23, 35], "logs": [15, 23, 8, 50]}
2017-02-15 10:04:51 +01:00
# Indicator for the health of a queue (green(0), red(2), yellow(1))
QUEUE_STATUS = {}
2017-02-15 10:04:51 +01:00
# Maintain the state of the CPU objects
2018-04-16 14:50:04 +02:00
CPU_TABLE = {}
CPU_OBJECT_TABLE = {}
2017-01-16 12:18:23 +01:00
# Path of the current item for a pid
COMPLETE_PASTE_PATH_PER_PID = {}
'''
ASCIIMATICS WIDGETS EXTENSION
'''
2017-02-15 10:04:51 +01:00
# Custom listbox
2017-01-16 12:18:23 +01:00
class CListBox(ListBox):
2017-01-16 12:18:23 +01:00
def __init__(self, queue_name, *args, **kwargs):
self.queue_name = queue_name
super(CListBox, self).__init__(*args, **kwargs)
def update(self, frame_no):
self._options = TABLES[self.queue_name]
self._draw_label()
# Calculate new visible limits if needed.
width = self._w - self._offset
height = self._h
dx = dy = 0
# Clear out the existing box content
(colour, attr, bg) = self._frame.palette["field"]
for i in range(height):
self._frame.canvas.print_at(
" " * width,
self._x + self._offset + dx,
self._y + i + dy,
colour, attr, bg)
# Don't bother with anything else if there are no options to render.
if len(self._options) <= 0:
return
# Render visible portion of the text.
self._start_line = max(0, max(self._line - height + 1,
min(self._start_line, self._line)))
for i, (text, pid) in enumerate(self._options):
if self._start_line <= i < self._start_line + height:
colour, attr, bg = self._pick_colours("field", i == self._line)
self._frame.canvas.print_at(
"{:{width}}".format(text, width=width),
self._x + self._offset + dx,
self._y + i + dy - self._start_line,
colour, attr, bg)
2017-02-15 10:04:51 +01:00
# Pick color depending on queue health
if self.queue_name == "running":
if QUEUE_STATUS[pid] == 2:
queueStatus = Screen.COLOUR_RED
elif QUEUE_STATUS[pid] == 1:
queueStatus = Screen.COLOUR_YELLOW
else:
queueStatus = Screen.COLOUR_GREEN
self._frame.canvas.print_at(" ",
self._x + 9 + dx,
self._y + i + dy - self._start_line,
colour, attr, queueStatus)
2017-01-16 12:18:23 +01:00
def process_event(self, event):
if isinstance(event, KeyboardEvent):
if len(self._options) > 0 and event.key_code == Screen.KEY_UP:
# Move up one line in text - use value to trigger on_select.
self._line = max(0, self._line - 1)
self._line = min(self._line, len(self._options)-1) #If we move a line cursor from a line that has dissapear
self.value = self._options[self._line][1]
2017-02-15 10:04:51 +01:00
elif len(self._options) > 0 and event.key_code == Screen.KEY_DOWN:
# Move down one line in text - use value to trigger on_select.
self._line = min(len(self._options) - 1, self._line + 1)
self.value = self._options[self._line][1]
2017-02-15 10:04:51 +01:00
2017-02-15 11:01:11 +01:00
elif len(self._options) > 0 and event.key_code in [ord(' '), ord('\n')] :
global current_selected_value, current_selected_queue
if self.queue_name == "logs":
return event
current_selected_value = self.value
current_selected_queue = self.queue_name
self._frame.save()
raise NextScene("action_choice")
# Quit if press q
elif event.key_code == ord('q'):
Dashboard._quit()
2018-04-16 14:50:04 +02:00
else:
# Ignore any other key press.
return event
2017-02-15 10:04:51 +01:00
elif isinstance(event, MouseEvent):
# Mouse event - rebase coordinates to Frame context.
new_event = self._frame.rebase_event(event)
if event.buttons != 0:
if (len(self._options) > 0 and
self.is_mouse_over(new_event, include_label=False)):
# Use property to trigger events.
self._line = min(new_event.y - self._y,
len(self._options) - 1)
self.value = self._options[self._line][1]
# If clicked on button <k>, kill the queue
if self._x+2 <= new_event.x < self._x+4:
if self.queue_name in ["running", "idle"]:
kill_module(PID_NAME_DICO[int(self.value)], self.value)
else:
restart_module(self.value)
return
# Ignore other mouse events.
return event
else:
# Ignore other events
return event
2017-02-15 10:04:51 +01:00
# Custom label centered in the middle
2017-01-16 12:18:23 +01:00
class CLabel(Label):
def __init__(self, label, listTitle=False):
2017-01-16 12:18:23 +01:00
super(Label, self).__init__(None, tab_stop=False)
# Although this is a label, we don't want it to contribute to the layout
# tab calculations, so leave internal `_label` value as None.
self._text = label
self.listTitle = listTitle
self._required_height = 1
2017-01-16 12:18:23 +01:00
def set_layout(self, x, y, offset, w, h):
# Do the usual layout work. then recalculate exact x/w values for the
# rendered button.
super(Label, self).set_layout(x, y, offset, w, h)
self._x += max(0, (self._w - self._offset - len(self._text)) // 2) if not self.listTitle else 0
2017-01-16 12:18:23 +01:00
self._w = min(self._w, len(self._text))
def update(self, frame_no):
(colour, attr, bg) = self._frame.palette["title"]
colour = Screen.COLOUR_YELLOW if not self.listTitle else colour
2017-01-16 12:18:23 +01:00
self._frame.canvas.print_at(
self._text, self._x, self._y, colour, attr, bg)
'''
END EXTENSION
'''
'''
SCENE DEFINITION
2018-04-16 14:50:04 +02:00
'''
class Dashboard(Frame):
2017-01-16 12:18:23 +01:00
def __init__(self, screen):
super(Dashboard, self).__init__(screen,
2017-01-16 12:18:23 +01:00
screen.height,
screen.width,
hover_focus=True,
reduce_cpu=True)
2017-01-16 12:18:23 +01:00
self._list_view_run_queue = CListBox(
"running",
screen.height // 2,
[], name="LIST")
2017-01-16 12:18:23 +01:00
self._list_view_idle_queue = CListBox(
"idle",
screen.height // 2,
[], name="LIST")
2017-01-16 12:18:23 +01:00
self._list_view_noRunning = CListBox(
"notRunning",
screen.height // 5,
[], name="LIST")
2017-01-16 12:18:23 +01:00
self._list_view_Log = CListBox(
"logs",
screen.height // 4,
[], name="LIST")
#self._list_view_Log.disabled = True
2017-01-16 12:18:23 +01:00
#Running Queues
layout = Layout([100])
self.add_layout(layout)
text_rq = CLabel("Running Queues")
layout.add_widget(text_rq)
layout.add_widget(CLabel(TABLES_TITLES["running"], listTitle=True))
2017-01-16 12:18:23 +01:00
layout.add_widget(self._list_view_run_queue)
layout.add_widget(Divider())
#Idling Queues
layout2 = Layout([1,1])
self.add_layout(layout2)
text_iq = CLabel("Idling Queues")
layout2.add_widget(text_iq, 0)
layout2.add_widget(CLabel(TABLES_TITLES["idle"], listTitle=True), 0)
2017-01-16 12:18:23 +01:00
layout2.add_widget(self._list_view_idle_queue, 0)
#Non Running Queues
text_nq = CLabel("Queues not running")
2017-01-16 12:18:23 +01:00
layout2.add_widget(text_nq, 1)
layout2.add_widget(CLabel(TABLES_TITLES["notRunning"], listTitle=True), 1)
2017-01-16 12:18:23 +01:00
layout2.add_widget(self._list_view_noRunning, 1)
layout2.add_widget(Divider(), 1)
#Log
text_l = CLabel("Logs")
layout2.add_widget(text_l, 1)
layout2.add_widget(CLabel(TABLES_TITLES["logs"], listTitle=True), 1)
2017-01-16 12:18:23 +01:00
layout2.add_widget(self._list_view_Log, 1)
self.fix()
@staticmethod
def _quit():
raise StopApplication("User pressed quit")
class Confirm(Frame):
def __init__(self, screen):
super(Confirm, self).__init__(screen,
screen.height * 1 // 8,
screen.width * 1 // 3,
hover_focus=True,
on_load=self._setValue,
title="Confirm action",
reduce_cpu=True)
# Create the form for displaying the list of contacts.
layout = Layout([100], fill_frame=True)
self.add_layout(layout)
self.label = CLabel("{} module {} {}?")
layout.add_widget(Label(" "))
layout.add_widget(self.label)
layout2 = Layout([1,1])
self.add_layout(layout2)
layout2.add_widget(Button("Ok", self._ok), 0)
layout2.add_widget(Button("Cancel", self._cancel), 1)
self.fix()
def _ok(self):
global current_selected_value, current_selected_queue, current_selected_action, current_selected_amount
if current_selected_action == "KILL":
kill_module(PID_NAME_DICO[int(current_selected_value)], current_selected_value)
else:
2017-02-15 10:04:51 +01:00
count = int(current_selected_amount) #Number of queue to start
if current_selected_queue in ["running", "idle"]:
restart_module(PID_NAME_DICO[int(current_selected_value)], count)
else:
restart_module(current_selected_value, count)
current_selected_value = 0
current_selected_amount = 0
current_selected_action = ""
self.label._text = "{} module {} {}?"
self.save()
raise NextScene("dashboard")
def _cancel(self):
global current_selected_value
current_selected_value = 0
current_selected_amount = 0
current_selected_action = ""
self.label._text = "{} module {} {}?"
self.save()
raise NextScene("dashboard")
def _setValue(self):
global current_selected_value, current_selected_queue, current_selected_action, current_selected_amount
if current_selected_queue in ["running", "idle"]:
action = current_selected_action if current_selected_action == "KILL" else current_selected_action +" "+ str(current_selected_amount) + "x"
modulename = PID_NAME_DICO[int(current_selected_value)]
pid = current_selected_value
else:
action = current_selected_action + " " + str(current_selected_amount) + "x"
modulename = current_selected_value
pid = ""
self.label._text = self.label._text.format(action, modulename, pid)
class Action_choice(Frame):
def __init__(self, screen):
super(Action_choice, self).__init__(screen,
screen.height * 1 // 8,
screen.width * 1 // 2,
hover_focus=True,
on_load=self._setValue,
title="Confirm action",
reduce_cpu=True)
# Create the form for displaying the list of contacts.
layout = Layout([100], fill_frame=True)
self.add_layout(layout)
self.label = CLabel("Choose action on module {} {}")
layout.add_widget(self.label)
layout2 = Layout([1,1,1,1])
self.add_layout(layout2)
layout2.add_widget(Button("Cancel", self._cancel), 0)
self._ShowPasteBtn = Button("Show current paste", self._showpaste)
layout2.add_widget(self._ShowPasteBtn, 1)
self._killBtn = Button("KILL", self._kill)
layout2.add_widget(self._killBtn, 2)
layout2.add_widget(Button("START", self._start), 3)
layout3 = Layout([1,1,1,1])
self.add_layout(layout3)
self.textEdit = Text("Amount", "amount")
layout3.add_widget(self.textEdit, 3)
self.fix()
def _kill(self):
global current_selected_action
current_selected_action = "KILL"
self.label._text = "Choose action on module {} {}"
self.save()
raise NextScene("confirm")
def _start(self):
global current_selected_action, current_selected_amount
current_selected_action = "START"
try:
count = int(self.textEdit.value)
count = count if count < 20 else 1
except Exception:
count = 1
current_selected_amount = count
self.label._text = "Choose action on module {} {}"
self.save()
raise NextScene("confirm")
def _cancel(self):
global current_selected_value
current_selected_value = 0
self.label._text = "Choose action on module {} {}"
self.save()
raise NextScene("dashboard")
def _showpaste(self):
self.label._text = "Choose action on module {} {}"
self.save()
raise NextScene("show_paste")
def _setValue(self):
self._killBtn.disabled = False
self._ShowPasteBtn.disabled = False
global current_selected_value, current_selected_queue
if current_selected_queue in ["running", "idle"]:
modulename = PID_NAME_DICO[int(current_selected_value)]
pid = current_selected_value
else:
self._killBtn.disabled = True
self._ShowPasteBtn.disabled = True
modulename = current_selected_value
pid = ""
self.label._text = self.label._text.format(modulename, pid)
2017-01-16 12:18:23 +01:00
class Show_paste(Frame):
def __init__(self, screen):
super(Show_paste, self).__init__(screen,
screen.height,
screen.width,
hover_focus=True,
on_load=self._setValue,
title="Show current paste",
reduce_cpu=True)
layout = Layout([100], fill_frame=True)
self.layout = layout
self.add_layout(layout)
self.label_list = []
2017-02-15 10:04:51 +01:00
self.num_label = 42 # Number of line available for displaying the paste
for i in range(self.num_label):
self.label_list += [Label("THE PASTE CONTENT " + str(i))]
layout.add_widget(self.label_list[i])
layout2 = Layout([100])
self.add_layout(layout2)
layout2.add_widget(Button("Ok", self._ok), 0)
self.fix()
def _ok(self):
global current_selected_value, current_selected_queue, current_selected_action, current_selected_amount
current_selected_value = 0
current_selected_amount = 0
current_selected_action = ""
self.save()
raise NextScene("dashboard")
def _setValue(self):
try:
#Verify that the module have a paste
if COMPLETE_PASTE_PATH_PER_PID[current_selected_value] is None:
self.label_list[0]._text = "No paste for this module"
for i in range(1,self.num_label):
self.label_list[i]._text = ""
return
item = Item(COMPLETE_PASTE_PATH_PER_PID[current_selected_value])
old_content = item.get_content()[0:4000] # Limit number of char to be displayed
# Replace unprintable char by ?
content = ""
for i, c in enumerate(old_content):
2017-02-15 10:04:51 +01:00
if ord(c) > 127: # Used to avoid printing unprintable char
content += '?'
2017-02-15 10:04:51 +01:00
elif c == "\t": # Replace tab by 4 spaces
content += " "
else:
content += c
# Print in the correct label, END or more
to_print = ""
i = 0
for line in content.split("\n"):
if i > self.num_label - 2:
break
self.label_list[i]._text = str(i) + ". " + line.replace("\r","")
i += 1
if i > self.num_label - 2:
2017-02-15 10:04:51 +01:00
self.label_list[i]._text = "- ALL PASTE NOT DISPLAYED -"
i += 1
else:
2017-02-15 10:04:51 +01:00
self.label_list[i]._text = "- END of PASTE -"
i += 1
while i<self.num_label: # Clear out remaining lines
self.label_list[i]._text = ""
i += 1
except OSError as e:
self.label_list[0]._text = "Error during parsing the filepath. Please, check manually"
self.label_list[1]._text = COMPLETE_PASTE_PATH_PER_PID[current_selected_value]
for i in range(2,self.num_label):
self.label_list[i]._text = ""
except Exception as e:
if current_selected_value in COMPLETE_PASTE_PATH_PER_PID:
self.label_list[0]._text = "Error while displaying the paste: " + COMPLETE_PASTE_PATH_PER_PID[current_selected_value]
else:
self.label_list[0]._text = "Error Generic exception caught"
self.label_list[1]._text = str(e)
for i in range(2,self.num_label):
self.label_list[i]._text = ""
'''
END SCENES DEFINITION
'''
2017-01-16 12:18:23 +01:00
'''
MANAGE MODULES AND GET INFOS
'''
2017-01-16 12:18:23 +01:00
def getPid(module):
p = Popen([command_search_pid.format(module+".py")], stdin=PIPE, stdout=PIPE, bufsize=1, shell=True)
for line in p.stdout:
splittedLine = line.split()
2018-04-17 09:37:58 +02:00
if 'python3' in splittedLine:
2017-01-16 12:18:23 +01:00
return int(splittedLine[0])
return None
def clearRedisModuleInfo():
for k in server.keys("MODULE_*"):
server.delete(k)
inst_time = datetime.datetime.fromtimestamp(int(time.time()))
2017-02-15 10:57:41 +01:00
log(([str(inst_time).split(' ')[1], "*", "-", "Cleared redis module info"], 0))
2017-01-16 12:18:23 +01:00
def cleanRedis():
for k in server.keys("MODULE_TYPE_*"):
2018-05-04 13:53:29 +02:00
moduleName = k[12:].split('_')[0]
2017-01-16 12:18:23 +01:00
for pid in server.smembers(k):
flag_pid_valid = False
proc = Popen([command_search_name.format(pid)], stdin=PIPE, stdout=PIPE, bufsize=1, shell=True)
try:
for line in proc.stdout:
2018-04-17 09:37:58 +02:00
line = line.decode('utf8')
splittedLine = line.split()
2018-04-17 09:43:45 +02:00
if ('python3.5' in splittedLine or 'python3' in splittedLine or 'python' in splittedLine):
2018-04-17 09:37:58 +02:00
moduleCommand = "./"+moduleName + ".py"
moduleCommand2 = moduleName + ".py"
if(moduleCommand in splittedLine or moduleCommand2 in splittedLine):
flag_pid_valid = True
if not flag_pid_valid:
#print flag_pid_valid, 'cleaning', pid, 'in', k
server.srem(k, pid)
inst_time = datetime.datetime.fromtimestamp(int(time.time()))
2018-05-04 13:53:29 +02:00
log(([str(inst_time).split(' ')[1], moduleName, pid, "Cleared invalid pid in " + (k)], 0))
#Error due to resize, interrupted sys call
except IOError as e:
2017-01-16 12:18:23 +01:00
inst_time = datetime.datetime.fromtimestamp(int(time.time()))
2017-02-15 10:57:41 +01:00
log(([str(inst_time).split(' ')[1], " - ", " - ", "Cleaning fail due to resize."], 0))
2017-01-16 12:18:23 +01:00
def restart_module(module, count=1):
for i in range(count):
p2 = Popen([command_restart_module.format(module, module)], stdin=PIPE, stdout=PIPE, bufsize=1, shell=True)
time.sleep(0.2)
inst_time = datetime.datetime.fromtimestamp(int(time.time()))
2017-02-15 10:57:41 +01:00
log(([str(inst_time).split(' ')[1], module, "?", "Restarted " + str(count) + "x"], 0))
2017-01-16 12:18:23 +01:00
def kill_module(module, pid):
#print '-> trying to kill module:', module
2017-01-16 12:18:23 +01:00
if pid is None:
#print 'pid was None'
inst_time = datetime.datetime.fromtimestamp(int(time.time()))
2017-02-15 10:57:41 +01:00
log(([str(inst_time).split(' ')[1], module, pid, "PID was None"], 0))
2017-01-16 12:18:23 +01:00
pid = getPid(module)
else: #Verify that the pid is at least in redis
if server.exists("MODULE_"+module+"_"+str(pid)) == 0:
return
lastTimeKillCommand[pid] = int(time.time())
if pid is not None:
try:
p = psutil.Process(int(pid))
p.terminate()
except Exception as e:
#print pid, 'already killed'
2017-01-16 12:18:23 +01:00
inst_time = datetime.datetime.fromtimestamp(int(time.time()))
2017-02-15 10:57:41 +01:00
log(([str(inst_time).split(' ')[1], module, pid, "Already killed"], 0))
2017-01-16 12:18:23 +01:00
return
time.sleep(0.2)
if not p.is_running():
#print module, 'has been killed'
#print 'restarting', module, '...'
2017-01-16 12:18:23 +01:00
inst_time = datetime.datetime.fromtimestamp(int(time.time()))
2017-02-15 10:57:41 +01:00
log(([str(inst_time).split(' ')[1], module, pid, "Killed"], 0))
#restart_module(module)
2017-01-16 12:18:23 +01:00
else:
#print 'killing failed, retrying...'
2017-01-16 12:18:23 +01:00
inst_time = datetime.datetime.fromtimestamp(int(time.time()))
2017-02-15 10:57:41 +01:00
log(([str(inst_time).split(' ')[1], module, pid, "Killing #1 failed."], 0))
2017-01-16 12:18:23 +01:00
p.terminate()
if not p.is_running():
#print module, 'has been killed'
#print 'restarting', module, '...'
2017-01-16 12:18:23 +01:00
inst_time = datetime.datetime.fromtimestamp(int(time.time()))
2017-02-15 10:57:41 +01:00
log(([str(inst_time).split(' ')[1], module, pid, "Killed"], 0))
#restart_module(module)
2017-01-16 12:18:23 +01:00
else:
#print 'killing failed!'
2017-01-16 12:18:23 +01:00
inst_time = datetime.datetime.fromtimestamp(int(time.time()))
2017-02-15 10:57:41 +01:00
log(([str(inst_time).split(' ')[1], module, pid, "Killing failed!"], 0))
2017-01-16 12:18:23 +01:00
else:
#print 'Module does not exist'
2017-01-16 12:18:23 +01:00
inst_time = datetime.datetime.fromtimestamp(int(time.time()))
2017-02-15 10:57:41 +01:00
log(([str(inst_time).split(' ')[1], module, pid, "Killing failed, module not found"], 0))
2017-01-16 12:18:23 +01:00
cleanRedis()
2017-02-15 10:19:03 +01:00
2017-02-15 10:37:59 +01:00
# Fetch the data for all queue
2017-01-16 12:18:23 +01:00
def fetchQueueData():
all_queue = set()
2017-02-15 10:37:59 +01:00
printarray_running = []
printarray_idle = []
printarray_notrunning = []
2018-04-16 14:50:04 +02:00
for queue, card in iter(server.hgetall("queues").items()):
2017-01-16 12:18:23 +01:00
all_queue.add(queue)
key = "MODULE_" + queue + "_"
keySet = "MODULE_TYPE_" + queue
array_module_type = []
2018-04-16 14:50:04 +02:00
2017-01-16 12:18:23 +01:00
for moduleNum in server.smembers(keySet):
2018-05-04 13:53:29 +02:00
value = server.get(key + str(moduleNum))
complete_paste_path = ( server.get(key + str(moduleNum) + "_PATH") )
if(complete_paste_path is not None):
2018-05-04 13:53:29 +02:00
complete_paste_path = complete_paste_path
COMPLETE_PASTE_PATH_PER_PID[moduleNum] = complete_paste_path
2017-02-15 10:37:59 +01:00
2017-01-16 12:18:23 +01:00
if value is not None:
timestamp, path = value.split(", ")
if timestamp is not None and path is not None:
2017-02-15 10:37:59 +01:00
# Queue health
2017-01-16 12:18:23 +01:00
startTime_readable = datetime.datetime.fromtimestamp(int(timestamp))
processed_time_readable = str((datetime.datetime.now() - startTime_readable)).split('.')[0]
if ((datetime.datetime.now() - startTime_readable).total_seconds()) > args.treshold:
QUEUE_STATUS[moduleNum] = 2
elif ((datetime.datetime.now() - startTime_readable).total_seconds()) > args.treshold/2:
QUEUE_STATUS[moduleNum] = 1
else:
QUEUE_STATUS[moduleNum] = 0
2018-04-16 14:50:04 +02:00
2017-02-15 10:37:59 +01:00
# Queue contain elements
2017-01-16 12:18:23 +01:00
if int(card) > 0:
2017-02-15 10:37:59 +01:00
# Queue need to be killed
2017-01-16 12:18:23 +01:00
if int((datetime.datetime.now() - startTime_readable).total_seconds()) > args.treshold:
log(([str(time.time()), queue, "-", "ST:"+str(timestamp)+" PT:"+str(time.time()-float(timestamp))], 0), True, show_in_board=False)
2017-01-16 12:18:23 +01:00
try:
last_kill_try = time.time() - lastTimeKillCommand[moduleNum]
except KeyError:
last_kill_try = kill_retry_threshold+1
if args.autokill == 1 and last_kill_try > kill_retry_threshold :
kill_module(queue, int(moduleNum))
2018-04-16 14:50:04 +02:00
2017-02-15 10:37:59 +01:00
# Create CPU objects
try:
cpu_percent = CPU_OBJECT_TABLE[int(moduleNum)].cpu_percent()
CPU_TABLE[moduleNum].insert(1, cpu_percent)
cpu_avg = sum(CPU_TABLE[moduleNum])/len(CPU_TABLE[moduleNum])
if len(CPU_TABLE[moduleNum]) > args.refresh*10:
CPU_TABLE[moduleNum].pop()
2018-04-17 09:37:58 +02:00
mem_percent = CPU_OBJECT_TABLE[int(moduleNum)].memory_percent()
except psutil.NoSuchProcess:
del CPU_OBJECT_TABLE[int(moduleNum)]
del CPU_TABLE[moduleNum]
cpu_percent = 0
cpu_avg = cpu_percent
mem_percent = 0
except KeyError:
2018-04-17 09:37:58 +02:00
#print('key error2')
try:
CPU_OBJECT_TABLE[int(moduleNum)] = psutil.Process(int(moduleNum))
cpu_percent = CPU_OBJECT_TABLE[int(moduleNum)].cpu_percent()
CPU_TABLE[moduleNum] = []
cpu_avg = cpu_percent
mem_percent = CPU_OBJECT_TABLE[int(moduleNum)].memory_percent()
except psutil.NoSuchProcess:
cpu_percent = 0
cpu_avg = cpu_percent
mem_percent = 0
2017-02-15 10:37:59 +01:00
array_module_type.append( ([" <K> [ ]", str(queue), str(moduleNum), str(card), str(startTime_readable),
2018-04-16 14:50:04 +02:00
str(processed_time_readable), str(path), "{0:.2f}".format(cpu_percent)+"%",
2017-02-15 10:37:59 +01:00
"{0:.2f}".format(mem_percent)+"%", "{0:.2f}".format(cpu_avg)+"%"], moduleNum) )
2018-04-16 14:50:04 +02:00
2017-01-16 12:18:23 +01:00
else:
2017-02-15 10:37:59 +01:00
printarray_idle.append( ([" <K> ", str(queue), str(moduleNum), str(processed_time_readable), str(path)], moduleNum) )
PID_NAME_DICO[int(moduleNum)] = str(queue)
2018-04-17 09:37:58 +02:00
#array_module_type.sort(lambda x,y: cmp(x[0][4], y[0][4]), reverse=True) #Sort by num of pastes
2017-01-16 12:18:23 +01:00
for e in array_module_type:
2017-02-15 10:37:59 +01:00
printarray_running.append(e)
2018-04-16 14:50:04 +02:00
2017-01-16 12:18:23 +01:00
for curr_queue in module_file_array:
2017-02-15 10:37:59 +01:00
if curr_queue not in all_queue: #Module not running by default
printarray_notrunning.append( ([" <S> ", curr_queue, "Not running by default"], curr_queue) )
else: #Module did not process anything yet
2017-01-16 12:18:23 +01:00
if len(list(server.smembers('MODULE_TYPE_'+curr_queue))) == 0:
if curr_queue not in no_info_modules:
no_info_modules[curr_queue] = int(time.time())
2017-02-15 10:37:59 +01:00
printarray_notrunning.append( ([" <S> ", curr_queue, "No data"], curr_queue) )
2017-01-16 12:18:23 +01:00
else:
#If no info since long time, try to kill
if args.autokill == 1:
if int(time.time()) - no_info_modules[curr_queue] > args.treshold:
kill_module(curr_queue, None)
no_info_modules[curr_queue] = int(time.time())
2017-02-15 10:37:59 +01:00
printarray_notrunning.append( ([" <S> ", curr_queue, "Stuck or idle, restarting in " + str(abs(args.treshold - (int(time.time()) - no_info_modules[curr_queue]))) + "s"], curr_queue) )
2017-01-16 12:18:23 +01:00
else:
2017-02-15 10:37:59 +01:00
printarray_notrunning.append( ([" <S> ", curr_queue, "Stuck or idle, restarting disabled"], curr_queue) )
2018-04-16 14:50:04 +02:00
2017-02-15 10:37:59 +01:00
printarray_running.sort(key=lambda x: x[0], reverse=False)
printarray_idle.sort(key=lambda x: x[0], reverse=False)
printarray_notrunning.sort(key=lambda x: x[0][1], reverse=False)
2017-01-16 12:18:23 +01:00
2017-02-15 10:37:59 +01:00
printstring_running = format_string(printarray_running, TABLES_PADDING["running"])
printstring_idle = format_string(printarray_idle, TABLES_PADDING["idle"])
printstring_notrunning = format_string(printarray_notrunning, TABLES_PADDING["notRunning"])
2017-01-16 12:18:23 +01:00
2017-02-15 10:37:59 +01:00
return {"running": printstring_running, "idle": printstring_idle, "notRunning": printstring_notrunning}
2017-01-16 12:18:23 +01:00
2017-02-15 10:19:03 +01:00
# Format the input string with its related padding to have collumn like text in CListBox
def format_string(tab, padding_row):
printstring = []
for row in tab:
if row is None:
continue
the_array = row[0]
the_pid = row[1]
text=""
for ite, elem in enumerate(the_array):
2018-04-16 14:50:04 +02:00
2018-09-19 15:45:06 +02:00
if elem is not None and type(elem) is str:
if len(elem) > padding_row[ite]:
text += "*" + elem[-padding_row[ite]+6:]
padd_off = " "*5
else:
text += elem
padd_off = " "*0
text += (padding_row[ite] - len(elem))*" " + padd_off
printstring.append( (text, the_pid) )
return printstring
def log(data, write_on_disk=False, show_in_board=True):
if show_in_board:
printarrayLog.insert(0, data)
printarrayLog.pop()
2017-02-15 10:57:41 +01:00
if write_on_disk:
with open(log_filename, 'a') as log:
log.write(json.dumps(data[0]) + "\n")
'''
END MANAGE
'''
def demo(screen):
dashboard = Dashboard(screen)
confirm = Confirm(screen)
action_choice = Action_choice(screen)
show_paste = Show_paste(screen)
scenes = [
Scene([dashboard], -1, name="dashboard"),
Scene([action_choice], -1, name="action_choice"),
Scene([confirm], -1, name="confirm"),
Scene([show_paste], -1, name="show_paste"),
]
2017-02-15 10:19:03 +01:00
screen.set_scenes(scenes)
2017-02-15 10:19:03 +01:00
time_cooldown = time.time() # Cooldown before refresh
global TABLES
while True:
#Stop on resize
if screen.has_resized():
screen._scenes[screen._scene_index].exit()
raise ResizeScreenError("Screen resized", screen._scenes[screen._scene_index])
if time.time() - time_cooldown > args.refresh:
cleanRedis()
2018-04-16 14:50:04 +02:00
for key, val in iter(fetchQueueData().items()): #fetch data and put it into the tables
TABLES[key] = val
2017-02-15 10:57:41 +01:00
TABLES["logs"] = format_string(printarrayLog, TABLES_PADDING["logs"])
2017-02-15 10:19:03 +01:00
#refresh dashboad only if the scene is active (no value selected)
if current_selected_value == 0:
dashboard._update(None)
screen.refresh()
time_cooldown = time.time()
screen.draw_next_frame()
2017-02-15 10:19:03 +01:00
time.sleep(0.02) #time between screen refresh (For UI navigation, not data actualisation)
2017-01-16 12:18:23 +01:00
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='Show info concerning running modules and log suspected stucked modules. May be use to automatically kill and restart stucked one.')
parser.add_argument('-r', '--refresh', type=int, required=False, default=5, help='Refresh rate')
2017-01-16 12:18:23 +01:00
parser.add_argument('-t', '--treshold', type=int, required=False, default=60*10*1, help='Refresh rate')
parser.add_argument('-k', '--autokill', type=int, required=False, default=0, help='Enable auto kill option (1 for TRUE, anything else for FALSE)')
parser.add_argument('-c', '--clear', type=int, required=False, default=0, help='Clear the current module information (Used to clear data from old launched modules)')
args = parser.parse_args()
config_loader = ConfigLoader.ConfigLoader()
2017-01-16 12:18:23 +01:00
# REDIS #
server = config_loader.get_redis_conn("Redis_Queues")
config_loader = None
2017-01-16 12:18:23 +01:00
if args.clear == 1:
clearRedisModuleInfo()
lastTime = datetime.datetime.now()
module_file_array = set()
no_info_modules = {}
path_allmod = os.path.join(os.environ['AIL_HOME'], 'doc/all_modules.txt')
try:
with open(path_allmod, 'r') as module_file:
for line in module_file:
module_file_array.add(line[:-1])
except IOError as e:
if e.errno == 2: #module_file not found, creating a new one
print(path_allmod + " not found.\nCreating a new one.")
os.system("./../doc/generate_modules_data_flow_graph.sh")
with open(path_allmod, 'r') as module_file:
for line in module_file:
module_file_array.add(line[:-1])
cleanRedis()
2017-01-16 12:18:23 +01:00
2018-04-16 14:50:04 +02:00
TABLES_TITLES["running"] = format_string([([" Action", "Queue name", "PID", "#", "S Time", "R Time", "Processed element", "CPU %", "Mem %", "Avg CPU%"],0)], TABLES_PADDING["running"])[0][0]
TABLES_TITLES["idle"] = format_string([([" Action", "Queue", "PID", "Idle Time", "Last paste hash"],0)], TABLES_PADDING["idle"])[0][0]
TABLES_TITLES["notRunning"] = format_string([([" Action", "Queue", "State"],0)], TABLES_PADDING["notRunning"])[0][0]
TABLES_TITLES["logs"] = format_string([(["Time", "Module", "PID", "Info"],0)], TABLES_PADDING["logs"])[0][0]
2017-01-16 12:18:23 +01:00
try:
input("Press < ENTER > to launch the manager...")
except SyntaxError:
pass
2017-01-16 12:18:23 +01:00
last_scene = None
2017-01-16 12:18:23 +01:00
while True:
try:
Screen.wrapper(demo)
sys.exit(0)
except ResizeScreenError as e:
pass
except StopApplication:
sys.exit(0)