CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

fix ModulesInnformationV2

pull/201/head
Terrtia 2018-04-17 09:37:58 +02:00
parent 15ef02fe53
commit d0261233aa
2 changed files with 30 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
bin/Duplicates.py
View File

@ -73,6 +73,7 @@ if __name__ == "__main__":
PST = Paste.Paste(path) PST = Paste.Paste(path)
else: else:
publisher.debug("Script Attribute is idling 10s") publisher.debug("Script Attribute is idling 10s")
print('sleeping')
time.sleep(10) time.sleep(10)
continue continue

35
bin/ModulesInformationV2.py
View File

@ -474,12 +474,15 @@ class Show_paste(Frame):
i += 1 i += 1
except OSError as e: except OSError as e:
print('exceptoserror')
self.label_list[0]._text = "Error during parsing the filepath. Please, check manually" self.label_list[0]._text = "Error during parsing the filepath. Please, check manually"
self.label_list[1]._text = COMPLETE_PASTE_PATH_PER_PID[current_selected_value] self.label_list[1]._text = COMPLETE_PASTE_PATH_PER_PID[current_selected_value]
for i in range(2,self.num_label): for i in range(2,self.num_label):
self.label_list[i]._text = "" self.label_list[i]._text = ""
except Exception as e: except Exception as e:
print('except')
print(e)
self.label_list[0]._text = "Error while displaying the paste: " + COMPLETE_PASTE_PATH_PER_PID[current_selected_value] self.label_list[0]._text = "Error while displaying the paste: " + COMPLETE_PASTE_PATH_PER_PID[current_selected_value]
self.label_list[1]._text = str(e) self.label_list[1]._text = str(e)
for i in range(2,self.num_label): for i in range(2,self.num_label):
@ -499,7 +502,7 @@ def getPid(module):
for line in p.stdout: for line in p.stdout:
print(line) print(line)
splittedLine = line.split() splittedLine = line.split()
if 'python2' in splittedLine: if 'python3' in splittedLine:
return int(splittedLine[0]) return int(splittedLine[0])
return None return None
@ -513,13 +516,22 @@ def cleanRedis():
for k in server.keys("MODULE_TYPE_*"): for k in server.keys("MODULE_TYPE_*"):
moduleName = (k[12:].decode('utf8')).split('_')[0] moduleName = (k[12:].decode('utf8')).split('_')[0]
for pid in server.smembers(k): for pid in server.smembers(k):
pid = pid.decode('utf8')
flag_pid_valid = False flag_pid_valid = False
proc = Popen([command_search_name.format(pid)], stdin=PIPE, stdout=PIPE, bufsize=1, shell=True) proc = Popen([command_search_name.format(pid)], stdin=PIPE, stdout=PIPE, bufsize=1, shell=True)
try: try:
for line in proc.stdout: for line in proc.stdout:
line = line.decode('utf8')
#print('-------------------------')
splittedLine = line.split() splittedLine = line.split()
if ('python3' in splittedLine or 'python' in splittedLine) and "./"+moduleName + ".py" in splittedLine: if ('python3.5' in splittedLine or 'python3' in splittedLine):
flag_pid_valid = True #print(splittedLine)
moduleCommand = "./"+moduleName + ".py"
moduleCommand2 = moduleName + ".py"
if(moduleCommand in splittedLine or moduleCommand2 in splittedLine):
flag_pid_valid = True
#print(flag_pid_valid)
if not flag_pid_valid: if not flag_pid_valid:
#print flag_pid_valid, 'cleaning', pid, 'in', k #print flag_pid_valid, 'cleaning', pid, 'in', k
@ -529,6 +541,7 @@ def cleanRedis():
#Error due to resize, interrupted sys call #Error due to resize, interrupted sys call
except IOError as e: except IOError as e:
print('exceptp')
inst_time = datetime.datetime.fromtimestamp(int(time.time())) inst_time = datetime.datetime.fromtimestamp(int(time.time()))
log(([str(inst_time).split(' ')[1], " - ", " - ", "Cleaning fail due to resize."], 0)) log(([str(inst_time).split(' ')[1], " - ", " - ", "Cleaning fail due to resize."], 0))
@ -559,6 +572,8 @@ def kill_module(module, pid):
p = psutil.Process(int(pid)) p = psutil.Process(int(pid))
p.terminate() p.terminate()
except Exception as e: except Exception as e:
print('except')
print(e)
#print pid, 'already killed' #print pid, 'already killed'
inst_time = datetime.datetime.fromtimestamp(int(time.time())) inst_time = datetime.datetime.fromtimestamp(int(time.time()))
log(([str(inst_time).split(' ')[1], module, pid, "Already killed"], 0)) log(([str(inst_time).split(' ')[1], module, pid, "Already killed"], 0))
@ -610,8 +625,9 @@ def fetchQueueData():
array_module_type = [] array_module_type = []
for moduleNum in server.smembers(keySet): for moduleNum in server.smembers(keySet):
value = server.get(key + str(moduleNum)) moduleNum = moduleNum.decode('utf8')
complete_paste_path = server.get(key + str(moduleNum) + "_PATH") value = ( server.get(key + str(moduleNum)) ).decode('utf8')
complete_paste_path = ( server.get(key + str(moduleNum) + "_PATH") ).decode('utf8')
COMPLETE_PASTE_PATH_PER_PID[moduleNum] = complete_paste_path COMPLETE_PASTE_PATH_PER_PID[moduleNum] = complete_paste_path
if value is not None: if value is not None:
@ -635,6 +651,7 @@ def fetchQueueData():
try: try:
last_kill_try = time.time() - lastTimeKillCommand[moduleNum] last_kill_try = time.time() - lastTimeKillCommand[moduleNum]
except KeyError: except KeyError:
print('KeyError')
last_kill_try = kill_retry_threshold+1 last_kill_try = kill_retry_threshold+1
if args.autokill == 1 and last_kill_try > kill_retry_threshold : if args.autokill == 1 and last_kill_try > kill_retry_threshold :
kill_module(queue, int(moduleNum)) kill_module(queue, int(moduleNum))
@ -646,14 +663,17 @@ def fetchQueueData():
cpu_avg = sum(CPU_TABLE[moduleNum])/len(CPU_TABLE[moduleNum]) cpu_avg = sum(CPU_TABLE[moduleNum])/len(CPU_TABLE[moduleNum])
if len(CPU_TABLE[moduleNum]) > args.refresh*10: if len(CPU_TABLE[moduleNum]) > args.refresh*10:
CPU_TABLE[moduleNum].pop() CPU_TABLE[moduleNum].pop()
mem_percent = CPU_OBJECT_TABLE[int(moduleNum)].memory_percent() mem_percent = CPU_OBJECT_TABLE[int(moduleNum)].memory_percent()
except psutil.NoSuchProcess: except psutil.NoSuchProcess:
print('no such process')
del CPU_OBJECT_TABLE[int(moduleNum)] del CPU_OBJECT_TABLE[int(moduleNum)]
del CPU_TABLE[moduleNum] del CPU_TABLE[moduleNum]
cpu_percent = 0 cpu_percent = 0
cpu_avg = cpu_percent cpu_avg = cpu_percent
mem_percent = 0 mem_percent = 0
except KeyError: except KeyError:
#print('key error2')
try: try:
CPU_OBJECT_TABLE[int(moduleNum)] = psutil.Process(int(moduleNum)) CPU_OBJECT_TABLE[int(moduleNum)] = psutil.Process(int(moduleNum))
cpu_percent = CPU_OBJECT_TABLE[int(moduleNum)].cpu_percent() cpu_percent = CPU_OBJECT_TABLE[int(moduleNum)].cpu_percent()
@ -661,6 +681,7 @@ def fetchQueueData():
cpu_avg = cpu_percent cpu_avg = cpu_percent
mem_percent = CPU_OBJECT_TABLE[int(moduleNum)].memory_percent() mem_percent = CPU_OBJECT_TABLE[int(moduleNum)].memory_percent()
except psutil.NoSuchProcess: except psutil.NoSuchProcess:
print('no such process2')
cpu_percent = 0 cpu_percent = 0
cpu_avg = cpu_percent cpu_avg = cpu_percent
mem_percent = 0 mem_percent = 0
@ -673,7 +694,7 @@ def fetchQueueData():
printarray_idle.append( ([" <K> ", str(queue), str(moduleNum), str(processed_time_readable), str(path)], moduleNum) ) printarray_idle.append( ([" <K> ", str(queue), str(moduleNum), str(processed_time_readable), str(path)], moduleNum) )
PID_NAME_DICO[int(moduleNum)] = str(queue) PID_NAME_DICO[int(moduleNum)] = str(queue)
array_module_type.sort(lambda x,y: cmp(x[0][4], y[0][4]), reverse=True) #Sort by num of pastes #array_module_type.sort(lambda x,y: cmp(x[0][4], y[0][4]), reverse=True) #Sort by num of pastes
for e in array_module_type: for e in array_module_type:
printarray_running.append(e) printarray_running.append(e)
@ -820,6 +841,7 @@ if __name__ == "__main__":
for line in module_file: for line in module_file:
module_file_array.add(line[:-1]) module_file_array.add(line[:-1])
except IOError as e: except IOError as e:
print('IOError')
if e.errno == 2: #module_file not found, creating a new one if e.errno == 2: #module_file not found, creating a new one
print(path_allmod + " not found.\nCreating a new one.") print(path_allmod + " not found.\nCreating a new one.")
os.system("./../doc/generate_modules_data_flow_graph.sh") os.system("./../doc/generate_modules_data_flow_graph.sh")
@ -837,6 +859,7 @@ if __name__ == "__main__":
try: try:
input("Press < ENTER > to launch the manager...") input("Press < ENTER > to launch the manager...")
except SyntaxError: except SyntaxError:
print('excepterror')
pass pass
last_scene = None last_scene = None