mirror of https://github.com/CIRCL/AIL-framework
Added not running module info + swapped order of module launch
Mokaddem
parent
07d7559060
commit
8b33b4e7ba
|
|
@ -112,6 +112,8 @@ function launching_scripts {
|
||||||
|
|
||||||
echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT
|
echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT
|
||||||
|
|
||||||
|
screen -S "Script" -X screen -t "ModuleInformation" bash -c './ModuleInformation.py -k 0 -c 1; read x'
|
||||||
|
sleep 0.1
|
||||||
screen -S "Script" -X screen -t "Global" bash -c './Global.py; read x'
|
screen -S "Script" -X screen -t "Global" bash -c './Global.py; read x'
|
||||||
sleep 0.1
|
sleep 0.1
|
||||||
screen -S "Script" -X screen -t "Duplicates" bash -c './Duplicates.py; read x'
|
screen -S "Script" -X screen -t "Duplicates" bash -c './Duplicates.py; read x'
|
||||||
|
|
@ -159,8 +161,6 @@ function launching_scripts {
|
||||||
screen -S "Script" -X screen -t "Browse_warning_paste" bash -c './Browse_warning_paste.py; read x'
|
screen -S "Script" -X screen -t "Browse_warning_paste" bash -c './Browse_warning_paste.py; read x'
|
||||||
sleep 0.1
|
sleep 0.1
|
||||||
screen -S "Script" -X screen -t "SentimentAnalysis" bash -c './SentimentAnalysis.py; read x'
|
screen -S "Script" -X screen -t "SentimentAnalysis" bash -c './SentimentAnalysis.py; read x'
|
||||||
sleep 0.1
|
|
||||||
screen -S "Script" -X screen -t "ModuleInformation" bash -c './ModuleInformation.py -k 0; read x'
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -87,82 +87,100 @@ if __name__ == "__main__":
|
||||||
if args.clear == 1:
|
if args.clear == 1:
|
||||||
clearRedisModuleInfo()
|
clearRedisModuleInfo()
|
||||||
|
|
||||||
while True:
|
|
||||||
|
|
||||||
curr_range = 50
|
module_file_array = set()
|
||||||
printarray1 = []
|
with open('../doc/all_modules.txt', 'r') as module_file:
|
||||||
printarray2 = []
|
for line in module_file:
|
||||||
for queue, card in server.hgetall("queues").iteritems():
|
module_file_array.add(line[:-1])
|
||||||
key = "MODULE_" + queue + "_"
|
|
||||||
for i in range(1, 50):
|
|
||||||
curr_num = server.get("MODULE_"+ queue + "_" + str(i))
|
|
||||||
if curr_num is None:
|
|
||||||
curr_range = i
|
|
||||||
break
|
|
||||||
|
|
||||||
for moduleNum in range(1, curr_range):
|
while True:
|
||||||
value = server.get(key + str(moduleNum))
|
|
||||||
if value is not None:
|
|
||||||
timestamp, path = value.split(", ")
|
|
||||||
if timestamp is not None and path is not None:
|
|
||||||
startTime_readable = datetime.datetime.fromtimestamp(int(timestamp))
|
|
||||||
processed_time_readable = str((datetime.datetime.now() - startTime_readable)).split('.')[0]
|
|
||||||
|
|
||||||
if int(card) > 0:
|
all_queue = set()
|
||||||
if int((datetime.datetime.now() - startTime_readable).total_seconds()) > threshold_stucked_module:
|
curr_range = 50
|
||||||
log = open(log_filename, 'a')
|
printarray1 = []
|
||||||
log.write(json.dumps([queue, card, str(startTime_readable), str(processed_time_readable), path]) + "\n")
|
printarray2 = []
|
||||||
if args.autokill == 1:
|
printarray3 = []
|
||||||
kill_module(queue)
|
for queue, card in server.hgetall("queues").iteritems():
|
||||||
|
all_queue.add(queue)
|
||||||
|
key = "MODULE_" + queue + "_"
|
||||||
|
for i in range(1, 50):
|
||||||
|
curr_num = server.get("MODULE_"+ queue + "_" + str(i))
|
||||||
|
if curr_num is None:
|
||||||
|
curr_range = i
|
||||||
|
break
|
||||||
|
|
||||||
printarray1.append([str(queue), str(moduleNum), str(card), str(startTime_readable), str(processed_time_readable), str(path)])
|
for moduleNum in range(1, curr_range):
|
||||||
|
value = server.get(key + str(moduleNum))
|
||||||
|
if value is not None:
|
||||||
|
timestamp, path = value.split(", ")
|
||||||
|
if timestamp is not None and path is not None:
|
||||||
|
startTime_readable = datetime.datetime.fromtimestamp(int(timestamp))
|
||||||
|
processed_time_readable = str((datetime.datetime.now() - startTime_readable)).split('.')[0]
|
||||||
|
|
||||||
else:
|
if int(card) > 0:
|
||||||
printarray2.append([str(queue), str(moduleNum), str(card), str(startTime_readable), str(processed_time_readable), str(path)])
|
if int((datetime.datetime.now() - startTime_readable).total_seconds()) > threshold_stucked_module:
|
||||||
|
log = open(log_filename, 'a')
|
||||||
|
log.write(json.dumps([queue, card, str(startTime_readable), str(processed_time_readable), path]) + "\n")
|
||||||
|
if args.autokill == 1:
|
||||||
|
kill_module(queue)
|
||||||
|
|
||||||
printarray1.sort(lambda x,y: cmp(x[4], y[4]), reverse=True)
|
printarray1.append([str(queue), str(moduleNum), str(card), str(startTime_readable), str(processed_time_readable), str(path)])
|
||||||
printarray2.sort(lambda x,y: cmp(x[4], y[4]), reverse=True)
|
|
||||||
printarray1.insert(0,["Queue", "#", "Amount", "Paste start time", "Processing time for current paste (H:M:S)", "Paste hash"])
|
|
||||||
printarray2.insert(0,["Queue", "#","Amount", "Paste start time", "Time since idle (H:M:S)", "Last paste hash"])
|
|
||||||
|
|
||||||
os.system('clear')
|
else:
|
||||||
t1 = AsciiTable(printarray1, title="Working queues")
|
printarray2.append([str(queue), str(moduleNum), str(card), str(startTime_readable), str(processed_time_readable), str(path)])
|
||||||
t1.column_max_width(1)
|
|
||||||
if not t1.ok:
|
|
||||||
longest_col = t1.column_widths.index(max(t1.column_widths))
|
|
||||||
max_length_col = t1.column_max_width(longest_col)
|
|
||||||
if max_length_col > 0:
|
|
||||||
for i, content in enumerate(t1.table_data):
|
|
||||||
if len(content[longest_col]) > max_length_col:
|
|
||||||
temp = ''
|
|
||||||
for l in content[longest_col].splitlines():
|
|
||||||
if len(l) > max_length_col:
|
|
||||||
temp += '\n'.join(textwrap.wrap(l, max_length_col)) + '\n'
|
|
||||||
else:
|
|
||||||
temp += l + '\n'
|
|
||||||
content[longest_col] = temp.strip()
|
|
||||||
t1.table_data[i] = content
|
|
||||||
|
|
||||||
t2 = AsciiTable(printarray2, title="Idling queues")
|
for curr_queue in module_file_array:
|
||||||
t2.column_max_width(1)
|
if curr_queue not in all_queue:
|
||||||
if not t2.ok:
|
printarray3.append([curr_queue, "Not running"])
|
||||||
longest_col = t2.column_widths.index(max(t2.column_widths))
|
|
||||||
max_length_col = t2.column_max_width(longest_col)
|
|
||||||
if max_length_col > 0:
|
|
||||||
for i, content in enumerate(t2.table_data):
|
|
||||||
if len(content[longest_col]) > max_length_col:
|
|
||||||
temp = ''
|
|
||||||
for l in content[longest_col].splitlines():
|
|
||||||
if len(l) > max_length_col:
|
|
||||||
temp += '\n'.join(textwrap.wrap(l, max_length_col)) + '\n'
|
|
||||||
else:
|
|
||||||
temp += l + '\n'
|
|
||||||
content[longest_col] = temp.strip()
|
|
||||||
t2.table_data[i] = content
|
|
||||||
|
|
||||||
|
printarray1.sort(lambda x,y: cmp(x[4], y[4]), reverse=True)
|
||||||
|
printarray2.sort(lambda x,y: cmp(x[4], y[4]), reverse=True)
|
||||||
|
printarray1.insert(0,["Queue", "#", "Amount", "Paste start time", "Processing time for current paste (H:M:S)", "Paste hash"])
|
||||||
|
printarray2.insert(0,["Queue", "#","Amount", "Paste start time", "Time since idle (H:M:S)", "Last paste hash"])
|
||||||
|
printarray3.insert(0,["Queue", "State"])
|
||||||
|
|
||||||
print t1.table
|
os.system('clear')
|
||||||
print '\n'
|
t1 = AsciiTable(printarray1, title="Working queues")
|
||||||
print t2.table
|
t1.column_max_width(1)
|
||||||
|
if not t1.ok:
|
||||||
|
longest_col = t1.column_widths.index(max(t1.column_widths))
|
||||||
|
max_length_col = t1.column_max_width(longest_col)
|
||||||
|
if max_length_col > 0:
|
||||||
|
for i, content in enumerate(t1.table_data):
|
||||||
|
if len(content[longest_col]) > max_length_col:
|
||||||
|
temp = ''
|
||||||
|
for l in content[longest_col].splitlines():
|
||||||
|
if len(l) > max_length_col:
|
||||||
|
temp += '\n'.join(textwrap.wrap(l, max_length_col)) + '\n'
|
||||||
|
else:
|
||||||
|
temp += l + '\n'
|
||||||
|
content[longest_col] = temp.strip()
|
||||||
|
t1.table_data[i] = content
|
||||||
|
|
||||||
time.sleep(args.refresh)
|
t2 = AsciiTable(printarray2, title="Idling queues")
|
||||||
|
t2.column_max_width(1)
|
||||||
|
if not t2.ok:
|
||||||
|
longest_col = t2.column_widths.index(max(t2.column_widths))
|
||||||
|
max_length_col = t2.column_max_width(longest_col)
|
||||||
|
if max_length_col > 0:
|
||||||
|
for i, content in enumerate(t2.table_data):
|
||||||
|
if len(content[longest_col]) > max_length_col:
|
||||||
|
temp = ''
|
||||||
|
for l in content[longest_col].splitlines():
|
||||||
|
if len(l) > max_length_col:
|
||||||
|
temp += '\n'.join(textwrap.wrap(l, max_length_col)) + '\n'
|
||||||
|
else:
|
||||||
|
temp += l + '\n'
|
||||||
|
content[longest_col] = temp.strip()
|
||||||
|
t2.table_data[i] = content
|
||||||
|
|
||||||
|
t3 = AsciiTable(printarray3, title="Not running queues")
|
||||||
|
t3.column_max_width(1)
|
||||||
|
|
||||||
|
print t1.table
|
||||||
|
print '\n'
|
||||||
|
print t2.table
|
||||||
|
print '\n'
|
||||||
|
print t3.table
|
||||||
|
|
||||||
|
time.sleep(args.refresh)
|
||||||
|
|
|
||||||
|
|
@ -8,6 +8,8 @@ sleep 0.1
|
||||||
|
|
||||||
echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT
|
echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT
|
||||||
|
|
||||||
|
screen -S "Script" -X screen -t "ModuleInformation" bash -c './ModuleInformation.py -k 0 -c 1; read x'
|
||||||
|
sleep 0.1
|
||||||
screen -S "Script" -X screen -t "Global" bash -c './Global.py; read x'
|
screen -S "Script" -X screen -t "Global" bash -c './Global.py; read x'
|
||||||
sleep 0.1
|
sleep 0.1
|
||||||
screen -S "Script" -X screen -t "Duplicates" bash -c './Duplicates.py; read x'
|
screen -S "Script" -X screen -t "Duplicates" bash -c './Duplicates.py; read x'
|
||||||
|
|
@ -55,5 +57,3 @@ echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT
|
||||||
screen -S "Script" -X screen -t "Browse_warning_paste" bash -c './Browse_warning_paste.py; read x'
|
screen -S "Script" -X screen -t "Browse_warning_paste" bash -c './Browse_warning_paste.py; read x'
|
||||||
sleep 0.1
|
sleep 0.1
|
||||||
screen -S "Script" -X screen -t "SentimentAnalysis" bash -c './SentimentAnalysis.py; read x'
|
screen -S "Script" -X screen -t "SentimentAnalysis" bash -c './SentimentAnalysis.py; read x'
|
||||||
sleep 0.1
|
|
||||||
screen -S "Script" -X screen -t "ModuleInformation" bash -c './ModuleInformation.py -k 0; read x'
|
|
||||||
|
|
|
||||||
|
|
@ -32,6 +32,9 @@ with open('../bin/packages/modules.cfg', 'r') as f:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
output_set_graph = set()
|
output_set_graph = set()
|
||||||
|
with open('all_modules.txt', 'w') as f2:
|
||||||
|
for e in all_modules:
|
||||||
|
f2.write(e+"\n")
|
||||||
|
|
||||||
for module in modules.keys():
|
for module in modules.keys():
|
||||||
for stream_in in modules[module]['sub']:
|
for stream_in in modules[module]['sub']:
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue