From 0c760d763b95ccb62ba891857a24a2722dabe138 Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Fri, 26 Aug 2016 15:27:37 +0200 Subject: [PATCH] Added support of PID in ModuleInformation and index.html --- ...warning_paste.py => BrowseWarningPaste.py} | 2 +- bin/Helper.py | 10 ++--- bin/LAUNCH.sh | 2 +- bin/ModuleInformation.py | 39 +++++++++++++------ bin/packages/modules.cfg | 2 +- var/www/Flask_server.py | 14 ++----- var/www/static/js/indexjavascript.js | 32 ++++++++++----- var/www/templates/index.html | 13 ++++++- 8 files changed, 73 insertions(+), 41 deletions(-) rename bin/{Browse_warning_paste.py => BrowseWarningPaste.py} (97%) diff --git a/bin/Browse_warning_paste.py b/bin/BrowseWarningPaste.py similarity index 97% rename from bin/Browse_warning_paste.py rename to bin/BrowseWarningPaste.py index 49444979..01704902 100755 --- a/bin/Browse_warning_paste.py +++ b/bin/BrowseWarningPaste.py @@ -24,7 +24,7 @@ if __name__ == "__main__": publisher.port = 6380 publisher.channel = "Script" - config_section = 'Browse_warning_paste' + config_section = 'BrowseWarningPaste' p = Process(config_section) diff --git a/bin/Helper.py b/bin/Helper.py index e7338ceb..05b73bf3 100755 --- a/bin/Helper.py +++ b/bin/Helper.py @@ -119,13 +119,7 @@ class Process(object): port=self.config.get('RedisPubSub', 'port'), db=self.config.get('RedisPubSub', 'db')) - self.moduleNum = 1 - for i in range(1, 50): - curr_num = self.r_temp.get("MODULE_"+self.subscriber_name + "_" + str(i)) - if curr_num is None: - self.moduleNum = i - break - + self.moduleNum = os.getpid() def populate_set_in(self): @@ -158,12 +152,14 @@ class Process(object): path = "?" value = str(timestamp) + ", " + path self.r_temp.set("MODULE_"+self.subscriber_name + "_" + str(self.moduleNum), value) + self.r_temp.sadd("MODULE_TYPE_"+self.subscriber_name, str(self.moduleNum)) return message except: path = "?" value = str(timestamp) + ", " + path self.r_temp.set("MODULE_"+self.subscriber_name + "_" + str(self.moduleNum), value) + self.r_temp.sadd("MODULE_TYPE_"+self.subscriber_name, str(self.moduleNum)) return message def populate_set_out(self, msg, channel=None): diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index b50a75d6..d7c31472 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -158,7 +158,7 @@ function launching_scripts { sleep 0.1 screen -S "Script" -X screen -t "SQLInjectionDetection" bash -c './SQLInjectionDetection.py; read x' sleep 0.1 - screen -S "Script" -X screen -t "Browse_warning_paste" bash -c './Browse_warning_paste.py; read x' + screen -S "Script" -X screen -t "BrowseWarningPaste" bash -c './BrowseWarningPaste.py; read x' sleep 0.1 screen -S "Script" -X screen -t "SentimentAnalysis" bash -c './SentimentAnalysis.py; read x' diff --git a/bin/ModuleInformation.py b/bin/ModuleInformation.py index d783418f..1538f57a 100755 --- a/bin/ModuleInformation.py +++ b/bin/ModuleInformation.py @@ -29,6 +29,7 @@ import textwrap threshold_stucked_module = 60*60*1 #1 hour log_filename = "../logs/moduleInfo.log" command_search_pid = "ps a -o pid,cmd | grep {}" +command_search_name = "ps a -o pid,cmd | grep {}" command_restart_module = "screen -S \"Script\" -X screen -t \"{}\" bash -c \"./{}.py; read x\"" @@ -45,6 +46,23 @@ def clearRedisModuleInfo(): for k in server.keys("MODULE_*"): server.delete(k) +def cleanRedis(): + for k in server.keys("MODULE_TYPE_*"): + moduleName = k[12:].split('_')[0] + for pid in server.smembers(k): + flag_pid_valid = False + proc = Popen([command_search_name.format(pid)], stdin=PIPE, stdout=PIPE, bufsize=1, shell=True) + for line in proc.stdout: + splittedLine = line.split() + if ('python2' in splittedLine or 'python' in splittedLine) and "./"+moduleName+".py" in splittedLine: + flag_pid_valid = True + + if not flag_pid_valid: + print flag_pid_valid, 'cleaning', pid, 'in', k + server.srem(k, pid) + time.sleep(5) + + def kill_module(module): print '' print '-> trying to kill module:', module @@ -76,8 +94,8 @@ if __name__ == "__main__": parser = argparse.ArgumentParser(description='Show info concerning running modules and log suspected stucked modules. May be use to automatically kill and restart stucked one.') parser.add_argument('-r', '--refresh', type=int, required=False, default=1, help='Refresh rate') - parser.add_argument('-k', '--autokill', type=int, required=True, default=1, help='Enable auto kill option (1 for TRUE, anything else for FALSE)') - parser.add_argument('-c', '--clear', type=int, required=False, default=1, help='Clear the current module information (Used to clear data from old launched modules)') + parser.add_argument('-k', '--autokill', type=int, required=False, default=0, help='Enable auto kill option (1 for TRUE, anything else for FALSE)') + parser.add_argument('-c', '--clear', type=int, required=False, default=0, help='Clear the current module information (Used to clear data from old launched modules)') args = parser.parse_args() @@ -99,6 +117,7 @@ if __name__ == "__main__": if args.clear == 1: clearRedisModuleInfo() + lastTime = datetime.datetime.now() module_file_array = set() with open('../doc/all_modules.txt', 'r') as module_file: @@ -108,20 +127,15 @@ if __name__ == "__main__": while True: all_queue = set() - curr_range = 50 printarray1 = [] printarray2 = [] printarray3 = [] for queue, card in server.hgetall("queues").iteritems(): all_queue.add(queue) key = "MODULE_" + queue + "_" - for i in range(1, 50): - curr_num = server.get("MODULE_"+ queue + "_" + str(i)) - if curr_num is None: - curr_range = i - break + keySet = "MODULE_TYPE_" + queue - for moduleNum in range(1, curr_range): + for moduleNum in server.smembers(keySet): value = server.get(key + str(moduleNum)) if value is not None: timestamp, path = value.split(", ") @@ -147,8 +161,8 @@ if __name__ == "__main__": printarray1.sort(lambda x,y: cmp(x[4], y[4]), reverse=True) printarray2.sort(lambda x,y: cmp(x[4], y[4]), reverse=True) - printarray1.insert(0,["Queue", "#", "Amount", "Paste start time", "Processing time for current paste (H:M:S)", "Paste hash"]) - printarray2.insert(0,["Queue", "#","Amount", "Paste start time", "Time since idle (H:M:S)", "Last paste hash"]) + printarray1.insert(0,["Queue", "PID", "Amount", "Paste start time", "Processing time for current paste (H:M:S)", "Paste hash"]) + printarray2.insert(0,["Queue", "PID","Amount", "Paste start time", "Time since idle (H:M:S)", "Last paste hash"]) printarray3.insert(0,["Queue", "State"]) os.system('clear') @@ -195,4 +209,7 @@ if __name__ == "__main__": print '\n' print t3.table + if (datetime.datetime.now() - lastTime).total_seconds() > args.refresh*5: + lastTime = datetime.datetime.now() + cleanRedis() time.sleep(args.refresh) diff --git a/bin/packages/modules.cfg b/bin/packages/modules.cfg index 0243038a..782536c5 100644 --- a/bin/packages/modules.cfg +++ b/bin/packages/modules.cfg @@ -63,7 +63,7 @@ publish = Redis_BrowseWarningPaste,Redis_Duplicate [ModuleStats] subscribe = Redis_ModuleStats -[Browse_warning_paste] +[BrowseWarningPaste] subscribe = Redis_BrowseWarningPaste #[send_to_queue] diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index 3c4346f1..4715aaac 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -81,19 +81,13 @@ def event_stream(): def get_queues(r): # We may want to put the llen in a pipeline to do only one query. - data = [(queue, int(card)) for queue, card in r.hgetall("queues").iteritems()] newData = [] - - curr_range = 50 - for queue, card in data: + for queue, card in r.hgetall("queues").iteritems(): key = "MODULE_" + queue + "_" - for i in range(1, 50): - curr_num = r.get("MODULE_"+ queue + "_" + str(i)) - if curr_num is None: - curr_range = i - break + keySet = "MODULE_TYPE_" + queue - for moduleNum in range(1, curr_range): + for moduleNum in r.smembers(keySet): + value = r.get(key + str(moduleNum)) if value is not None: timestamp, path = value.split(", ") diff --git a/var/www/static/js/indexjavascript.js b/var/www/static/js/indexjavascript.js index 55e14081..d145df65 100644 --- a/var/www/static/js/indexjavascript.js +++ b/var/www/static/js/indexjavascript.js @@ -207,7 +207,7 @@ function create_queue_table() { table.appendChild(tableHead); table.appendChild(tableBody); var heading = new Array(); - heading[0] = "Queue Name" + heading[0] = "Queue Name.PID" heading[1] = "Amount" var tr = document.createElement('TR'); tableHead.appendChild(tr); @@ -255,13 +255,17 @@ function load_queues() { var x = new Date(); for (i = 0; i < glob_tabvar.row1.length; i++){ - if (glob_tabvar.row1[i][0] == 'Categ' || glob_tabvar.row1[i][0] == 'Curve'){ - tmp_tab2.push(0); - curves_labels2.push(glob_tabvar.row1[i][0]); + if (glob_tabvar.row1[i][0].split(".")[0] == 'Categ' || glob_tabvar.row1[i][0].split(".")[0] == 'Curve'){ + if (curves_labels2.indexOf(glob_tabvar.row1[i][0].split(".")[0]) == -1) { + tmp_tab2.push(0); + curves_labels2.push(glob_tabvar.row1[i][0].split(".")); + } } else { - tmp_tab.push(0); - curves_labels.push(glob_tabvar.row1[i][0]); + if (curves_labels.indexOf(glob_tabvar.row1[i][0].split(".")[0]) == -1) { + tmp_tab.push(0); + curves_labels.push(glob_tabvar.row1[i][0].split(".")); + } } } tmp_tab.unshift(x); @@ -320,19 +324,29 @@ function load_queues() { update_values(); if($('#button-toggle-queues').prop('checked')){ + $("#queue-color-legend").show(); create_queue_table(); } else{ $("#queueing").html(''); + $("#queue-color-legend").hide(); } + queues_pushed = [] for (i = 0; i < (glob_tabvar.row1).length; i++){ - if (glob_tabvar.row1[i][0] == 'Categ' || glob_tabvar.row1[i][0] == 'Curve'){ - tmp_values2.push(glob_tabvar.row1[i][1]); + if (glob_tabvar.row1[i][0].split(".")[0] == 'Categ' || glob_tabvar.row1[i][0].split(".")[0] == 'Curve'){ + if (queues_pushed.indexOf(glob_tabvar.row1[i][0].split(".")[0]) == -1) { + queues_pushed.push(glob_tabvar.row1[i][0].split(".")); + tmp_values2.push(glob_tabvar.row1[i][1]); + } } else { - tmp_values.push(glob_tabvar.row1[i][1]); + if (curves_labels.indexOf(glob_tabvar.row1[i][0].split(".")[0]) == -1) { + queues_pushed.push(glob_tabvar.row1[i][0].split(".")); + tmp_values.push(glob_tabvar.row1[i][1]); + } + } } tmp_values.unshift(x); diff --git a/var/www/templates/index.html b/var/www/templates/index.html index 28c3aff9..65b9c0c8 100644 --- a/var/www/templates/index.html +++ b/var/www/templates/index.html @@ -66,7 +66,18 @@ Display queues
-
+
+ + + + + + + + +
Working queues
 Idling queues
 Stucked queues
+
+