CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

Added support of PID in ModuleInformation and index.html

pull/75/head
Mokaddem 2016-08-26 15:27:37 +02:00
parent 5c565068a6
commit 0c760d763b
8 changed files with 73 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
bin/Browse_warning_paste.py → bin/BrowseWarningPaste.py
View File

@ -24,7 +24,7 @@ if __name__ == "__main__":
publisher.port = 6380
publisher.channel = "Script"
config_section = 'Browse_warning_paste'
config_section = 'BrowseWarningPaste'
p = Process(config_section)

10
bin/Helper.py
View File

@ -119,13 +119,7 @@ class Process(object):
port=self.config.get('RedisPubSub', 'port'),
db=self.config.get('RedisPubSub', 'db'))
self.moduleNum = 1
for i in range(1, 50):
curr_num = self.r_temp.get("MODULE_"+self.subscriber_name + "_" + str(i))
if curr_num is None:
self.moduleNum = i
break
self.moduleNum = os.getpid()
def populate_set_in(self):
@ -158,12 +152,14 @@ class Process(object):
path = "?"
value = str(timestamp) + ", " + path
self.r_temp.set("MODULE_"+self.subscriber_name + "_" + str(self.moduleNum), value)
self.r_temp.sadd("MODULE_TYPE_"+self.subscriber_name, str(self.moduleNum))
return message
except:
path = "?"
value = str(timestamp) + ", " + path
self.r_temp.set("MODULE_"+self.subscriber_name + "_" + str(self.moduleNum), value)
self.r_temp.sadd("MODULE_TYPE_"+self.subscriber_name, str(self.moduleNum))
return message
def populate_set_out(self, msg, channel=None):

2
bin/LAUNCH.sh
View File

@ -158,7 +158,7 @@ function launching_scripts {
sleep 0.1
screen -S "Script" -X screen -t "SQLInjectionDetection" bash -c './SQLInjectionDetection.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Browse_warning_paste" bash -c './Browse_warning_paste.py; read x'
screen -S "Script" -X screen -t "BrowseWarningPaste" bash -c './BrowseWarningPaste.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "SentimentAnalysis" bash -c './SentimentAnalysis.py; read x'

39
bin/ModuleInformation.py
View File

@ -29,6 +29,7 @@ import textwrap
threshold_stucked_module = 60*60*1 #1 hour
log_filename = "../logs/moduleInfo.log"
command_search_pid = "ps a -o pid,cmd | grep {}"
command_search_name = "ps a -o pid,cmd | grep {}"
command_restart_module = "screen -S \"Script\" -X screen -t \"{}\" bash -c \"./{}.py; read x\""
@ -45,6 +46,23 @@ def clearRedisModuleInfo():
for k in server.keys("MODULE_*"):
server.delete(k)
def cleanRedis():
for k in server.keys("MODULE_TYPE_*"):
moduleName = k[12:].split('_')[0]
for pid in server.smembers(k):
flag_pid_valid = False
proc = Popen([command_search_name.format(pid)], stdin=PIPE, stdout=PIPE, bufsize=1, shell=True)
for line in proc.stdout:
splittedLine = line.split()
if ('python2' in splittedLine or 'python' in splittedLine) and "./"+moduleName+".py" in splittedLine:
flag_pid_valid = True
if not flag_pid_valid:
print flag_pid_valid, 'cleaning', pid, 'in', k
server.srem(k, pid)
time.sleep(5)
def kill_module(module):
print ''
print '-> trying to kill module:', module
@ -76,8 +94,8 @@ if __name__ == "__main__":
parser = argparse.ArgumentParser(description='Show info concerning running modules and log suspected stucked modules. May be use to automatically kill and restart stucked one.')
parser.add_argument('-r', '--refresh', type=int, required=False, default=1, help='Refresh rate')
parser.add_argument('-k', '--autokill', type=int, required=True, default=1, help='Enable auto kill option (1 for TRUE, anything else for FALSE)')
parser.add_argument('-c', '--clear', type=int, required=False, default=1, help='Clear the current module information (Used to clear data from old launched modules)')
parser.add_argument('-k', '--autokill', type=int, required=False, default=0, help='Enable auto kill option (1 for TRUE, anything else for FALSE)')
parser.add_argument('-c', '--clear', type=int, required=False, default=0, help='Clear the current module information (Used to clear data from old launched modules)')
args = parser.parse_args()
@ -99,6 +117,7 @@ if __name__ == "__main__":
if args.clear == 1:
clearRedisModuleInfo()
lastTime = datetime.datetime.now()
module_file_array = set()
with open('../doc/all_modules.txt', 'r') as module_file:
@ -108,20 +127,15 @@ if __name__ == "__main__":
while True:
all_queue = set()
curr_range = 50
printarray1 = []
printarray2 = []
printarray3 = []
for queue, card in server.hgetall("queues").iteritems():
all_queue.add(queue)
key = "MODULE_" + queue + "_"
for i in range(1, 50):
curr_num = server.get("MODULE_"+ queue + "_" + str(i))
if curr_num is None:
curr_range = i
break
keySet = "MODULE_TYPE_" + queue
for moduleNum in range(1, curr_range):
for moduleNum in server.smembers(keySet):
value = server.get(key + str(moduleNum))
if value is not None:
timestamp, path = value.split(", ")
@ -147,8 +161,8 @@ if __name__ == "__main__":
printarray1.sort(lambda x,y: cmp(x[4], y[4]), reverse=True)
printarray2.sort(lambda x,y: cmp(x[4], y[4]), reverse=True)
printarray1.insert(0,["Queue", "#", "Amount", "Paste start time", "Processing time for current paste (H:M:S)", "Paste hash"])
printarray2.insert(0,["Queue", "#","Amount", "Paste start time", "Time since idle (H:M:S)", "Last paste hash"])
printarray1.insert(0,["Queue", "PID", "Amount", "Paste start time", "Processing time for current paste (H:M:S)", "Paste hash"])
printarray2.insert(0,["Queue", "PID","Amount", "Paste start time", "Time since idle (H:M:S)", "Last paste hash"])
printarray3.insert(0,["Queue", "State"])
os.system('clear')
@ -195,4 +209,7 @@ if __name__ == "__main__":
print '\n'
print t3.table
if (datetime.datetime.now() - lastTime).total_seconds() > args.refresh*5:
lastTime = datetime.datetime.now()
cleanRedis()
time.sleep(args.refresh)

2
bin/packages/modules.cfg
View File

@ -63,7 +63,7 @@ publish = Redis_BrowseWarningPaste,Redis_Duplicate
[ModuleStats]
subscribe = Redis_ModuleStats
[Browse_warning_paste]
[BrowseWarningPaste]
subscribe = Redis_BrowseWarningPaste
#[send_to_queue]

14
var/www/Flask_server.py
View File

@ -81,19 +81,13 @@ def event_stream():
def get_queues(r):
# We may want to put the llen in a pipeline to do only one query.
data = [(queue, int(card)) for queue, card in r.hgetall("queues").iteritems()]
newData = []
curr_range = 50
for queue, card in data:
for queue, card in r.hgetall("queues").iteritems():
key = "MODULE_" + queue + "_"
for i in range(1, 50):
curr_num = r.get("MODULE_"+ queue + "_" + str(i))
if curr_num is None:
curr_range = i
break
keySet = "MODULE_TYPE_" + queue
for moduleNum in range(1, curr_range):
for moduleNum in r.smembers(keySet):
value = r.get(key + str(moduleNum))
if value is not None:
timestamp, path = value.split(", ")

32
var/www/static/js/indexjavascript.js
View File

@ -207,7 +207,7 @@ function create_queue_table() {
table.appendChild(tableHead);
table.appendChild(tableBody);
var heading = new Array();
heading[0] = "Queue Name"
heading[0] = "Queue Name.PID"
heading[1] = "Amount"
var tr = document.createElement('TR');
tableHead.appendChild(tr);
@ -255,13 +255,17 @@ function load_queues() {
var x = new Date();
for (i = 0; i < glob_tabvar.row1.length; i++){
if (glob_tabvar.row1[i][0] == 'Categ' || glob_tabvar.row1[i][0] == 'Curve'){
tmp_tab2.push(0);
curves_labels2.push(glob_tabvar.row1[i][0]);
if (glob_tabvar.row1[i][0].split(".")[0] == 'Categ' || glob_tabvar.row1[i][0].split(".")[0] == 'Curve'){
if (curves_labels2.indexOf(glob_tabvar.row1[i][0].split(".")[0]) == -1) {
tmp_tab2.push(0);
curves_labels2.push(glob_tabvar.row1[i][0].split("."));
}
}
else {
tmp_tab.push(0);
curves_labels.push(glob_tabvar.row1[i][0]);
if (curves_labels.indexOf(glob_tabvar.row1[i][0].split(".")[0]) == -1) {
tmp_tab.push(0);
curves_labels.push(glob_tabvar.row1[i][0].split("."));
}
}
}
tmp_tab.unshift(x);
@ -320,19 +324,29 @@ function load_queues() {
update_values();
if($('#button-toggle-queues').prop('checked')){
$("#queue-color-legend").show();
create_queue_table();
}
else{
$("#queueing").html('');
$("#queue-color-legend").hide();
}
queues_pushed = []
for (i = 0; i < (glob_tabvar.row1).length; i++){
if (glob_tabvar.row1[i][0] == 'Categ' || glob_tabvar.row1[i][0] == 'Curve'){
tmp_values2.push(glob_tabvar.row1[i][1]);
if (glob_tabvar.row1[i][0].split(".")[0] == 'Categ' || glob_tabvar.row1[i][0].split(".")[0] == 'Curve'){
if (queues_pushed.indexOf(glob_tabvar.row1[i][0].split(".")[0]) == -1) {
queues_pushed.push(glob_tabvar.row1[i][0].split("."));
tmp_values2.push(glob_tabvar.row1[i][1]);
}
}
else {
tmp_values.push(glob_tabvar.row1[i][1]);
if (curves_labels.indexOf(glob_tabvar.row1[i][0].split(".")[0]) == -1) {
queues_pushed.push(glob_tabvar.row1[i][0].split("."));
tmp_values.push(glob_tabvar.row1[i][1]);
}
}
}
tmp_values.unshift(x);

13
var/www/templates/index.html
View File

@ -66,7 +66,18 @@
</label>
<strong style="top: 3px; position: relative;">Display queues</strong>
<div>
<div class="table-responsive", id="queueing" style="margin-top:10px;"></div>
<div style="padding: 6px">
<table id="queue-color-legend">
<thead>
</thead>
<tbody>
<tr><td class="legendColorBox"><div style="border:1px solid #ccc;padding:1px"><div style="width:4px;height:0;border:5px solid #d0e9c6;overflow:hidden"></div></div></td><td> Working queues</td></tr>
<tr><td class="legendColorBox"><div style="border:1px solid #ccc;padding:1px"><div style="width:4px;height:0;border:5px solid #faf2cc;overflow:hidden"></div></div></td><td> Idling queues</td></tr>
<tr><td class="legendColorBox"><div style="border:1px solid #ccc;padding:1px"><div style="width:4px;height:0;border:5px solid #ebcccc;overflow:hidden"></div></div></td><td> Stucked queues</td></tr>
</tbody>
</table>
</div>
<div class="table-responsive", id="queueing" style="margin-top:10px; font-size: small;"></div>
<a href="{{ url_for('index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
</div>
<!-- /.navbar-static-side -->