CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #59 from mokaddem/processed-pastes-graph 

Processed pastes graph
pull/64/merge
Raphaël Vinot 2016-07-19 16:31:20 +02:00 committed by GitHub
parent b5292123c3 4ecb9a4b44
commit dc01fd894b
6 changed files with 89 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
bin/Attribute.py
View File

@ -51,12 +51,13 @@ if __name__ == "__main__":
PST = Paste.Paste(message)
else:
publisher.debug("Script Attribute is idling 1s")
print 'sleeping'
time.sleep(1)
continue
# FIXME do it directly in the class
PST.save_attribute_redis("p_encoding", PST._get_p_encoding())
PST.save_attribute_redis("p_language", PST._get_p_language())
#PST.save_attribute_redis("p_language", PST._get_p_language())
# FIXME why not all saving everything there.
PST.save_all_attributes_redis()
# FIXME Not used.

9
bin/Global.py
View File

@ -31,6 +31,8 @@ from Helper import Process
if __name__ == '__main__':
publisher.port = 6380
publisher.channel = 'Script'
processed_paste = 0
time_1 = time.time()
config_section = 'Global'
@ -54,6 +56,12 @@ if __name__ == '__main__':
continue
else:
print "Empty Queues: Waiting..."
if int(time.time() - time_1) > 30:
to_print = 'Global; ; ; ;glob Processed {0} paste(s)'.format(processed_paste)
print to_print
publisher.info(to_print)
time_1 = time.time()
processed_paste = 0
time.sleep(1)
continue
# Creating the full filepath
@ -66,3 +74,4 @@ if __name__ == '__main__':
with open(filename, 'wb') as f:
f.write(base64.standard_b64decode(gzip64encoded))
p.populate_set_out(filename)
processed_paste+=1

5
bin/packages/config.cfg.sample
View File

@ -4,6 +4,11 @@ pastes = PASTES
wordtrending_csv = var/www/static/csv/wordstrendingdata
wordsfile = files/wordfile
##### Flask #####
[Flask]
#Number of minutes displayed for the number of processed pastes.
minute_processed_paste = 10
##### Redis #####
[Redis_Cache]
host = localhost

3
var/www/Flask_server.py
View File

@ -122,7 +122,8 @@ def search():
@app.route("/")
def index():
return render_template("index.html")
default_minute = cfg.get("Flask", "minute_processed_paste")
return render_template("index.html", default_minute = default_minute)
@app.route("/monitoring/")

58
var/www/static/js/indexjavascript.js
View File

@ -1,3 +1,53 @@
// Plot and update the number of processed pastes
$(function() {
var data = [];
var default_minute = (typeof window.default_minute !== "undefined") ? parseInt(window.default_minute) : 10;
var totalPoints = 60*parseInt(default_minute); //60s*minute
var curr_max = 0;
function getData() {
if (data.length > 0){
var data_old = data[0];
data = data.slice(1);
curr_max = curr_max == data_old ? Math.max.apply(null, data) : curr_max;
}
while (data.length < totalPoints) {
var y = (typeof window.paste_num_tabvar !== "undefined") ? parseInt(window.paste_num_tabvar) : 0;
curr_max = y > curr_max ? y : curr_max;
data.push(y);
}
// Zip the generated y values with the x values
var res = [];
for (var i = 0; i < data.length; ++i) {
res.push([i, data[i]])
}
return res;
}
var updateInterval = 1000; //1s
var options = {
series: { shadowSize: 1 },
lines: { fill: true, fillColor: { colors: [ { opacity: 1 }, { opacity: 0.1 } ] }},
yaxis: { min: 0, max: 40 },
colors: ["#a971ff"],
grid: {
tickColor: "#dddddd",
borderWidth: 0
},
};
var plot = $.plot("#realtimechart", [ getData() ], options);
function update() {
plot.setData([getData()]);
plot.getOptions().yaxes[0].max = curr_max;
plot.setupGrid();
plot.draw();
setTimeout(update, updateInterval);
}
update();
});
function initfunc( csvay, scroot) {
window.csv = csvay;
window.scroot = scroot;
@ -38,6 +88,12 @@ function create_log_table(obj_json) {
var chansplit = obj_json.channel.split('.');
var parsedmess = obj_json.data.split(';');
if (parsedmess[0] == "Global"){
var paste_processed = parsedmess[4].split(" ")[2];
window.paste_num_tabvar = paste_processed;
return;
}
if( chansplit[1] == "INFO" ){
tr.className = "info";
}
@ -270,3 +326,5 @@ $(document).ready(function () {
}
});

16
var/www/templates/index.html
View File

@ -14,10 +14,12 @@
<link href="{{ url_for('static', filename='css/dygraph_gallery.css') }}" rel="stylesheet" type="text/css" />
<!-- JS -->
<script type="text/javascript" src="{{ url_for('static', filename='js/dygraph-combined.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.tagcanvas.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery-1.4.2.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
<script>
window.default_minute = {{ default_minute }};
function update_values() {
$SCRIPT_ROOT = {{ request.script_root|tojson|safe }};
$.getJSON($SCRIPT_ROOT+"/_stuff",
@ -50,7 +52,16 @@
<!-- /#side-menu -->
</div>
<!-- /.sidebar-collapse -->
<div class="table-responsive", id="queueing" style="margin-top:40px;"></div>
<div class="panel panel-default">
<div class="panel-heading">
<i class="fa fa-dashboard fa-fw"></i> Pastes since {{ default_minute }} min
</div>
<div class="panel-body">
<div id="realtimechart" style="height: 90px; padding: 0px; position: relative;"></div>
</div>
</div>
<!-- <div id="Graph_paste_num" style="height:90px; width:100%;"></div> -->
<div class="table-responsive", id="queueing" style="margin-top:10px;"></div>
<a href="{{ url_for('index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
</div>
<!-- /.navbar-static-side -->
@ -132,7 +143,6 @@
initfunc( "{{ url_for('static', filename='csv/wordstrendingdata.csv') }}", {{ request.script_root|tojson|safe }} );
</script>
<script src="{{ url_for('static', filename='js/jquery.js') }}"></script>
<script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/plugins/metisMenu/metisMenu.js') }}"></script>