From fb9a35960ed0b9e676972214cd5f05c030ab8309 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Mon, 30 Oct 2017 16:28:32 +0100 Subject: [PATCH 01/65] Added contrib webpage --- server.py | 45 +++++- templates/contrib.html | 328 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 371 insertions(+), 2 deletions(-) create mode 100644 templates/contrib.html diff --git a/server.py b/server.py index b0b3de7..012f302 100755 --- a/server.py +++ b/server.py @@ -95,7 +95,7 @@ def getZrange(keyCateg, date, topNum): date_str = str(date.year)+str(date.month)+str(date.day) keyname = "{}:{}".format(keyCateg, date_str) data = serv_redis_db.zrange(keyname, 0, 5, desc=True, withscores=True) - data = [ [record[0].decode('utf8'), record[1]] for record in data ] + data = [ [record[0].decode('utf8'), record[1]] for record in data ] return data @@ -108,7 +108,7 @@ def index(): "{:.0f}".format(cfg.getint('Dashboard' ,'size_world_pannel_perc')/100*ratioCorrection), "{:.0f}".format((100-cfg.getint('Dashboard' ,'size_world_pannel_perc'))/100*ratioCorrection) ] - return render_template('index.html', + return render_template('index.html', pannelSize=pannelSize, size_dashboard_width=[cfg.getint('Dashboard' ,'size_dashboard_left_width'), 12-cfg.getint('Dashboard', 'size_dashboard_left_width')], itemToPlot=cfg.get('Dashboard', 'item_to_plot'), @@ -128,6 +128,47 @@ def geo(): default_updateFrequency=cfg.getint('GEO' ,'updateFrequency') ) +@app.route("/contrib") +def contrib(): + return render_template('contrib.html', + ) + +@app.route("/_getTopContributor") +def getTopContributor(): + data = [ + { + 'progression': '', + 'logo_path': 'logo1', + 'org': 'CIRCL', + }, + { + 'progression': '', + 'logo_path': 'logo2', + 'org': 'CASES', + }, + { + 'progression': '', + 'logo_path': 'logo3', + 'org': 'SMILE', + }, + { + 'progression': '', + 'logo_path': 'logo4', + 'org': 'ORG4', + }, + { + 'progression': '', + 'logo_path': 'logo5', + 'org': 'ORG5', + }, + ] + return jsonify(data) + +@app.route("/_getTop5Overtime") +def getTop5Overtime(): + data = [{'label': 'CIRCL', 'data': [[0, 4], [1, 7], [2,14]]}, {'label': 'CASES', 'data': [[0, 1], [1, 5], [2,2]]}] + return jsonify(data) + @app.route("/_getTopCoord") def getTopCoord(): try: diff --git a/templates/contrib.html b/templates/contrib.html new file mode 100644 index 0000000..b7ff665 --- /dev/null +++ b/templates/contrib.html @@ -0,0 +1,328 @@ + + + + + + + + + + + MISP live dashboard + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + +
+
+
+
+
+
+ +
+
+ Contributor Ranking +
+
+ +
+ + + + + + + + + + + +
LogoOrganization name
+
+ +
+ +
+ + +
+ + +
+ +
+
+ Hall Of Fame +
+
+ +
+ + + + + + + + + + +
LogoOrganization name
+
+ +
+ +
+ +
+
+ Top 5 Contributor overtime +
+
+
+
+ + +
+ +
+ +
+
+ +
+ +
+ + +
+ + + + + + + From 36f1cd727f35e85357f32ddd2152732bbb881c4e Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Tue, 31 Oct 2017 12:35:05 +0100 Subject: [PATCH 02/65] Continuation of contrib webpage UI --- server.py | 9 ++ static/css/ranking.css | 74 +++++++++++++ templates/contrib.html | 230 ++++++++++++++++++++++++++++++++--------- 3 files changed, 264 insertions(+), 49 deletions(-) create mode 100644 static/css/ranking.css diff --git a/server.py b/server.py index 012f302..03fc056 100755 --- a/server.py +++ b/server.py @@ -169,6 +169,15 @@ def getTop5Overtime(): data = [{'label': 'CIRCL', 'data': [[0, 4], [1, 7], [2,14]]}, {'label': 'CASES', 'data': [[0, 1], [1, 5], [2,2]]}] return jsonify(data) +@app.route("/_getCategPerContrib") +def getCategPerContrib(): + data = [["", "", "CIRCL", 723, 21, 32], +["", "", "CASES", 32, 435, 2], +["", "", "SMILE", 65, 231, 3], +["", "", "ORG4", 34, 21, 7], +["", "", "ORG5", 45, 211, 9]] + return jsonify(data) + @app.route("/_getTopCoord") def getTopCoord(): try: diff --git a/static/css/ranking.css b/static/css/ranking.css new file mode 100644 index 0000000..031f46e --- /dev/null +++ b/static/css/ranking.css @@ -0,0 +1,74 @@ + +.ranking { + position: relative; + height: auto; +} + +.table > tbody > tr > td.centerCell { + text-align: center; + min-width: 45px; +} + + +.table > tbody > tr > td.centerCellPic { + padding: 5px; +} + +.star1-1{ + +} + +.star2-1 { + +} +.star2-2 { + left: 0px; +} + +.star3-1 { + top: 6px; + left: 5px; +} +.star3-2 { + top: -8px; + left: 0px; +} +.star3-3 { + top: 6px; + left: -5px; +} + +.star4-1 { + +} +.star4-2 { + left: -6px; +} +.star4-3 { + left: -36px; + top: -10px; +} +.star4-4 { + left: -42px; + top: -10px; +} + +.star5-1 { + top: 7px; +} +.star5-2 { + left: -3px; + top: 7px; +} +.star5-3 { + left: -37px; + top: -7px; +} +.star5-4 { + left: -47px; + top: -0px; +} +.star5-5 { + left: -57px; + top: -7px; +} diff --git a/templates/contrib.html b/templates/contrib.html index b7ff665..7155820 100644 --- a/templates/contrib.html +++ b/templates/contrib.html @@ -65,6 +65,11 @@ padding-left: 5px; } +.col-lg-12 { + padding-right: 5px; + padding-left: 5px; +} + .leftSepa { border-left-color: rgb(221, 221, 221); border-left-style: solid; @@ -119,30 +124,117 @@ small {
-
-
- Contributor Ranking -
-
- -
- - - - - - - - - - - -
LogoOrganization name
+
+
+
+ Contributor Ranking
+
-
- -
+
+ + + + + + + + + + + +
LogoOrganization name
+
+ +
+ +
+
+ +
+
+
+ Top progression +
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
LogoOrganization name
+
+
+ +
+
+ +
+
+
+ Contributors and Categories +
+
+ +
+ + + + + + + + + + + + + +
LogoOrganization nameNetwork ActivityPayload deliveryOthers
+
+ +
+ +
+
@@ -152,7 +244,7 @@ small {
- Hall Of Fame + Hall Of Fame
@@ -179,7 +271,7 @@ small { Top 5 Contributor overtime
-
+
@@ -240,11 +332,32 @@ small { var datatableTop; var datatableFame; - var optionDatatable = { + + var optionsLineChart = { + series: { + shadowSize: 0 , + lines: { + fill: true, + fillColor: { + colors: [ { opacity: 1 }, { opacity: 0.1 } ] + } + } + }, + points: { show: true }, + lines: { show: true, fill: true }, + grid: { + tickColor: "#dddddd", + borderWidth: 0 + }, + legend: { + show: true, + position: "nw" + } + }; + var optionDatatable_light = { responsive: true, searching: false, ordering: false, - order: [[0, "asc"]], scrollY: '50vh', scrollCollapse: true, paging: false, @@ -259,10 +372,50 @@ small { { className: "centerCell", "targets": [ 0 ] } ] }; + var optionDatatable_light2 = { + responsive: true, + searching: false, + ordering: false, + scrollY: '50vh', + scrollCollapse: true, + paging: false, + "language": { + "lengthMenu": "", + "info": "", + "infoFiltered": "", + "infoEmpty": "", + }, + "info": false, + "columnDefs": [ + { className: "centerCellPic", "targets": [ 0 ] } + ] + }; + + var optionDatatable_Categ = { + ajax: { + url: "{{ url_for('getCategPerContrib') }}", + dataSrc: '' + }, + responsive: true, + searching: false, + scrollY: '50vh', + scrollCollapse: true, + paging: false, + "info": false, + "columnDefs": [ + { className: "centerCell", "targets": [ 0 ] } + ] + }; + + $.getJSON( "{{ url_for('getCategPerContrib') }}", function( data ) { + console.log(data); + }); $(document).ready(function() { - datatableTop = $('#topContribTable').DataTable(optionDatatable); - datatableFame = $('#fameTable').DataTable(optionDatatable); + datatableTop = $('#topContribTable').DataTable(optionDatatable_light); + datatableFame = $('#fameTable').DataTable(optionDatatable_light); + datatableCateg = $('#categTable').DataTable(optionDatatable_Categ); + datatableProgress = $('#topProgressTable').DataTable(optionDatatable_light2); // top contributors $.getJSON( "{{ url_for('getTopContributor') }}", function( data ) { for (i in data) { @@ -300,27 +453,6 @@ small { }); }); - var optionsLineChart = { - series: { - shadowSize: 0 , - lines: { - fill: true, - fillColor: { - colors: [ { opacity: 1 }, { opacity: 0.1 } ] - } - } - }, - points: { show: true }, - lines: { show: true, fill: true }, - grid: { - tickColor: "#dddddd", - borderWidth: 0 - }, - legend: { - show: true, - position: "nw" - } - }; From cd276488040e4e36143db1c301e3e810bede390c Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Tue, 31 Oct 2017 12:35:39 +0100 Subject: [PATCH 03/65] Added contrib webpage --- templates/contrib.html | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/templates/contrib.html b/templates/contrib.html index 7155820..b2de773 100644 --- a/templates/contrib.html +++ b/templates/contrib.html @@ -170,31 +170,31 @@ small { - + - + - + - + - + From 7438af529217e7221af09948f20b8abd7e74e6da Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Tue, 31 Oct 2017 15:08:44 +0100 Subject: [PATCH 04/65] Continuation of contrib page (2) --- server.py | 96 +++++++++++++++++++++---- static/css/ranking.css | 2 + templates/contrib.html | 155 +++++++++++++++++++---------------------- 3 files changed, 156 insertions(+), 97 deletions(-) diff --git a/server.py b/server.py index 03fc056..72d57a7 100755 --- a/server.py +++ b/server.py @@ -133,36 +133,67 @@ def contrib(): return render_template('contrib.html', ) -@app.route("/_getTopContributor") -def getTopContributor(): +@app.route("/_getLastContributor") +def getLastContributor(): data = [ { - 'progression': '', + 'rank': random.randint(1,16), 'logo_path': 'logo1', 'org': 'CIRCL', }, { - 'progression': '', + 'rank': random.randint(1,16), 'logo_path': 'logo2', 'org': 'CASES', }, { - 'progression': '', + 'rank': random.randint(1,16), 'logo_path': 'logo3', 'org': 'SMILE', }, { - 'progression': '', + 'rank': random.randint(1,16), 'logo_path': 'logo4', 'org': 'ORG4', }, { - 'progression': '', + 'rank': random.randint(1,16), 'logo_path': 'logo5', 'org': 'ORG5', }, ] - return jsonify(data) + return jsonify(data*2) + +@app.route("/_getTopContributor") +def getTopContributor(): + data = [ + { + 'rank': random.randint(1,16), + 'logo_path': 'logo1', + 'org': 'CIRCL', + }, + { + 'rank': random.randint(1,16), + 'logo_path': 'logo2', + 'org': 'CASES', + }, + { + 'rank': random.randint(1,16), + 'logo_path': 'logo3', + 'org': 'SMILE', + }, + { + 'rank': random.randint(1,16), + 'logo_path': 'logo4', + 'org': 'ORG4', + }, + { + 'rank': random.randint(1,16), + 'logo_path': 'logo5', + 'org': 'ORG5', + }, + ] + return jsonify(data*2) @app.route("/_getTop5Overtime") def getTop5Overtime(): @@ -171,12 +202,49 @@ def getTop5Overtime(): @app.route("/_getCategPerContrib") def getCategPerContrib(): - data = [["", "", "CIRCL", 723, 21, 32], -["", "", "CASES", 32, 435, 2], -["", "", "SMILE", 65, 231, 3], -["", "", "ORG4", 34, 21, 7], -["", "", "ORG5", 45, 211, 9]] - return jsonify(data) + data = [ + { + 'rank': random.randint(1,16), + 'logo_path': 'logo1', + 'org': 'CIRCL', + 'network_activity': random.randint(100,1600), + 'payload_delivery': random.randint(100,1600), + 'others': random.randint(1,16) + }, + { + 'rank': random.randint(1,16), + 'logo_path': 'logo2', + 'org': 'CASES', + 'network_activity': random.randint(10,1600), + 'payload_delivery': random.randint(10,1600), + 'others': random.randint(1,16) + }, + { + 'rank': random.randint(1,16), + 'logo_path': 'logo3', + 'org': 'SMILE', + 'network_activity': random.randint(1,160), + 'payload_delivery': random.randint(1,160), + 'others': random.randint(1,160) + }, + { + 'rank': random.randint(1,16), + 'logo_path': 'logo4', + 'org': 'ORG4', + 'network_activity': random.randint(1,160), + 'payload_delivery': random.randint(1,160), + 'others': random.randint(1,16) + }, + { + 'rank': random.randint(1,16), + 'logo_path': 'logo5', + 'org': 'ORG5', + 'network_activity': random.randint(1,16), + 'payload_delivery': random.randint(1,16), + 'others': random.randint(1,16) + }, + ] + return jsonify(data*2) @app.route("/_getTopCoord") def getTopCoord(): diff --git a/static/css/ranking.css b/static/css/ranking.css index 031f46e..ff8cca4 100644 --- a/static/css/ranking.css +++ b/static/css/ranking.css @@ -12,6 +12,8 @@ .table > tbody > tr > td.centerCellPic { padding: 5px; + text-align: center; + min-width: 45px; } .star1-1{ diff --git a/templates/contrib.html b/templates/contrib.html index b2de773..64acf1a 100644 --- a/templates/contrib.html +++ b/templates/contrib.html @@ -135,8 +135,8 @@ small { - - + + @@ -154,51 +154,21 @@ small {
- Top progression + Last contributors
-
+
-
#Org rank Logo Organization name
+
- - + + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
#Org rank LogoOrganization nameOrg name
@@ -218,7 +188,8 @@ small { - + + @@ -252,7 +223,8 @@ small {
#Org rank Logo Organization name Network Activity
- + + @@ -271,7 +243,7 @@ small { Top 5 Contributor overtime
-
+
@@ -295,6 +267,17 @@ small { /* DATA FROM CONF */ + function getRankIcon(rank) { + var imgsPath = "{{ url_for('static', filename='pics/rankingMISP/1.png') }}"; + imgsPath = imgsPath.substring(0, imgsPath.length-5); + rankLogoPath = imgsPath+rank+'.png'; + var img = document.createElement('img'); + img.src = rankLogoPath; + img.height = 26; + img.width = 26; + return img.outerHTML; + } + function create_stars_dom(n) { var n = 4-n; //1 is best -> 3 stars var div = document.createElement('DIV'); @@ -358,7 +341,7 @@ small { responsive: true, searching: false, ordering: false, - scrollY: '50vh', + scrollY: '30vh', scrollCollapse: true, paging: false, "language": { @@ -368,61 +351,34 @@ small { "infoEmpty": "", }, "info": false, - "columnDefs": [ - { className: "centerCell", "targets": [ 0 ] } - ] - }; - var optionDatatable_light2 = { - responsive: true, - searching: false, - ordering: false, - scrollY: '50vh', - scrollCollapse: true, - paging: false, - "language": { - "lengthMenu": "", - "info": "", - "infoFiltered": "", - "infoEmpty": "", - }, - "info": false, - "columnDefs": [ - { className: "centerCellPic", "targets": [ 0 ] } - ] }; + var optionDatatable_top = jQuery.extend({}, optionDatatable_light) + var optionDatatable_last = jQuery.extend({}, optionDatatable_light) + var optionDatatable_fame = jQuery.extend({}, optionDatatable_light) + optionDatatable_fame.scrollY = '50vh'; var optionDatatable_Categ = { - ajax: { - url: "{{ url_for('getCategPerContrib') }}", - dataSrc: '' - }, responsive: true, - searching: false, - scrollY: '50vh', + searching: true, + scrollY: '39vh', scrollCollapse: true, paging: false, "info": false, - "columnDefs": [ - { className: "centerCell", "targets": [ 0 ] } - ] }; - $.getJSON( "{{ url_for('getCategPerContrib') }}", function( data ) { - console.log(data); - }); - $(document).ready(function() { - datatableTop = $('#topContribTable').DataTable(optionDatatable_light); - datatableFame = $('#fameTable').DataTable(optionDatatable_light); + datatableTop = $('#topContribTable').DataTable(optionDatatable_top); + datatableFame = $('#fameTable').DataTable(optionDatatable_fame); datatableCateg = $('#categTable').DataTable(optionDatatable_Categ); - datatableProgress = $('#topProgressTable').DataTable(optionDatatable_light2); + datatableLast = $('#lastTable').DataTable(optionDatatable_last); // top contributors $.getJSON( "{{ url_for('getTopContributor') }}", function( data ) { for (i in data) { var row = data[i]; + i = parseInt(i); var to_add = [ - getIndexRank(), - row.progression, + i+1, + getRankIcon(row.rank), row.logo_path, row.org ]; @@ -434,11 +390,12 @@ small { // hall of fame $.getJSON( "{{ url_for('getTopContributor') }}", function( data ) { datatableIndex = 1; - var table = document.getElementById('fameTableBody'); for (i in data) { var row = data[i]; + i = parseInt(i); var to_add = [ - getIndexRank(), + i+1, + getRankIcon(row.rank), row.logo_path, row.org ]; @@ -447,6 +404,38 @@ small { datatableFame.draw(); }); + $.getJSON( "{{ url_for('getTopContributor') }}", function( data ) { + for (i in data) { + var row = data[i]; + i = parseInt(i); + var to_add = [ + i+1, + getRankIcon(row.rank), + row.logo_path, + row.org + ]; + datatableLast.row.add(to_add); + } + datatableLast.draw(); + }); + + $.getJSON( "{{ url_for('getCategPerContrib') }}", function( data ) { + for (i in data) { + var row = data[i]; + i = parseInt(i); + var to_add = [ + i+1, + getRankIcon(row.rank), + row.logo_path, + row.org, + row.network_activity, + row.payload_delivery, + row.others + ]; + datatableCateg.row.add(to_add); + } + datatableCateg.draw(); + }); // top 5 contrib overtime $.getJSON( "{{ url_for('getTop5Overtime') }}", function( data ) { var plotLineChart = $.plot("#divTop5Overtime", data, optionsLineChart); From 4293ad52c957bb166f5f8218bc821753301d9c31 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Tue, 31 Oct 2017 15:11:59 +0100 Subject: [PATCH 05/65] Minor css change --- static/css/ranking.css | 3 +++ 1 file changed, 3 insertions(+) diff --git a/static/css/ranking.css b/static/css/ranking.css index ff8cca4..18db351 100644 --- a/static/css/ranking.css +++ b/static/css/ranking.css @@ -9,6 +9,9 @@ min-width: 45px; } +.dataTables_filter > label { + margin-bottom: 0px; +} .table > tbody > tr > td.centerCellPic { padding: 5px; From 9fc82589ab321882ba4f02183c7df354ddd4f39b Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Tue, 31 Oct 2017 15:49:04 +0100 Subject: [PATCH 06/65] Added typeahead orgname --- server.py | 5 +++++ templates/contrib.html | 51 +++++++++++++++--------------------------- templates/geo.html | 9 ++++---- 3 files changed, 28 insertions(+), 37 deletions(-) diff --git a/server.py b/server.py index 72d57a7..0b5bb5c 100755 --- a/server.py +++ b/server.py @@ -246,6 +246,11 @@ def getCategPerContrib(): ] return jsonify(data*2) +@app.route("/_getAllOrg") +def getAllOrg(): + data = ['CIRCL', 'CASES', 'SMILE' ,'ORG4' ,'ORG5'] + return jsonify(data) + @app.route("/_getTopCoord") def getTopCoord(): try: diff --git a/templates/contrib.html b/templates/contrib.html index 64acf1a..dfdf332 100644 --- a/templates/contrib.html +++ b/templates/contrib.html @@ -26,6 +26,7 @@ + @@ -112,6 +113,11 @@ small {
  • MISP Geolocalisation
  • MISP Contributors
    • +
    +
    + Your organisation name: +
    +
    @@ -278,39 +284,18 @@ small { return img.outerHTML; } - function create_stars_dom(n) { - var n = 4-n; //1 is best -> 3 stars - var div = document.createElement('DIV'); - div.classList.add('ranking'); - for(var i=1; i<=n; i++) { - var span = document.createElement('span'); - span.classList.add("glyphicon"); - span.classList.add("glyphicon-star"); - span.classList.add("star"+n+"-"+i); - div.appendChild(span); + var allOrg = []; + var typeaheadOption = { + source: function (query, process) { + if (allOrg.length == 0) { // caching + return $.getJSON("{{ url_for('getAllOrg') }}", function (data) { + allOrg = data; + return process(data); + }); + } else { + return process(allOrg); + } } - return div.outerHTML; - } - - var datatableIndex = 1; - function getIndexRank() { - var to_return; - switch (datatableIndex) { - case 1: - to_return = create_stars_dom(1); - break; - case 2: - to_return = create_stars_dom(2); - break; - case 3: - to_return = create_stars_dom(3); - break; - - default: - to_return = datatableIndex; - } - datatableIndex++; - return to_return; } var datatableTop; @@ -367,6 +352,7 @@ small { }; $(document).ready(function() { + $('#orgName').typeahead(typeaheadOption); datatableTop = $('#topContribTable').DataTable(optionDatatable_top); datatableFame = $('#fameTable').DataTable(optionDatatable_fame); datatableCateg = $('#categTable').DataTable(optionDatatable_Categ); @@ -389,7 +375,6 @@ small { // hall of fame $.getJSON( "{{ url_for('getTopContributor') }}", function( data ) { - datatableIndex = 1; for (i in data) { var row = data[i]; i = parseInt(i); diff --git a/templates/geo.html b/templates/geo.html index 50302b2..2dfbfde 100644 --- a/templates/geo.html +++ b/templates/geo.html @@ -58,7 +58,7 @@ } .textTopHeader { - height: 50px; + height: 50px; text-align: center; margin-left: 8px; float: left; @@ -90,6 +90,7 @@ small {
    @@ -184,7 +185,7 @@ small {
    - +
    Hit map @@ -198,7 +199,7 @@ small {
    Geospatial information - Dates: + Dates: @@ -225,7 +226,7 @@ small { - + From cfd1818a33bbb2e9b9f5aab36b12664b49f065e8 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Thu, 2 Nov 2017 10:16:34 +0100 Subject: [PATCH 09/65] Added ranking table --- server.py | 2 +- static/css/ranking.css | 13 +++++ static/js/contrib.js | 118 +++++++++++++++++++++++------------------ templates/contrib.html | 18 +++---- 4 files changed, 90 insertions(+), 61 deletions(-) diff --git a/server.py b/server.py index c94ccf9..37d07c4 100755 --- a/server.py +++ b/server.py @@ -249,7 +249,7 @@ def getCategPerContrib(): @app.route("/_getAllOrg") def getAllOrg(): - data = ['CIRCL', 'CASES', 'SMILE' ,'ORG4' ,'ORG5'] + data = ['CIRCL', 'CASES', 'SMILE' ,'ORG4' ,'ORG5', 'SUPER HYPER LONG ORGINZATION NAME'] return jsonify(data) @app.route("/_getTopCoord") diff --git a/static/css/ranking.css b/static/css/ranking.css index d6c1205..f9e85cf 100644 --- a/static/css/ranking.css +++ b/static/css/ranking.css @@ -3,6 +3,19 @@ height: auto; } +.popover-content { + padding: 3px +} + +.popOverBtn { + float: left; + padding: 0px; + padding-right: 8px; + height: 46px; + margin-top: 2px; + margin-bottom: 2px; +} + .table > tbody > tr > td.centerCell { text-align: center; min-width: 45px; diff --git a/static/js/contrib.js b/static/js/contrib.js index 1acccfc..72f7c27 100644 --- a/static/js/contrib.js +++ b/static/js/contrib.js @@ -1,7 +1,17 @@ -/* CONFIG */ +/* GLOB VAR */ var allOrg = []; var datatableTop; var datatableFame; + +/* CONFIG */ +var maxRank = 16; +var popOverOption = { + trigger: "hover", + title: "Ranking icons", + html: true, + placement: 'bottom', + content: generateRankingSheet() +} var optionsLineChart = { series: { shadowSize: 0 , @@ -65,11 +75,12 @@ var typeaheadOption = { }, updater: function(item) { $('#orgText').text(item); + $('#btnCurrRank').show(); } } /* FUNCTIONS */ -function getRankIcon(rank, size) { +function getRankIcon(rank, size, header) { rankLogoPath = url_baseRankLogo+rank+'.png'; var img = document.createElement('img'); img.src = rankLogoPath; @@ -77,11 +88,53 @@ function getRankIcon(rank, size) { img.height = 26; img.width = 26; } else { - img.height = size; - img.width = size; + if (header) { + img.height = size; + img.width = size; + img.style.position = 'absolute'; + img.style.top = '0'; + img.style.bottom = '0'; + img.style.margin = 'auto'; + img.style.left = '0px'; + } else { + img.height = size; + img.width = size; + } } return img.outerHTML; } +function generateRankingSheet() { + var table = document.createElement('table'); + table.classList.add('table', 'table-striped'); + table.style.marginBottom = '0px'; + //head + var thead = document.createElement('thead'); + var tr = document.createElement('tr'); + var th = document.createElement('th'); + th.innerHTML = "Ranking"; + tr.appendChild(th); + var th = document.createElement('th'); + th.innerHTML = "Requirement"; + tr.appendChild(th); + thead.appendChild(tr); + //body + var tbody = document.createElement('tbody'); + for (var i=1; i<=maxRank; i++) { + var tr = document.createElement('tr'); + var td = document.createElement('td'); + td.innerHTML = getRankIcon(i, 20); + td.style.padding = "2px"; + tr.appendChild(td); + var td = document.createElement('td'); + td.innerHTML = i+" pnts"; + td.style.padding = "2px"; + tr.appendChild(td); + tbody.appendChild(tr); + } + table.appendChild(thead); + table.appendChild(tbody); + return table.outerHTML; +} function addToTableFromJson(datatable, url) { $.getJSON( url, function( data ) { @@ -105,60 +158,23 @@ function addToTableFromJson(datatable, url) { $(document).ready(function() { $('#orgName').typeahead(typeaheadOption); - $('#orgRankDiv').html(getRankIcon(8, 50)); - $('#orgNextRankDiv').html(getRankIcon(9, 50)); + $('#orgRankDiv').html(getRankIcon(8, 40, true)); + $('#orgNextRankDiv').html(getRankIcon(9, 40, true)); $('#orgText').text(currOrg); + if(currOrg != "") // currOrg selected + $('#btnCurrRank').show(); + $('[data-toggle="popover"]').popover(popOverOption); datatableTop = $('#topContribTable').DataTable(optionDatatable_top); datatableFame = $('#fameTable').DataTable(optionDatatable_fame); datatableCateg = $('#categTable').DataTable(optionDatatable_Categ); datatableLast = $('#lastTable').DataTable(optionDatatable_last); // top contributors - $.getJSON(url_getTopContributor , function( data ) { - for (i in data) { - var row = data[i]; - i = parseInt(i); - var to_add = [ - i+1, - getRankIcon(row.rank), - row.logo_path, - row.org - ]; - datatableTop.row.add(to_add); - } - datatableTop.draw(); - }); - + addToTableFromJson(datatableTop, url_getTopContributor); // hall of fame - $.getJSON( url_getTopContributor, function( data ) { - for (i in data) { - var row = data[i]; - i = parseInt(i); - var to_add = [ - i+1, - getRankIcon(row.rank), - row.logo_path, - row.org - ]; - datatableFame.row.add(to_add); - } - datatableFame.draw(); - }); - - $.getJSON( url_getTopContributor, function( data ) { - for (i in data) { - var row = data[i]; - i = parseInt(i); - var to_add = [ - i+1, - getRankIcon(row.rank), - row.logo_path, - row.org - ]; - datatableLast.row.add(to_add); - } - datatableLast.draw(); - }); - + addToTableFromJson(datatableFame, url_getTopContributor); + // last contributors + addToTableFromJson(datatableLast, url_getTopContributor); + // category per contributors $.getJSON( url_getCategPerContrib, function( data ) { for (i in data) { var row = data[i]; diff --git a/templates/contrib.html b/templates/contrib.html index d26bfd2..e37296e 100644 --- a/templates/contrib.html +++ b/templates/contrib.html @@ -67,16 +67,16 @@
    -
    -
    -
    -
    -
    -
    +
    -
    -
    +
    +
    From 8e2b5477f2a217b3ed867d5e3416ac38c533c7b1 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Thu, 2 Nov 2017 10:57:16 +0100 Subject: [PATCH 10/65] Dynamic update iof org rank in header --- server.py | 9 +++++++++ static/css/ranking.css | 4 +++- static/js/contrib.js | 34 ++++++++++++++++++++++------------ templates/contrib.html | 9 +++++---- 4 files changed, 39 insertions(+), 17 deletions(-) diff --git a/server.py b/server.py index 37d07c4..38fad50 100755 --- a/server.py +++ b/server.py @@ -252,6 +252,15 @@ def getAllOrg(): data = ['CIRCL', 'CASES', 'SMILE' ,'ORG4' ,'ORG5', 'SUPER HYPER LONG ORGINZATION NAME'] return jsonify(data) +@app.route("/_getOrgRank") +def getOrgRank(): + try: + org = request.args.get('org') + except: + org = '' + data = {'org': org, 'rank': random.randint(1,16)} + return jsonify(data) + @app.route("/_getTopCoord") def getTopCoord(): try: diff --git a/static/css/ranking.css b/static/css/ranking.css index f9e85cf..ef1f564 100644 --- a/static/css/ranking.css +++ b/static/css/ranking.css @@ -9,11 +9,13 @@ .popOverBtn { float: left; - padding: 0px; + padding-top: 2px; + padding-bottom: 2px; padding-right: 8px; height: 46px; margin-top: 2px; margin-bottom: 2px; + display: none; } .table > tbody > tr > td.centerCell { diff --git a/static/js/contrib.js b/static/js/contrib.js index 72f7c27..4fd438b 100644 --- a/static/js/contrib.js +++ b/static/js/contrib.js @@ -73,9 +73,8 @@ var typeaheadOption = { return process(allOrg); } }, - updater: function(item) { - $('#orgText').text(item); - $('#btnCurrRank').show(); + updater: function(org) { + updateProgressHeader(org); } } @@ -103,7 +102,7 @@ function getRankIcon(rank, size, header) { } return img.outerHTML; } -function generateRankingSheet() { +function generateRankingSheet(rank) { var table = document.createElement('table'); table.classList.add('table', 'table-striped'); table.style.marginBottom = '0px'; @@ -111,7 +110,7 @@ function generateRankingSheet() { var thead = document.createElement('thead'); var tr = document.createElement('tr'); var th = document.createElement('th'); - th.innerHTML = "Ranking"; + th.innerHTML = "Rank"; tr.appendChild(th); var th = document.createElement('th'); th.innerHTML = "Requirement"; @@ -128,6 +127,10 @@ function generateRankingSheet() { var td = document.createElement('td'); td.innerHTML = i+" pnts"; td.style.padding = "2px"; + tr.style.textAlign = "center"; + if (i == rank) { // current org rank + tr.classList.add('info') + } tr.appendChild(td); tbody.appendChild(tr); } @@ -154,16 +157,23 @@ function addToTableFromJson(datatable, url) { } - +function updateProgressHeader(org) { + // get Org rank + $.getJSON( url_getOrgRank+'?org='+org, function( data ) { + $('#btnCurrRank').show(); + $('#orgText').text(data.org); + var popoverRank = $('#btnCurrRank').data('bs.popover'); + popoverRank.options.content = generateRankingSheet(data.rank); + $('#orgRankDiv').html(getRankIcon(data.rank, 40, true)); + $('#orgNextRankDiv').html(getRankIcon(data.rank+1, 40, true)); + }); +} $(document).ready(function() { - $('#orgName').typeahead(typeaheadOption); - $('#orgRankDiv').html(getRankIcon(8, 40, true)); - $('#orgNextRankDiv').html(getRankIcon(9, 40, true)); - $('#orgText').text(currOrg); if(currOrg != "") // currOrg selected - $('#btnCurrRank').show(); - $('[data-toggle="popover"]').popover(popOverOption); + updateProgressHeader(org) + $('#orgName').typeahead(typeaheadOption); + $('#btnCurrRank').popover(popOverOption); datatableTop = $('#topContribTable').DataTable(optionDatatable_top); datatableFame = $('#fameTable').DataTable(optionDatatable_fame); datatableCateg = $('#categTable').DataTable(optionDatatable_Categ); diff --git a/templates/contrib.html b/templates/contrib.html index e37296e..4bea1aa 100644 --- a/templates/contrib.html +++ b/templates/contrib.html @@ -68,14 +68,14 @@
    @@ -234,6 +234,7 @@ var url_getCategPerContrib = "{{ url_for('getCategPerContrib') }}"; var url_getTop5Overtime = "{{ url_for('getTop5Overtime') }}"; var url_getAllOrg = "{{ url_for('getAllOrg') }}"; + var url_getOrgRank = "{{ url_for('getOrgRank') }}"; var url_baseRankLogo = "{{ url_for('static', filename='pics/rankingMISP/1.png') }}"; url_baseRankLogo = url_baseRankLogo.substring(0, url_baseRankLogo.length-5); From 2685a08019220cddfed4ea47a20ebcd04e845b57 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Thu, 2 Nov 2017 11:36:23 +0100 Subject: [PATCH 11/65] Added support of rank from org points --- server.py | 13 ++++++++++++- static/js/contrib.js | 9 ++++++--- templates/contrib.html | 2 +- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/server.py b/server.py index 38fad50..5641287 100755 --- a/server.py +++ b/server.py @@ -98,6 +98,16 @@ def getZrange(keyCateg, date, topNum): data = [ [record[0].decode('utf8'), record[1]] for record in data ] return data +# max lvl is 16 +def getRankLevel(points): + return float("{:.2f}".format(math.log(points, 2))) +def getRemainingPoints(points): + prev = 0 + for i in [2**x for x in range(1,17)]: + if prev <= points < i: + return i-points + prev = i + return 0 @app.route("/") def index(): @@ -258,7 +268,8 @@ def getOrgRank(): org = request.args.get('org') except: org = '' - data = {'org': org, 'rank': random.randint(1,16)} + points = random.randint(1,2**16) + data = {'org': org, 'points': points, 'rank': getRankLevel(points), 'remainingPts': getRemainingPoints(points)} return jsonify(data) @app.route("/_getTopCoord") diff --git a/static/js/contrib.js b/static/js/contrib.js index 4fd438b..26287dd 100644 --- a/static/js/contrib.js +++ b/static/js/contrib.js @@ -160,12 +160,15 @@ function addToTableFromJson(datatable, url) { function updateProgressHeader(org) { // get Org rank $.getJSON( url_getOrgRank+'?org='+org, function( data ) { + console.log(data); + var rank = Math.floor(data.rank); $('#btnCurrRank').show(); $('#orgText').text(data.org); var popoverRank = $('#btnCurrRank').data('bs.popover'); - popoverRank.options.content = generateRankingSheet(data.rank); - $('#orgRankDiv').html(getRankIcon(data.rank, 40, true)); - $('#orgNextRankDiv').html(getRankIcon(data.rank+1, 40, true)); + popoverRank.options.content = generateRankingSheet(rank); + $('#orgRankDiv').html(getRankIcon(rank, 40, true)); + $('#orgNextRankDiv').html(getRankIcon(rank+1, 40, true)); + $('#progressBarDiv').width((data.rank - rank)*150); //150 is empty bar width }); } diff --git a/templates/contrib.html b/templates/contrib.html index 4bea1aa..d1c9387 100644 --- a/templates/contrib.html +++ b/templates/contrib.html @@ -71,7 +71,7 @@
    -
    +
    From eed0528f525df898d5623bd2c5cb7569f524b7f0 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Thu, 2 Nov 2017 11:47:52 +0100 Subject: [PATCH 12/65] Put rankLvl multiplier in config + fixed bug log 0 --- server.py | 11 ++++++++--- static/js/contrib.js | 2 +- templates/contrib.html | 1 + 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/server.py b/server.py index 5641287..d81a645 100755 --- a/server.py +++ b/server.py @@ -100,10 +100,13 @@ def getZrange(keyCateg, date, topNum): # max lvl is 16 def getRankLevel(points): - return float("{:.2f}".format(math.log(points, 2))) + if points == 0: + return 0 + else: + return float("{:.2f}".format(math.log(points, cfg.getint('CONTRIB' ,'rankMultiplier')))) def getRemainingPoints(points): prev = 0 - for i in [2**x for x in range(1,17)]: + for i in [cfg.getint('CONTRIB' ,'rankMultiplier')**x for x in range(1,17)]: if prev <= points < i: return i-points prev = i @@ -141,7 +144,8 @@ def geo(): @app.route("/contrib") def contrib(): return render_template('contrib.html', - currOrg="" + currOrg="", + rankMultiplier=cfg.getint('CONTRIB' ,'rankMultiplier') ) @app.route("/_getLastContributor") @@ -269,6 +273,7 @@ def getOrgRank(): except: org = '' points = random.randint(1,2**16) + #FIXME put 0 if org has no points data = {'org': org, 'points': points, 'rank': getRankLevel(points), 'remainingPts': getRemainingPoints(points)} return jsonify(data) diff --git a/static/js/contrib.js b/static/js/contrib.js index 26287dd..7a7a412 100644 --- a/static/js/contrib.js +++ b/static/js/contrib.js @@ -125,7 +125,7 @@ function generateRankingSheet(rank) { td.style.padding = "2px"; tr.appendChild(td); var td = document.createElement('td'); - td.innerHTML = i+" pnts"; + td.innerHTML = Math.pow(rankMultiplier, i)+" pnts"; td.style.padding = "2px"; tr.style.textAlign = "center"; if (i == rank) { // current org rank diff --git a/templates/contrib.html b/templates/contrib.html index d1c9387..56913e1 100644 --- a/templates/contrib.html +++ b/templates/contrib.html @@ -240,6 +240,7 @@ /* DATA FROM CONF */ var currOrg = "{{ currOrg }}"; + var rankMultiplier = {{ rankMultiplier }}; From aabd4779f9147ced54c359ea1ce5cc736f76f5e9 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Thu, 2 Nov 2017 13:13:13 +0100 Subject: [PATCH 13/65] Added row coloring for selected ORG --- static/js/contrib.js | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/static/js/contrib.js b/static/js/contrib.js index 7a7a412..5d1e930 100644 --- a/static/js/contrib.js +++ b/static/js/contrib.js @@ -160,7 +160,7 @@ function addToTableFromJson(datatable, url) { function updateProgressHeader(org) { // get Org rank $.getJSON( url_getOrgRank+'?org='+org, function( data ) { - console.log(data); + datatableTop.draw(); var rank = Math.floor(data.rank); $('#btnCurrRank').show(); $('#orgText').text(data.org); @@ -169,6 +169,23 @@ function updateProgressHeader(org) { $('#orgRankDiv').html(getRankIcon(rank, 40, true)); $('#orgNextRankDiv').html(getRankIcon(rank+1, 40, true)); $('#progressBarDiv').width((data.rank - rank)*150); //150 is empty bar width + // update color in other dataTables + datatableTop.rows().every( function() { + var row = this.node(); + if(this.data()[3] == data.org) { row.classList.add('info'); } else { row.classList.remove('info'); } + }); + datatableFame.rows().every( function() { + var row = this.node(); + if(this.data()[3] == data.org) { row.classList.add('info'); } else { row.classList.remove('info'); } + }); + datatableCateg.rows().every( function() { + var row = this.node(); + if(this.data()[3] == data.org) { row.classList.add('info'); } else { row.classList.remove('info'); } + }); + datatableLast.rows().every( function() { + var row = this.node(); + if(this.data()[3] == data.org) { row.classList.add('info'); } else { row.classList.remove('info'); } + }); }); } From 3bfa5185dc01e7d96cad7df8f5f30a501ac5736c Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Thu, 2 Nov 2017 14:02:54 +0100 Subject: [PATCH 14/65] Few additional ui adjustement + filtering categ feature on button click --- static/css/ranking.css | 9 +++++++++ static/js/contrib.js | 11 +++++++---- templates/contrib.html | 6 ++---- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/static/css/ranking.css b/static/css/ranking.css index ef1f564..e63626d 100644 --- a/static/css/ranking.css +++ b/static/css/ranking.css @@ -3,6 +3,15 @@ height: auto; } +.centerInBtn { + float: left; + text-align: center; + vertical-align: middle; + line-height: 40px; + margin-right: 10px; + font-size: x-large; +} + .popover-content { padding: 3px } diff --git a/static/js/contrib.js b/static/js/contrib.js index 5d1e930..2abe1ff 100644 --- a/static/js/contrib.js +++ b/static/js/contrib.js @@ -7,7 +7,6 @@ var datatableFame; var maxRank = 16; var popOverOption = { trigger: "hover", - title: "Ranking icons", html: true, placement: 'bottom', content: generateRankingSheet() @@ -113,7 +112,7 @@ function generateRankingSheet(rank) { th.innerHTML = "Rank"; tr.appendChild(th); var th = document.createElement('th'); - th.innerHTML = "Requirement"; + th.innerHTML = "Requirement (CP)"; tr.appendChild(th); thead.appendChild(tr); //body @@ -125,7 +124,7 @@ function generateRankingSheet(rank) { td.style.padding = "2px"; tr.appendChild(td); var td = document.createElement('td'); - td.innerHTML = Math.pow(rankMultiplier, i)+" pnts"; + td.innerHTML = Math.pow(rankMultiplier, i); td.style.padding = "2px"; tr.style.textAlign = "center"; if (i == rank) { // current org rank @@ -156,10 +155,10 @@ function addToTableFromJson(datatable, url) { }); } - function updateProgressHeader(org) { // get Org rank $.getJSON( url_getOrgRank+'?org='+org, function( data ) { + console.log(data); datatableTop.draw(); var rank = Math.floor(data.rank); $('#btnCurrRank').show(); @@ -189,6 +188,10 @@ function updateProgressHeader(org) { }); } +function showOnlyOrg() { + datatableCateg.search( $('#orgText').text() ).draw(); +} + $(document).ready(function() { if(currOrg != "") // currOrg selected updateProgressHeader(org) diff --git a/templates/contrib.html b/templates/contrib.html index 56913e1..1a13c0c 100644 --- a/templates/contrib.html +++ b/templates/contrib.html @@ -64,10 +64,8 @@
    -
    - -
    -
    - - - - + {% for categ in categ_list_str %} + + {% endfor %} @@ -239,6 +238,8 @@ /* DATA FROM CONF */ var currOrg = "{{ currOrg }}"; var rankMultiplier = {{ rankMultiplier }}; + var categ_list = JSON.parse('{{ categ_list|safe }}'); + var categ_list_str = JSON.parse("{{ categ_list_str|safe }}"); From be20f75ad83e26bd089b2b2cc2ff1896e7d390d5 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Thu, 2 Nov 2017 16:21:04 +0100 Subject: [PATCH 20/65] Other UI adjustement --- server.py | 2 +- static/js/contrib.js | 2 +- templates/contrib.html | 18 +++++++++--------- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/server.py b/server.py index 68efcfb..28c880d 100755 --- a/server.py +++ b/server.py @@ -238,7 +238,7 @@ def getCategPerContrib(): @app.route("/_getAllOrg") def getAllOrg(): - data = ['CIRCL', 'CASES', 'SMILE' ,'ORG4' ,'ORG5', 'SUPER HYPER LONG ORGINZATION NAME'] + data = ['CIRCL', 'CASES', 'SMILE' ,'ORG4' ,'ORG5', 'SUPER HYPER LONG ORGINZATION NAME', 'Org3'] return jsonify(data) @app.route("/_getOrgRank") diff --git a/static/js/contrib.js b/static/js/contrib.js index 26aa59e..a17e541 100644 --- a/static/js/contrib.js +++ b/static/js/contrib.js @@ -55,7 +55,7 @@ optionDatatable_fame.scrollY = '50vh'; var optionDatatable_Categ = { responsive: true, searching: true, - scrollY: '39vh', + scrollY: '38vh', "scrollX": true, scrollCollapse: true, paging: false, diff --git a/templates/contrib.html b/templates/contrib.html index 0b09812..940d471 100644 --- a/templates/contrib.html +++ b/templates/contrib.html @@ -100,9 +100,9 @@ - + - + @@ -127,9 +127,9 @@ - + - + @@ -153,10 +153,11 @@ - + + {% for categ in categ_list_str %} - + {% endfor %} @@ -187,9 +188,9 @@ - + - + @@ -239,7 +240,6 @@ var currOrg = "{{ currOrg }}"; var rankMultiplier = {{ rankMultiplier }}; var categ_list = JSON.parse('{{ categ_list|safe }}'); - var categ_list_str = JSON.parse("{{ categ_list_str|safe }}"); From 76d93ecda81aa9d4d33b55e2365cd87f98c27f7a Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Fri, 3 Nov 2017 08:55:03 +0100 Subject: [PATCH 21/65] Improved ranking sheet with overall progression --- static/js/contrib.js | 37 ++++++++++++++++++++++++++++++------- templates/contrib.html | 2 +- 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/static/js/contrib.js b/static/js/contrib.js index a17e541..123766e 100644 --- a/static/js/contrib.js +++ b/static/js/contrib.js @@ -106,10 +106,13 @@ function getRankIcon(rank, size, header) { } return img.outerHTML; } -function generateRankingSheet(rank, stepPnt, pnt, Rpnt) { +function generateRankingSheet(rank, rankDec, stepPnt, pnt, Rpnt) { var Cpnt = pnt - stepPnt; var Tpnt = Cpnt + Rpnt; + var OuterDiv = document.createElement('div'); var gdiv = document.createElement('div'); + gdiv.id = "globalDiv"; + gdiv.style.float = 'left'; //progressBar var div = document.createElement('div'); div.classList.add('progress'); @@ -119,19 +122,18 @@ function generateRankingSheet(rank, stepPnt, pnt, Rpnt) { var div1 = document.createElement('div') div1.classList.add('progress-bar') div1.style.width = 100*(Cpnt)/Tpnt+'%'; - div1.innerHTML = pnt; + div1.innerHTML = ""+Cpnt+""; div.appendChild(div1); var div1 = document.createElement('div') div1.classList.add('progress-bar', 'progress-bar-warning') div1.style.width = 100*(Rpnt)/Tpnt+'%' - div1.innerHTML = Rpnt; + div1.innerHTML = ""+Rpnt+""; div.appendChild(div1); gdiv.appendChild(div); - // table var table = document.createElement('table'); table.classList.add('table', 'table-striped'); - table.style.marginBottom = '0px'; + table.style.marginBottom = '5px'; //head var thead = document.createElement('thead'); var tr = document.createElement('tr'); @@ -167,7 +169,26 @@ function generateRankingSheet(rank, stepPnt, pnt, Rpnt) { table.appendChild(thead); table.appendChild(tbody); gdiv.appendChild(table); - return gdiv.outerHTML; + OuterDiv.appendChild(gdiv); + // Tot nbr points + var tableHeight = 440; //HARDCODED... + var div = document.createElement('div'); + div.classList.add('progress'); + div.style.width = '20px'; + div.style.height = tableHeight+'px'; //HARDCODED... + div.style.display = 'flex' + div.style.float = 'left'; + div.style.marginTop = '56px'; + div.style.marginBottom = '0px'; + div.style.marginLeft = '2px'; + var div1 = document.createElement('div') + div1.classList.add('progress-bar', 'progress-bar-success', 'progress-bar-striped') + div1.style.height = ((rank+rankDec)*100/16)+'%'; + div1.style.width = '100%'; + div.appendChild(div1); + OuterDiv.appendChild(div); + + return OuterDiv.outerHTML; } function addToTableFromJson(datatable, url) { @@ -190,12 +211,14 @@ function addToTableFromJson(datatable, url) { function updateProgressHeader(org) { // get Org rank $.getJSON( url_getOrgRank+'?org='+org, function( data ) { + console.log(data); datatableTop.draw(); var rank = Math.floor(data.rank); + var rankDec = data.rank-rank; $('#btnCurrRank').show(); $('#orgText').text(data.org); var popoverRank = $('#btnCurrRank').data('bs.popover'); - popoverRank.options.content = generateRankingSheet(rank, data.stepPts, data.points, data.remainingPts); + popoverRank.options.content = generateRankingSheet(rank, rankDec, data.stepPts, data.points, data.remainingPts); $('#orgRankDiv').html(getRankIcon(rank, 40, true)); $('#orgNextRankDiv').html(getRankIcon(rank+1, 40, true)); if (data.rank > 16){ diff --git a/templates/contrib.html b/templates/contrib.html index 940d471..1dcb8e6 100644 --- a/templates/contrib.html +++ b/templates/contrib.html @@ -118,7 +118,7 @@
    - Last contributors + Last contributors (24h)
    From 0e5b03fee6ee058e9a30e43e70d95c00556a68fd Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Fri, 3 Nov 2017 09:32:07 +0100 Subject: [PATCH 22/65] Draft of adding contribution info to redis --- config/config.cfg.default | 8 +++++-- server.py | 15 ++++++++----- zmq_subscriber.py | 47 +++++++++++++++++++++++++++++++++------ 3 files changed, 56 insertions(+), 14 deletions(-) diff --git a/config/config.cfg.default b/config/config.cfg.default index 988cc7b..4a568fb 100644 --- a/config/config.cfg.default +++ b/config/config.cfg.default @@ -20,9 +20,13 @@ zoomlevel = 11 clusteringDistance = 10 [CONTRIB] -#[1.5 -> +inf] +max_number_of_last_contributor = 10 +#How much harder it gets to rank up (exponential multiplier) [1.5 -> +inf] rankMultiplier = 2 -categories = ["internal_reference", "targeting_data", "antivirus_detection", "payload_delivery", "artifacts_dropped", "payload_installation", "persistence_mechanism", "network_activity", "payload_type", "attribution", "external_analysis", "financial_fraud", "support_Tool", "social_network", "person", "other" ] +categories_in_datatable = ["internal_reference", "targeting_data", "antivirus_detection", "payload_delivery", "artifacts_dropped", "payload_installation", "persistence_mechanism", "network_activity", "payload_type", "attribution", "external_analysis", "financial_fraud", "support_Tool", "social_network", "person", "other" ] +default_pnts_per_contribution = 1 +# array of the form [[category, pntsRcv], ...] +pnts_per_contribution = [["payload_delivery", 5], ["artifact_dropped", 20], ["network_activity", 5]] [Log] field_to_plot = Attribute.category diff --git a/server.py b/server.py index 28c880d..842871a 100755 --- a/server.py +++ b/server.py @@ -28,7 +28,7 @@ serv_redis_db = redis.StrictRedis( port=cfg.getint('RedisGlobal', 'port'), db=cfg.getint('RedisDB', 'db')) -categories = json.loads(cfg.get('CONTRIB', 'categories')) +categories_in_datatable = json.loads(cfg.get('CONTRIB', 'categories_in_datatable')) subscriber_log = redis_server_log.pubsub(ignore_subscribe_messages=True) subscriber_log.psubscribe(cfg.get('RedisLog', 'channel')) @@ -145,8 +145,8 @@ def geo(): @app.route("/contrib") def contrib(): - categ_list = json.loads(cfg.get('CONTRIB', 'categories')) - categ_list_str = [ s[0].upper() + s[1:].replace('_', ' ') for s in json.loads(cfg.get('CONTRIB', 'categories'))] + categ_list = categories_in_datatable + categ_list_str = [ s[0].upper() + s[1:].replace('_', ' ') for s in categories_in_datatable] return render_template('contrib.html', currOrg="", rankMultiplier=cfg.getfloat('CONTRIB' ,'rankMultiplier'), @@ -230,7 +230,7 @@ def getCategPerContrib(): dic['rank'] = random.randint(1,16) dic['logo_path'] = 'logo' dic['org'] = 'Org'+str(d) - for f in categories: + for f in categories_in_datatable: dic[f] = random.randint(0,1600) data.append(dic) @@ -250,7 +250,12 @@ def getOrgRank(): points = random.randint(1,math.floor(cfg.getfloat('CONTRIB' ,'rankMultiplier')**16)) #FIXME put 0 if org has no points remainingPts = getRemainingPoints(points) - data = {'org': org, 'points': points, 'rank': getRankLevel(points), 'remainingPts': remainingPts['remainingPts'], 'stepPts': remainingPts['stepPts']} + data = {'org': org, + 'points': points, + 'rank': getRankLevel(points), + 'remainingPts': remainingPts['remainingPts'], + 'stepPts': remainingPts['stepPts'], + } return jsonify(data) @app.route("/_getTopCoord") diff --git a/zmq_subscriber.py b/zmq_subscriber.py index e0f6b99..cd01536 100755 --- a/zmq_subscriber.py +++ b/zmq_subscriber.py @@ -24,6 +24,14 @@ CHANNELDISP = cfg.get('RedisMap', 'channelDisp') CHANNEL_PROC = cfg.get('RedisMap', 'channelProc') PATH_TO_DB = cfg.get('RedisMap', 'pathMaxMindDB') +DEFAULT_PNTS_REWARD = cfg.get('CONTRIB', 'default_pnts_per_contribution') +categories_in_datatable = json.loads(cfg.get('CONTRIB', 'categories_in_datatable')) +DICO_PNTS_REWARD = {} +temp = json.loads(cfg.get('CONTRIB', 'pnts_per_contribution')) +for categ, pnts in temp: + DICO_PNTS_REWARD[categ] = pnts +MAX_NUMBER_OF_LAST_CONTRIBUTOR = cfg.getint('CONTRIB', 'max_number_of_last_contributor') + serv_log = redis.StrictRedis( host=cfg.get('RedisGlobal', 'host'), port=cfg.getint('RedisGlobal', 'port'), @@ -44,19 +52,18 @@ def publish_log(zmq_name, name, content): to_send = { 'name': name, 'log': json.dumps(content), 'zmqName': zmq_name } serv_log.publish(CHANNEL, json.dumps(to_send)) -def push_to_redis_zset(keyCateg, toAdd): +def push_to_redis_zset(keyCateg, toAdd, endSubkey="", count=1): now = datetime.datetime.now() - today_str = str(now.year)+str(now.month)+str(now.day) - keyname = "{}:{}".format(keyCateg, today_str) - serv_redis_db.zincrby(keyname, toAdd) + today_str = str(now.year)+str(now.month).zfill(2)+str(now.day).zfill(2) + keyname = "{}:{}{}".format(keyCateg, today_str, endSubkey) + serv_redis_db.zincrby(keyname, toAdd, count) def push_to_redis_geo(keyCateg, lon, lat, content): now = datetime.datetime.now() - today_str = str(now.year)+str(now.month)+str(now.day) + today_str = str(now.year)+str(now.month).zfill(2)+str(now.day).zfill(2) keyname = "{}:{}".format(keyCateg, today_str) serv_redis_db.geoadd(keyname, lon, lat, content) - def ip_to_coord(ip): resp = reader.city(ip) lat = float(resp.location.latitude) @@ -109,6 +116,28 @@ def getFields(obj, fields): except KeyError as e: return "" +def noSpaceLower(str): + return str.lower().replace(' ', '_') + +#pntMultiplier if one contribution rewards more than others. (e.g. shighting may gives more points than editing) +def handleContribution(org, categ, action, pntMultiplier=1): + if action in ['edit']: + pass + #return #not a contribution? + # is a valid contribution + try: + pnts_to_add = DICO_PNTS_REWARD[noSpaceLower(categ)] + except KeyError: + pnts_to_add = DEFAULT_PNTS_REWARD + pnts_to_add *= pntMultiplier + + push_to_redis_zset('CONTRIB_DAY', org, count=pnts_to_add) + #CONTRIB_CATEG retain the contribution per category, not the point earned in this categ + push_to_redis_zset('CONTRIB_CATEG', org, count=DEFAULT_PNTS_REWARD, endSubkey=':'+noSpaceLower(categ)) + serv_redis_db.sadd('CONTRIB_ALL_ORG', org) + serv_redis_db.lpush('CONTRIB_LAST', org) + serv_redis_db.ltrim('CONTRIB_LAST', 0, MAX_NUMBER_OF_LAST_CONTRIBUTOR-1) # Limit list size + #serv_redis_db.lrange('CONTRIB_LAST', 0, MAX_NUMBER_OF_LAST_CONTRIBUTOR-1) # get the last 10 contributors ############## ## HANDLERS ## @@ -129,6 +158,10 @@ def handler_keepalive(zmq_name, jsonevent): def handler_sighting(zmq_name, jsonsight): print('sending' ,'sighting') + org = jsonsight['org'] + categ = jsonsight['categ'] + action = jsonsight['action'] + handleContribution(org, categ, action) return def handler_event(zmq_name, jsonobj): @@ -165,6 +198,7 @@ def handler_attribute(zmq_name, jsonobj): if jsonattr['category'] == "Network activity": getCoordAndPublish(zmq_name, jsonattr['value'], jsonattr['category']) + handleContribution(jsonobj['Event']['Orgc']['name'], jsonattr['category'], jsonobj['action']) # Push to log publish_log(zmq_name, 'Attribute', to_push) @@ -210,4 +244,3 @@ if __name__ == "__main__": main(args.zmqname) reader.close() - From 2ca318b68e030abf23064591324f28026632a0b7 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Fri, 3 Nov 2017 09:35:51 +0100 Subject: [PATCH 23/65] Moved function into section for server script --- server.py | 204 +++++++++++++++++++++++++++++------------------------- 1 file changed, 108 insertions(+), 96 deletions(-) diff --git a/server.py b/server.py index 842871a..a771f41 100755 --- a/server.py +++ b/server.py @@ -114,6 +114,12 @@ def getRemainingPoints(points): prev = i return { 'remainingPts': 0, 'stepPts': cfg.getfloat('CONTRIB' ,'rankMultiplier')**16 } +########### +## ROUTE ## +########### + +''' MAIN ROUTE ''' + @app.route("/") def index(): ratioCorrection = 88 @@ -154,6 +160,108 @@ def contrib(): categ_list_str=categ_list_str ) +''' INDEX ''' + +@app.route("/_logs") +def logs(): + return Response(event_stream_log(), mimetype="text/event-stream") + +@app.route("/_maps") +def maps(): + return Response(event_stream_maps(), mimetype="text/event-stream") + +@app.route("/_get_log_head") +def getLogHead(): + return json.dumps(LogItem('').get_head_row()) + +def event_stream_log(): + for msg in subscriber_log.listen(): + content = msg['data'] + yield EventMessage(content).to_json() + +def event_stream_maps(): + for msg in subscriber_map.listen(): + content = msg['data'].decode('utf8') + yield 'data: {}\n\n'.format(content) + +''' GEO ''' + +@app.route("/_getTopCoord") +def getTopCoord(): + try: + date = datetime.datetime.fromtimestamp(float(request.args.get('date'))) + except: + date = datetime.datetime.now() + keyCateg = "GEO_COORD" + topNum = 6 # default Num + data = getZrange(keyCateg, date, topNum) + return jsonify(data) + +@app.route("/_getHitMap") +def getHitMap(): + try: + date = datetime.datetime.fromtimestamp(float(request.args.get('date'))) + except: + date = datetime.datetime.now() + keyCateg = "GEO_COUNTRY" + topNum = -1 # default Num + data = getZrange(keyCateg, date, topNum) + return jsonify(data) + +def isCloseTo(coord1, coord2): + clusterMeter = cfg.getfloat('GEO' ,'clusteringDistance') + clusterThres = math.pow(10, len(str(abs(clusterMeter)))-7) #map meter to coord threshold (~ big approx) + if abs(float(coord1[0]) - float(coord2[0])) <= clusterThres: + if abs(float(coord1[1]) - float(coord2[1])) <= clusterThres: + return True + return False + +@app.route("/_getCoordsByRadius") +def getCoordsByRadius(): + dico_coord = {} + to_return = [] + try: + dateStart = datetime.datetime.fromtimestamp(float(request.args.get('dateStart'))) + dateEnd = datetime.datetime.fromtimestamp(float(request.args.get('dateEnd'))) + centerLat = request.args.get('centerLat') + centerLon = request.args.get('centerLon') + radius = int(math.ceil(float(request.args.get('radius')))) + except: + return jsonify(to_return) + + delta = dateEnd - dateStart + for i in range(delta.days+1): + correctDatetime = dateStart + datetime.timedelta(days=i) + date_str = str(correctDatetime.year)+str(correctDatetime.month)+str(correctDatetime.day) + keyCateg = 'GEO_RAD' + keyname = "{}:{}".format(keyCateg, date_str) + res = serv_redis_db.georadius(keyname, centerLon, centerLat, radius, unit='km', withcoord=True) + + #sum up really close coord + for data, coord in res: + flag_added = False + coord = [coord[0], coord[1]] + #list all coord + for dicoCoordStr in dico_coord.keys(): + dicoCoord = json.loads(dicoCoordStr) + #if curCoord close to coord + if isCloseTo(dicoCoord, coord): + #add data to dico coord + dico_coord[dicoCoordStr].append(data) + flag_added = True + break + # coord not in dic + if not flag_added: + dico_coord[str(coord)] = [data] + + for dicoCoord, array in dico_coord.items(): + dicoCoord = json.loads(dicoCoord) + to_return.append([array, dicoCoord]) + + return jsonify(to_return) + +''' CONTRIB ''' + @app.route("/_getLastContributor") def getLastContributor(): data = [ @@ -258,101 +366,5 @@ def getOrgRank(): } return jsonify(data) -@app.route("/_getTopCoord") -def getTopCoord(): - try: - date = datetime.datetime.fromtimestamp(float(request.args.get('date'))) - except: - date = datetime.datetime.now() - keyCateg = "GEO_COORD" - topNum = 6 # default Num - data = getZrange(keyCateg, date, topNum) - return jsonify(data) - -@app.route("/_getHitMap") -def getHitMap(): - try: - date = datetime.datetime.fromtimestamp(float(request.args.get('date'))) - except: - date = datetime.datetime.now() - keyCateg = "GEO_COUNTRY" - topNum = -1 # default Num - data = getZrange(keyCateg, date, topNum) - return jsonify(data) - -def isCloseTo(coord1, coord2): - clusterMeter = cfg.getfloat('GEO' ,'clusteringDistance') - clusterThres = math.pow(10, len(str(abs(clusterMeter)))-7) #map meter to coord threshold (~ big approx) - if abs(float(coord1[0]) - float(coord2[0])) <= clusterThres: - if abs(float(coord1[1]) - float(coord2[1])) <= clusterThres: - return True - return False - -@app.route("/_getCoordsByRadius") -def getCoordsByRadius(): - dico_coord = {} - to_return = [] - try: - dateStart = datetime.datetime.fromtimestamp(float(request.args.get('dateStart'))) - dateEnd = datetime.datetime.fromtimestamp(float(request.args.get('dateEnd'))) - centerLat = request.args.get('centerLat') - centerLon = request.args.get('centerLon') - radius = int(math.ceil(float(request.args.get('radius')))) - except: - return jsonify(to_return) - - delta = dateEnd - dateStart - for i in range(delta.days+1): - correctDatetime = dateStart + datetime.timedelta(days=i) - date_str = str(correctDatetime.year)+str(correctDatetime.month)+str(correctDatetime.day) - keyCateg = 'GEO_RAD' - keyname = "{}:{}".format(keyCateg, date_str) - res = serv_redis_db.georadius(keyname, centerLon, centerLat, radius, unit='km', withcoord=True) - - #sum up really close coord - for data, coord in res: - flag_added = False - coord = [coord[0], coord[1]] - #list all coord - for dicoCoordStr in dico_coord.keys(): - dicoCoord = json.loads(dicoCoordStr) - #if curCoord close to coord - if isCloseTo(dicoCoord, coord): - #add data to dico coord - dico_coord[dicoCoordStr].append(data) - flag_added = True - break - # coord not in dic - if not flag_added: - dico_coord[str(coord)] = [data] - - for dicoCoord, array in dico_coord.items(): - dicoCoord = json.loads(dicoCoord) - to_return.append([array, dicoCoord]) - - return jsonify(to_return) - -@app.route("/_logs") -def logs(): - return Response(event_stream_log(), mimetype="text/event-stream") - -@app.route("/_maps") -def maps(): - return Response(event_stream_maps(), mimetype="text/event-stream") - -@app.route("/_get_log_head") -def getLogHead(): - return json.dumps(LogItem('').get_head_row()) - -def event_stream_log(): - for msg in subscriber_log.listen(): - content = msg['data'] - yield EventMessage(content).to_json() - -def event_stream_maps(): - for msg in subscriber_map.listen(): - content = msg['data'].decode('utf8') - yield 'data: {}\n\n'.format(content) - if __name__ == '__main__': app.run(host='localhost', port=8001, threaded=True) From ac90101fc2273301dc41201bfb2ae778e7eef16a Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Fri, 3 Nov 2017 09:38:11 +0100 Subject: [PATCH 24/65] Same for util function (move into section) --- server.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/server.py b/server.py index a771f41..194100d 100755 --- a/server.py +++ b/server.py @@ -36,6 +36,11 @@ subscriber_map = redis_server_map.pubsub(ignore_subscribe_messages=True) subscriber_map.psubscribe(cfg.get('RedisMap', 'channelDisp')) eventNumber = 0 +########## +## UTIL ## +########## + +''' INDEX ''' class LogItem(): FIELDNAME_ORDER = [] @@ -93,12 +98,7 @@ class EventMessage(): to_ret = { 'log': self.feed, 'feedName': self.feedName, 'zmqName': self.zmqName } return 'data: {}\n\n'.format(json.dumps(to_ret)) -def getZrange(keyCateg, date, topNum): - date_str = str(date.year)+str(date.month)+str(date.day) - keyname = "{}:{}".format(keyCateg, date_str) - data = serv_redis_db.zrange(keyname, 0, 5, desc=True, withscores=True) - data = [ [record[0].decode('utf8'), record[1]] for record in data ] - return data +''' CONTRIB ''' # max lvl is 16 def getRankLevel(points): @@ -106,6 +106,7 @@ def getRankLevel(points): return 0 else: return float("{:.2f}".format(math.log(points, cfg.getfloat('CONTRIB' ,'rankMultiplier')))) + def getRemainingPoints(points): prev = 0 for i in [math.floor(cfg.getfloat('CONTRIB' ,'rankMultiplier')**x) for x in range(1,17)]: @@ -114,6 +115,16 @@ def getRemainingPoints(points): prev = i return { 'remainingPts': 0, 'stepPts': cfg.getfloat('CONTRIB' ,'rankMultiplier')**16 } + +''' GENERAL ''' + +def getZrange(keyCateg, date, topNum): + date_str = str(date.year)+str(date.month).zfill(2)+str(date.day).zfill(2) + keyname = "{}:{}".format(keyCateg, date_str) + data = serv_redis_db.zrange(keyname, 0, 5, desc=True, withscores=True) + data = [ [record[0].decode('utf8'), record[1]] for record in data ] + return data + ########### ## ROUTE ## ########### From 9f6638bf293c3c7b49cf9e67389345bc928cd805 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Fri, 3 Nov 2017 09:42:05 +0100 Subject: [PATCH 25/65] Fixed small bug related to dateformat --- server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server.py b/server.py index 194100d..0a17cf5 100755 --- a/server.py +++ b/server.py @@ -243,7 +243,7 @@ def getCoordsByRadius(): delta = dateEnd - dateStart for i in range(delta.days+1): correctDatetime = dateStart + datetime.timedelta(days=i) - date_str = str(correctDatetime.year)+str(correctDatetime.month)+str(correctDatetime.day) + date_str = str(correctDatetime.year)+str(correctDatetime.month).zfill(2)+str(correctDatetime.day).zfill(2) keyCateg = 'GEO_RAD' keyname = "{}:{}".format(keyCateg, date_str) res = serv_redis_db.georadius(keyname, centerLon, centerLat, radius, unit='km', withcoord=True) From 264a4e52e2d7d334a9c2cab3f4d83201fa95a2fb Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Mon, 6 Nov 2017 09:50:28 +0100 Subject: [PATCH 26/65] Draft getting contribution info from redis --- server.py | 228 +++++++++++++++++++++++++++++++++-------- static/js/contrib.js | 8 +- templates/contrib.html | 10 +- zmq_subscriber.py | 14 ++- 4 files changed, 207 insertions(+), 53 deletions(-) diff --git a/server.py b/server.py index 0a17cf5..1903fdf 100755 --- a/server.py +++ b/server.py @@ -29,6 +29,7 @@ serv_redis_db = redis.StrictRedis( db=cfg.getint('RedisDB', 'db')) categories_in_datatable = json.loads(cfg.get('CONTRIB', 'categories_in_datatable')) +MAX_NUMBER_OF_LAST_CONTRIBUTOR = cfg.getint('CONTRIB', 'max_number_of_last_contributor') subscriber_log = redis_server_log.pubsub(ignore_subscribe_messages=True) subscriber_log.psubscribe(cfg.get('RedisLog', 'channel')) @@ -104,9 +105,14 @@ class EventMessage(): def getRankLevel(points): if points == 0: return 0 + elif points == 1: + return 1 else: return float("{:.2f}".format(math.log(points, cfg.getfloat('CONTRIB' ,'rankMultiplier')))) +def getTrueRank(ptns): + return int(getRankLevel(ptns)) + def getRemainingPoints(points): prev = 0 for i in [math.floor(cfg.getfloat('CONTRIB' ,'rankMultiplier')**x) for x in range(1,17)]: @@ -117,14 +123,99 @@ def getRemainingPoints(points): ''' GENERAL ''' +def getMonthSpan(date): + ds = datetime.datetime(date.year, date.month, 1) + dyear = 1 if ds.month+1 > 12 else 0 + dmonth = -12 if ds.month+1 > 12 else 0 + de = datetime.datetime(ds.year + dyear, ds.month+1 + dmonth, 1) -def getZrange(keyCateg, date, topNum): - date_str = str(date.year)+str(date.month).zfill(2)+str(date.day).zfill(2) - keyname = "{}:{}".format(keyCateg, date_str) - data = serv_redis_db.zrange(keyname, 0, 5, desc=True, withscores=True) + delta = de - ds + to_return = [] + for i in range(delta.days): + to_return.append(ds + datetime.timedelta(days=i)) + return to_return + +def getDateStrFormat(date): + return str(date.year)+str(date.month).zfill(2)+str(date.day).zfill(2) + +def getZrange(keyCateg, date, topNum, endSubkey=""): + date_str = getDateStrFormat(date) + keyname = "{}:{}{}".format(keyCateg, date_str, endSubkey) + data = serv_redis_db.zrange(keyname, 0, topNum-1, desc=True, withscores=True) data = [ [record[0].decode('utf8'), record[1]] for record in data ] return data +def getOrgPntFromRedis(org, date): + keyCateg = 'CONTRIB_DAY' + scoreSum = 0 + for curDate in getMonthSpan(date): + date_str = getDateStrFormat(curDate) + keyname = "{}:{}".format(keyCateg, date_str) + data = serv_redis_db.zscore(keyname, org) + if data is None: + data = 0 + scoreSum += data + return scoreSum + +def getOrgRankFromRedis(org, date): + ptns = getOrgPntFromRedis(org, date) + return getTrueRank(ptns) + +def getOrgLogoFromRedis(org): + return 'logo_'+org + +def getTopContributor_fromRedis(date): + data2 = [ + { + 'rank': random.randint(1,16), + 'logo_path': 'logo1', + 'org': 'CIRCL', + }, + { + 'rank': random.randint(1,16), + 'logo_path': 'logo2', + 'org': 'CASES', + }, + { + 'rank': random.randint(1,16), + 'logo_path': 'logo3', + 'org': 'SMILE', + }, + { + 'rank': random.randint(1,16), + 'logo_path': 'logo4', + 'org': 'ORG4', + }, + { + 'rank': random.randint(1,16), + 'logo_path': 'logo5', + 'org': 'ORG5', + }, + ] + + orgDicoPnts = {} + for curDate in getMonthSpan(date): + keyCateg = "CONTRIB_DAY" + topNum = 0 # all + contrib_org = getZrange(keyCateg, curDate, topNum) + for org, pnts in contrib_org: + if org not in orgDicoPnts: + orgDicoPnts[org] = 0 + orgDicoPnts[org] += pnts + + data = [] + for org, pnts in orgDicoPnts.items(): + dic = {} + dic['rank'] = getTrueRank(pnts) + dic['logo_path'] = getOrgLogoFromRedis(org) + dic['org'] = org + dic['pnts'] = pnts + data.append(dic) + data.sort(key=lambda x: x['pnts'], reverse=True) + + return data + #return data2 + ########### ## ROUTE ## ########### @@ -215,7 +306,7 @@ def getHitMap(): except: date = datetime.datetime.now() keyCateg = "GEO_COUNTRY" - topNum = -1 # default Num + topNum = 0 # all data = getZrange(keyCateg, date, topNum) return jsonify(data) @@ -243,7 +334,7 @@ def getCoordsByRadius(): delta = dateEnd - dateStart for i in range(delta.days+1): correctDatetime = dateStart + datetime.timedelta(days=i) - date_str = str(correctDatetime.year)+str(correctDatetime.month).zfill(2)+str(correctDatetime.day).zfill(2) + date_str = getDateStrFormat(correctDatetime) keyCateg = 'GEO_RAD' keyname = "{}:{}".format(keyCateg, date_str) res = serv_redis_db.georadius(keyname, centerLon, centerLat, radius, unit='km', withcoord=True) @@ -275,7 +366,7 @@ def getCoordsByRadius(): @app.route("/_getLastContributor") def getLastContributor(): - data = [ + data2 = [ { 'rank': random.randint(1,16), 'logo_path': 'logo1', @@ -302,48 +393,77 @@ def getLastContributor(): 'org': 'ORG5', }, ] - return jsonify(data*2) + try: + date = datetime.datetime.fromtimestamp(float(request.args.get('date'))) + except: + date = datetime.datetime.now() + keyCateg = "CONTRIB_LAST" + topNum = MAX_NUMBER_OF_LAST_CONTRIBUTOR # default Num + last_contrib_org = getZrange(keyCateg, date, topNum) + data = [] + for org, sec in last_contrib_org: + dic = {} + dic['rank'] = getOrgRankFromRedis(org, datetime.datetime.now()) + dic['logo_path'] = getOrgLogoFromRedis(org) + dic['org'] = org + dic['pnts'] = getOrgPntFromRedis(org, date) + data.append(dic) + return jsonify(data) + #return jsonify(data2*2) @app.route("/_getTopContributor") -def getTopContributor(): - data = [ - { - 'rank': random.randint(1,16), - 'logo_path': 'logo1', - 'org': 'CIRCL', - }, - { - 'rank': random.randint(1,16), - 'logo_path': 'logo2', - 'org': 'CASES', - }, - { - 'rank': random.randint(1,16), - 'logo_path': 'logo3', - 'org': 'SMILE', - }, - { - 'rank': random.randint(1,16), - 'logo_path': 'logo4', - 'org': 'ORG4', - }, - { - 'rank': random.randint(1,16), - 'logo_path': 'logo5', - 'org': 'ORG5', - }, - ] - return jsonify(data*2) +def getTopContributor(suppliedDate=None): + if suppliedDate is None: + try: + date = datetime.datetime.fromtimestamp(float(request.args.get('date'))) + except: + date = datetime.datetime.now() + else: + date = suppliedDate + + data = getTopContributor_fromRedis(date) + return jsonify(data) + +@app.route("/_getFameContributor") +def getFameContributor(): + try: + date = datetime.datetime.fromtimestamp(float(request.args.get('date'))) + except: + today = datetime.datetime.now() + # get previous month + date = (datetime.datetime(today.year, today.month, 1) - datetime.timedelta(days=1)) + return getTopContributor(date) + @app.route("/_getTop5Overtime") def getTop5Overtime(): - data = [{'label': 'CIRCL', 'data': [[0, 4], [1, 7], [2,14]]}, {'label': 'CASES', 'data': [[0, 1], [1, 5], [2,2]]}] + data2 = [ + {'label': 'CIRCL', 'data': [[0, 4], [1, 7], [2,14]]}, + {'label': 'CASES', 'data': [[0, 1], [1, 5], [2,2]]} + ] + data = [] + today = datetime.datetime.now() + topSortedOrg = getTopContributor_fromRedis(today) #Get current top + # show current top 5 org points overtime (last 5 days) + for dic in topSortedOrg[0:5]: + org = dic['org'] + overtime = [] + for deltaD in range(1,6,1): + date = (datetime.datetime(today.year, today.month, today.day) - datetime.timedelta(days=deltaD)) + keyname = 'CONTRIB_DAY:'+getDateStrFormat(date) + org_score = serv_redis_db.zscore(keyname, org) + if org_score is None: + org_score = 0 + overtime.append([deltaD, org_score]) + to_append = {'label': org, 'data': overtime} + data.append(to_append) return jsonify(data) + #return jsonify(data2) @app.route("/_getCategPerContrib") def getCategPerContrib(): - data = [] + data2 = [] for d in range(15): dic = {} dic['rank'] = random.randint(1,16) @@ -351,14 +471,40 @@ def getCategPerContrib(): dic['org'] = 'Org'+str(d) for f in categories_in_datatable: dic[f] = random.randint(0,1600) + data2.append(dic) + + try: + date = datetime.datetime.fromtimestamp(float(request.args.get('date'))) + except: + date = datetime.datetime.now() + keyCateg = "CONTRIB_DAY" + topNum = 0 # all + contrib_org = getZrange(keyCateg, date, topNum) + data = [] + for org, pnts in contrib_org: + dic = {} + dic['rank'] = getTrueRank(pnts) + dic['logo_path'] = getOrgLogoFromRedis(org) + dic['org'] = org + dic['pnts'] = pnts + for categ in categories_in_datatable: + keyname = 'CONTRIB_CATEG:'+getDateStrFormat(date)+':'+categ + categ_score = serv_redis_db.zscore(keyname, org) + if categ_score is None: + categ_score = 0 + dic[categ] = categ_score data.append(dic) return jsonify(data) + return jsonify(data2) @app.route("/_getAllOrg") def getAllOrg(): - data = ['CIRCL', 'CASES', 'SMILE' ,'ORG4' ,'ORG5', 'SUPER HYPER LONG ORGINZATION NAME', 'Org3'] + data = serv_redis_db.smembers('CONTRIB_ALL_ORG') + data = [x.decode('utf8') for x in data] + data2 = ['CIRCL', 'CASES', 'SMILE' ,'ORG4' ,'ORG5', 'SUPER HYPER LONG ORGINZATION NAME', 'Org3'] return jsonify(data) + #return jsonify(data2) @app.route("/_getOrgRank") def getOrgRank(): @@ -366,7 +512,9 @@ def getOrgRank(): org = request.args.get('org') except: org = '' + date = datetime.datetime.now() points = random.randint(1,math.floor(cfg.getfloat('CONTRIB' ,'rankMultiplier')**16)) + points = getOrgPntFromRedis(org, date) #FIXME put 0 if org has no points remainingPts = getRemainingPoints(points) data = {'org': org, diff --git a/static/js/contrib.js b/static/js/contrib.js index 123766e..38bbfb4 100644 --- a/static/js/contrib.js +++ b/static/js/contrib.js @@ -197,7 +197,7 @@ function addToTableFromJson(datatable, url) { var row = data[i]; i = parseInt(i); var to_add = [ - i+1, + row.pnts, getRankIcon(row.rank), row.logo_path, row.org @@ -211,7 +211,6 @@ function addToTableFromJson(datatable, url) { function updateProgressHeader(org) { // get Org rank $.getJSON( url_getOrgRank+'?org='+org, function( data ) { - console.log(data); datatableTop.draw(); var rank = Math.floor(data.rank); var rankDec = data.rank-rank; @@ -262,9 +261,9 @@ $(document).ready(function() { // top contributors addToTableFromJson(datatableTop, url_getTopContributor); // hall of fame - addToTableFromJson(datatableFame, url_getTopContributor); + addToTableFromJson(datatableFame, url_getFameContributor); // last contributors - addToTableFromJson(datatableLast, url_getTopContributor); + addToTableFromJson(datatableLast, url_getLastContributor); // category per contributors $.getJSON( url_getCategPerContrib, function( data ) { for (i in data) { @@ -286,6 +285,7 @@ $(document).ready(function() { }); // top 5 contrib overtime $.getJSON( url_getTop5Overtime, function( data ) { + console.log(data); var plotLineChart = $.plot("#divTop5Overtime", data, optionsLineChart); }); }); diff --git a/templates/contrib.html b/templates/contrib.html index 1dcb8e6..ad36e6b 100644 --- a/templates/contrib.html +++ b/templates/contrib.html @@ -99,7 +99,7 @@
    #prev. rank Logo Organization name
    # Org rank LogoOrganization nameNetwork ActivityPayload deliveryOthers{{ categ }}
    #Org rankRank LogoOrganization nameOrganisation
    #Org rankRank LogoOrg nameOrganisation
    #Org rankRank LogoOrganisation{{ categ }}{{ categ }}
    #prev. rankPrev. rank LogoOrganization nameOrganisation
    - + @@ -126,7 +126,7 @@
    #Points Rank Logo Organisation
    - + @@ -152,7 +152,7 @@
    #Points Rank Logo Organisation
    - + @@ -187,7 +187,7 @@
    #Points Rank Logo Organisation
    - + @@ -229,8 +229,10 @@ From a337ee1a853541eae66ad1693327811b8a0beab4 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Tue, 7 Nov 2017 12:59:32 +0100 Subject: [PATCH 40/65] Added pic centering in lastDataTable --- static/js/contrib.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/static/js/contrib.js b/static/js/contrib.js index 67935f8..c8dd1c5 100644 --- a/static/js/contrib.js +++ b/static/js/contrib.js @@ -57,6 +57,8 @@ var optionDatatable_light = { var optionDatatable_top = jQuery.extend({}, optionDatatable_light) var optionDatatable_last = jQuery.extend({}, optionDatatable_light) optionDatatable_last.columnDefs = [ + { className: "centerCellPicOrgRank", "targets": [ 2 ] }, + { className: "centerCellPicOrgLogo", "targets": [ 3 ] }, { 'orderData':[5], 'targets': [0] }, { 'targets': [5], From 90a518cc071cc68e7fa89fbdc9829a5d27adbb0c Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Wed, 8 Nov 2017 11:29:07 +0100 Subject: [PATCH 41/65] Added support of honorBadge + Added text related to global ranking requirements --- contributor_helper.py | 31 +++- server.py | 13 +- static/css/ranking.css | 13 +- static/js/contrib.js | 43 ++++-- static/pics/MISPHonorableIcons/1.svg | 178 ++++++++++++++++++++++ static/pics/MISPHonorableIcons/2.svg | 185 +++++++++++++++++++++++ static/pics/rankingMISPOrg/14.svg | 86 +++++------ static/pics/rankingMISPOrg/15.svg | 218 +++++++++------------------ static/pics/rankingMISPOrg/16.svg | 91 +++++++---- templates/contrib.html | 63 ++++++-- zmq_subscriber.py | 50 +++++- 11 files changed, 713 insertions(+), 258 deletions(-) create mode 100644 static/pics/MISPHonorableIcons/1.svg create mode 100644 static/pics/MISPHonorableIcons/2.svg diff --git a/contributor_helper.py b/contributor_helper.py index 530c4a7..6252302 100644 --- a/contributor_helper.py +++ b/contributor_helper.py @@ -10,16 +10,32 @@ class Contributor_helper: self.serv_redis_db = serv_redis_db self.cfg = cfg self.cfg_org_rank = configparser.ConfigParser() - self.cfg_org_rank.read(os.path.join(os.environ['DASH_CONFIG'], 'rankTitle.cfg')) + self.cfg_org_rank.read(os.path.join(os.environ['DASH_CONFIG'], 'ranking.cfg')) + #honorBadge + self.org_honor_badge_title = {} + for badgeNum in range(1, len(self.cfg_org_rank.options('HonorBadge'))+1): #get Num of honorBadge + self.org_honor_badge_title[badgeNum] = self.cfg_org_rank.get('HonorBadge', str(badgeNum)) + + #GLOBAL RANKING self.org_rank_maxLevel = self.cfg_org_rank.getint('rankTitle', 'maxLevel') self.org_rank = {} for rank in range(1, self.org_rank_maxLevel+1): self.org_rank[rank] = self.cfg_org_rank.get('rankTitle', str(rank)) + self.org_rank_requirement_pnts = {} + for rank in range(1, self.org_rank_maxLevel+1): + self.org_rank_requirement_pnts[rank] = self.cfg_org_rank.getint('rankRequirementsPnts', str(rank)) + self.org_rank_requirement_text = {} + for rank in range(1, self.org_rank_maxLevel+1): + self.org_rank_requirement_text[rank] = self.cfg_org_rank.get('rankRequirementsText', str(rank)) + self.org_rank_additional_info = json.loads(self.cfg_org_rank.get('additionalInfo', 'textsArray')) + #WEB STUFF self.misp_web_url = cfg.get('RedisGlobal', 'misp_web_url') self.MAX_NUMBER_OF_LAST_CONTRIBUTOR = cfg.getint('CONTRIB', 'max_number_of_last_contributor') self.categories_in_datatable = json.loads(cfg.get('CONTRIB', 'categories_in_datatable')) + + #MONTHLY RANKING self.default_pnts_per_contribution = json.loads(cfg.get('CONTRIB', 'default_pnts_per_contribution')) temp = json.loads(cfg.get('CONTRIB', 'pnts_per_contribution')) self.DICO_PNTS_REWARD = {} @@ -196,6 +212,7 @@ class Contributor_helper: dic = {} dic['rank'] = random.randint(1,self.levelMax) dic['orgRank'] = random.randint(1,self.levelMax), + dic['honorBadge'] = [random.randint(1,2)], dic['logo_path'] = 'logo' dic['org'] = 'Org'+str(d) dic['pnts'] = random.randint(1,2**self.levelMax) @@ -216,6 +233,7 @@ class Contributor_helper: { 'rank': random.randint(1,self.levelMax), 'orgRank': random.randint(1,self.levelMax), + 'honorBadge': [1,2], 'logo_path': self.getOrgLogoFromRedis('MISP'), 'org': 'MISP', 'pnts': random.randint(1,2**self.levelMax) @@ -223,6 +241,7 @@ class Contributor_helper: { 'rank': random.randint(1,self.levelMax), 'orgRank': random.randint(1,self.levelMax), + 'honorBadge': [1], 'logo_path': 'logo1', 'org': 'CIRCL', 'pnts': random.randint(1,2**self.levelMax) @@ -230,6 +249,7 @@ class Contributor_helper: { 'rank': random.randint(1,self.levelMax), 'orgRank': random.randint(1,self.levelMax), + 'honorBadge': [2], 'logo_path': 'logo2', 'org': 'CASES', 'pnts': random.randint(1,2**self.levelMax) @@ -237,6 +257,7 @@ class Contributor_helper: { 'rank': random.randint(1,self.levelMax), 'orgRank': random.randint(1,self.levelMax), + 'honorBadge': [], 'logo_path': 'logo3', 'org': 'SMILE', 'pnts': random.randint(1,2**self.levelMax) @@ -244,6 +265,7 @@ class Contributor_helper: { 'rank': random.randint(1,self.levelMax), 'orgRank': random.randint(1,self.levelMax), + 'honorBadge': [], 'logo_path': 'logo4', 'org': 'ORG4', 'pnts': random.randint(1,2**self.levelMax) @@ -251,6 +273,7 @@ class Contributor_helper: { 'rank': random.randint(1,self.levelMax), 'orgRank': random.randint(1,self.levelMax), + 'honorBadge': [], 'logo_path': 'logo5', 'org': 'ORG5', 'pnts': random.randint(1,2**self.levelMax) @@ -264,6 +287,7 @@ class Contributor_helper: { 'rank': random.randint(1,self.levelMax), 'orgRank': random.randint(1,self.levelMax), + 'honorBadge': [1,2], 'logo_path': self.getOrgLogoFromRedis('MISP'), 'org': 'MISP', 'pnts': random.randint(1,2**self.levelMax), @@ -272,6 +296,7 @@ class Contributor_helper: { 'rank': random.randint(1,self.levelMax), 'orgRank': random.randint(1,self.levelMax), + 'honorBadge': [1], 'logo_path': 'logo1', 'org': 'CIRCL', 'pnts': random.randint(1,2**self.levelMax), @@ -280,6 +305,7 @@ class Contributor_helper: { 'rank': random.randint(1,self.levelMax), 'orgRank': random.randint(1,self.levelMax), + 'honorBadge': [2], 'logo_path': 'logo2', 'org': 'CASES', 'pnts': random.randint(1,2**self.levelMax), @@ -288,6 +314,7 @@ class Contributor_helper: { 'rank': random.randint(1,self.levelMax), 'orgRank': random.randint(1,self.levelMax), + 'honorBadge': [], 'logo_path': 'logo3', 'org': 'SMILE', 'pnts': random.randint(1,2**self.levelMax), @@ -296,6 +323,7 @@ class Contributor_helper: { 'rank': random.randint(1,self.levelMax), 'orgRank': random.randint(1,self.levelMax), + 'honorBadge': [], 'logo_path': 'logo4', 'org': 'ORG4', 'pnts': random.randint(1,2**self.levelMax), @@ -304,6 +332,7 @@ class Contributor_helper: { 'rank': random.randint(1,self.levelMax), 'orgRank': random.randint(1,self.levelMax), + 'honorBadge': [], 'logo_path': 'logo5', 'org': 'ORG5', 'pnts': random.randint(1,2**self.levelMax), diff --git a/server.py b/server.py index 00a867d..0cc30ab 100755 --- a/server.py +++ b/server.py @@ -151,9 +151,17 @@ def contrib(): categ_list = contributor_helper.categories_in_datatable categ_list_str = [ s[0].upper() + s[1:].replace('_', ' ') for s in contributor_helper.categories_in_datatable] categ_list_points = [contributor_helper.DICO_PNTS_REWARD[categ] for categ in categ_list] + org_rank = contributor_helper.org_rank - org_rank_list = [[rank, title] for rank, title in org_rank.items()] + org_rank_requirement_pnts = contributor_helper.org_rank_requirement_pnts + org_rank_requirement_text = contributor_helper.org_rank_requirement_text + org_rank_list = [[rank, title, org_rank_requirement_pnts[rank], org_rank_requirement_text[rank]] for rank, title in org_rank.items()] org_rank_list.sort(key=lambda x: x[0]) + org_rank_additional_text = contributor_helper.org_rank_additional_info + + org_honor_badge_title = contributor_helper.org_honor_badge_title + org_honor_badge_title_list = [ [num, text] for num, text in contributor_helper.org_honor_badge_title.items()] + org_honor_badge_title_list.sort(key=lambda x: x[0]) currOrg = request.args.get('org') if currOrg is None: @@ -168,6 +176,9 @@ def contrib(): categ_list_points=categ_list_points, org_rank_json=json.dumps(org_rank), org_rank_list=org_rank_list, + org_rank_additional_text=org_rank_additional_text, + org_honor_badge_title=json.dumps(org_honor_badge_title), + org_honor_badge_title_list=org_honor_badge_title_list, min_between_reload=cfg.getint('CONTRIB', 'min_between_reload') ) diff --git a/static/css/ranking.css b/static/css/ranking.css index f29aac4..24f5d8d 100644 --- a/static/css/ranking.css +++ b/static/css/ranking.css @@ -3,6 +3,15 @@ height: auto; } +.circleBadge { + width: 80px; + height: 80px; + text-align: center; + border-radius: 65px; + background-color: #e1e1e1; + border: 1px solid #caccce; +} + .questionBadgeText { left: -15%; position: relative; @@ -27,8 +36,8 @@ left: 1%; } -.infoBlue { - background-color: #d1d1d1 !important; +.selectedOrgInTable { + background-color: #fff6c2 !important; color: black; } diff --git a/static/js/contrib.js b/static/js/contrib.js index c8dd1c5..e29cdae 100644 --- a/static/js/contrib.js +++ b/static/js/contrib.js @@ -51,7 +51,8 @@ var optionDatatable_light = { "info": false, "columnDefs": [ { className: "centerCellPicOrgRank", "targets": [ 2 ] }, - { className: "centerCellPicOrgLogo", "targets": [ 3 ] } + { className: "centerCellPicOrgLogo", "targets": [ 3 ] }, + { className: "centerCellPicOrgLogo", "targets": [ 4 ] } ] }; var optionDatatable_top = jQuery.extend({}, optionDatatable_light) @@ -59,9 +60,10 @@ var optionDatatable_last = jQuery.extend({}, optionDatatable_light) optionDatatable_last.columnDefs = [ { className: "centerCellPicOrgRank", "targets": [ 2 ] }, { className: "centerCellPicOrgLogo", "targets": [ 3 ] }, - { 'orderData':[5], 'targets': [0] }, + { className: "centerCellPicOrgLogo", "targets": [ 4 ] }, + { 'orderData':[6], 'targets': [0] }, { - 'targets': [5], + 'targets': [6], 'visible': false, 'searchable': false }, @@ -79,7 +81,8 @@ var optionDatatable_Categ = { "info": false, "columnDefs": [ { className: "centerCellPicOrgRank", "targets": [ 2 ] }, - { className: "centerCellPicOrgLogo", "targets": [ 3 ], 'searchable': false, 'sortable': false } + { className: "centerCellPicOrgLogo", "targets": [ 3 ], 'searchable': false, 'sortable': false }, + { className: "centerCellPicOrgLogo", "targets": [ 4 ]} ] }; @@ -144,16 +147,30 @@ function getOrgRankIcon(rank, size) { return obj.outerHTML; } -function createImg(source, size, return_obj) { +function createImg(source, size) { var obj = document.createElement('object'); obj.height = size; obj.width = size; obj.style.margin = 'auto'; obj.data = source; - obj.type = "image/png" return obj.outerHTML; } +function createHonorImg(array, size) { + size = 32; + var div = document.createElement('div'); + for (badgeNum of array) { + var obj = document.createElement('img'); + obj.height = size; + obj.width = size; + obj.style.margin = 'auto'; + obj.title = org_honor_badge_title[badgeNum]; + obj.src = url_baseHonorLogo+badgeNum+'.svg'; + div.appendChild(obj); + } + return div.outerHTML; +} + function generateRankingSheet(rank, rankDec, stepPnt, pnt, Rpnt) { var Cpnt = pnt - stepPnt; var Tpnt = Cpnt + Rpnt; @@ -241,7 +258,6 @@ function generateRankingSheet(rank, rankDec, stepPnt, pnt, Rpnt) { function addToTableFromJson(datatable, url) { $.getJSON( url, function( data ) { - console.log(data); for (i in data) { var row = data[i]; i = parseInt(i); @@ -249,6 +265,7 @@ function addToTableFromJson(datatable, url) { row.pnts, getMonthlyRankIcon(row.rank), getOrgRankIcon(row.orgRank, 60), + createHonorImg(row.honorBadge, 20), createImg(row.logo_path, 32), row.org ]; @@ -274,6 +291,7 @@ function addLastContributor(datatable, data, update) { data.pnts, getMonthlyRankIcon(data.rank), getOrgRankIcon(data.orgRank, 60), + createHonorImg(data.honorBadge, 20), createImg(data.logo_path, 32), data.org, data.epoch @@ -309,19 +327,19 @@ function updateProgressHeader(org) { // update color in other dataTables datatableTop.rows().every( function() { var row = this.node(); - if(this.data()[3] == data.org) { row.classList.add('infoBlue'); } else { row.classList.remove('infoBlue'); } + if(this.data()[5] == data.org) { row.classList.add('selectedOrgInTable'); } else { row.classList.remove('selectedOrgInTable'); } }); datatableFame.rows().every( function() { var row = this.node(); - if(this.data()[3] == data.org) { row.classList.add('infoBlue'); } else { row.classList.remove('infoBlue'); } + if(this.data()[5] == data.org) { row.classList.add('selectedOrgInTable'); } else { row.classList.remove('selectedOrgInTable'); } }); datatableCateg.rows().every( function() { var row = this.node(); - if(this.data()[3] == data.org) { row.classList.add('infoBlue'); } else { row.classList.remove('infoBlue'); } + if(this.data()[5] == data.org) { row.classList.add('selectedOrgInTable'); } else { row.classList.remove('selectedOrgInTable'); } }); datatableLast.rows().every( function() { var row = this.node(); - if(this.data()[3] == data.org) { row.classList.add('infoBlue'); } else { row.classList.remove('infoBlue'); } + if(this.data()[5] == data.org) { row.classList.add('selectedOrgInTable'); } else { row.classList.remove('selectedOrgInTable'); } }); }); } @@ -381,7 +399,8 @@ $(document).ready(function() { row.pnts, getMonthlyRankIcon(row.rank), getOrgRankIcon(row.orgRank, 44), - row.logo_path, + createHonorImg(row.honorBadge, 20), + createImg(row.logo_path, 32), row.org, ]; for (categ of categ_list) { diff --git a/static/pics/MISPHonorableIcons/1.svg b/static/pics/MISPHonorableIcons/1.svg new file mode 100644 index 0000000..16de876 --- /dev/null +++ b/static/pics/MISPHonorableIcons/1.svg @@ -0,0 +1,178 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + diff --git a/static/pics/MISPHonorableIcons/2.svg b/static/pics/MISPHonorableIcons/2.svg new file mode 100644 index 0000000..65bcf30 --- /dev/null +++ b/static/pics/MISPHonorableIcons/2.svg @@ -0,0 +1,185 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + diff --git a/static/pics/rankingMISPOrg/14.svg b/static/pics/rankingMISPOrg/14.svg index 3051dc1..abcfb78 100644 --- a/static/pics/rankingMISPOrg/14.svg +++ b/static/pics/rankingMISPOrg/14.svg @@ -15,7 +15,7 @@ version="1.1" id="svg8" inkscape:version="0.92.1 r15371" - sodipodi:docname="14-General.svg"> + sodipodi:docname="14.svg"> - - - - - - - - - - - + + + + + + + diff --git a/static/pics/rankingMISPOrg/15.svg b/static/pics/rankingMISPOrg/15.svg index f2782eb..ff4f422 100644 --- a/static/pics/rankingMISPOrg/15.svg +++ b/static/pics/rankingMISPOrg/15.svg @@ -15,7 +15,7 @@ version="1.1" id="svg8" inkscape:version="0.92.1 r15371" - sodipodi:docname="15-Marshal.svg"> + sodipodi:docname="15.svg"> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + diff --git a/static/pics/rankingMISPOrg/16.svg b/static/pics/rankingMISPOrg/16.svg index 4ace1cb..670e901 100644 --- a/static/pics/rankingMISPOrg/16.svg +++ b/static/pics/rankingMISPOrg/16.svg @@ -15,7 +15,7 @@ version="1.1" id="svg8" inkscape:version="0.92.1 r15371" - sodipodi:docname="16-Grand General.svg"> + sodipodi:docname="16.svg"> - - - - - - + id="g3760" + transform="matrix(0.26458333,0,0,0.26458333,0.25014905,248.81803)"> + + + + + + + + + + + + + diff --git a/templates/contrib.html b/templates/contrib.html index dbc3c5a..7e56ca2 100644 --- a/templates/contrib.html +++ b/templates/contrib.html @@ -46,23 +46,24 @@
    #Points Prev. rank Logo Organisation
    @@ -83,26 +84,53 @@
    -
    -

    Organisation rank title:

    +
    +

    Global organisation ranking:

    + + {% for item in org_rank_list %} - + + + {% endfor %}
    Org. rank TitlePoints requirementContribution requirement
    {{ item[1] }}{{ item[2] }}{{ item[3] }}
    +
    +
      + {% for text in org_rank_additional_text %} +
    • {{ text }}
      • + {% endfor %} +
    +
    +
    + + + {% for item in org_honor_badge_title_list %} + + + + + {% endfor %} + +
    +
    + +
    +     		{{ item[1] }}
    +
    @@ -166,7 +194,7 @@
    -
    +
    Contributor Ranking @@ -181,6 +209,7 @@ Cur. rank Org. rank + Organisation @@ -194,7 +223,7 @@
    -
    +
    Last contributors @@ -209,6 +238,7 @@ Cur. rank Org. rank + Organisation epoch @@ -237,6 +267,7 @@ Cur. rank Org. rank + Organisation {% for categ in categ_list_str %} {{ categ }} @@ -273,6 +304,7 @@ Prev. rank Org. rank + Organisation @@ -323,6 +355,8 @@ url_baseRankMonthlyLogo = url_baseRankMonthlyLogo.substring(0, url_baseRankMonthlyLogo.length-5); var url_baseOrgRankLogo = "{{ url_for('static', filename='pics/rankingMISPOrg/1.svg') }}"; url_baseOrgRankLogo = url_baseOrgRankLogo.substring(0, url_baseOrgRankLogo.length-5); + var url_baseHonorLogo = "{{ url_for('static', filename='pics/MISPHonorableIcons/1.svg') }}"; + url_baseHonorLogo = url_baseHonorLogo.substring(0, url_baseHonorLogo.length-5); /* DATA FROM CONF */ var currOrg = "{{ currOrg }}"; @@ -330,6 +364,7 @@ var rankMultiplier = {{ rankMultiplier }}; var categ_list = JSON.parse('{{ categ_list|safe }}'); var org_rank_obj = JSON.parse('{{ org_rank_json|safe }}'); + var org_honor_badge_title = JSON.parse('{{ org_honor_badge_title|safe }}'); diff --git a/zmq_subscriber.py b/zmq_subscriber.py index 1421ca6..316461d 100755 --- a/zmq_subscriber.py +++ b/zmq_subscriber.py @@ -18,6 +18,7 @@ configfile = os.path.join(os.environ['DASH_CONFIG'], 'config.cfg') cfg = configparser.ConfigParser() cfg.read(configfile) +ONE_DAY = 60*60*24 ZMQ_URL = cfg.get('RedisGlobal', 'zmq_url') CHANNEL = cfg.get('RedisLog', 'channel') CHANNEL_LASTCONTRIB = cfg.get('RedisLog', 'channelLastContributor') @@ -142,10 +143,57 @@ def handleContribution(zmq_name, org, categ, action, pntMultiplier=1): now = datetime.datetime.now() nowSec = int(time.time()) serv_redis_db.zadd('CONTRIB_LAST:'+getDateStrFormat(now), nowSec, org) - serv_redis_db.expire('CONTRIB_LAST:'+getDateStrFormat(now), 60*60*24) #expire after 1 day + serv_redis_db.expire('CONTRIB_LAST:'+getDateStrFormat(now), ONE_DAY) #expire after 1 day + + updateOrgRank(org, pnts_to_add, eventTime, eventClassification) publish_log(zmq_name, 'CONTRIBUTION', {'org': org, 'categ': categ, 'action': action, 'epoch': nowSec }, channel=CHANNEL_LASTCONTRIB) +def updateOrgRank(orgName, pnts_to_add, contribType, eventTime, isClassified): + keyname = 'CONTRIB_ORG:{org}:{orgCateg}' + #update total points + serv_redis_db.set(keyname.format(org=orgName, orgCateg='points'), pnts_to_add) + #update contribution Requirement + heavilyCount = 10 + recentDays = 31 + regularlyDays = 7 + isRecent = True if (datetime.datetime.now() - eventTime).days > recentDays + contrib = [] #[[contrib_level, contrib_ttl], [], ...] + if contribType == 'sighting': + #[contrib_level, contrib_ttl] + contrib.append([1, ONE_DAY*365]]) + if contribType == 'attribute' or contribType == 'object': + contrib.append([2, ONE_DAY*365]) + if contribType == 'proposal' or contribType == 'discussion': + contrib.append([3, ONE_DAY*365]) + if contribType == 'sighting' and isRecent: + contrib.append([4, ONE_DAY*recentDays]) + if contribType == 'proposal' and isRecent: + contrib.append([5, ONE_DAY*recentDays]) + if contribType == 'event': + contrib.append([6, ONE_DAY*365]) + if contribType == 'event': + contrib.append([7, ONE_DAY*recentDays]) + if contribType == 'event': + contrib.append([8, ONE_DAY*regularlyDays]) + if contribType == 'event' and isClassified: + contrib.append([9, ONE_DAY*regularlyDays]) + if contribType == 'sighting' and sightingWeekCount>heavilyCount: + contrib.append([10, ONE_DAY*regularlyDays]) + if (contribType == 'attribute' or contribType == 'object') and attributeWeekCount>heavilyCount: + contrib.append([11, ONE_DAY*regularlyDays]) + if contribType == 'proposal' and proposalWeekCount>heavilyCount: + contrib.append([12, ONE_DAY*regularlyDays]) + if contribType == 'event' and eventWeekCount>heavilyCount: + contrib.append([13, ONE_DAY*regularlyDays]) + if contribType == 'event' and eventWeekCount>heavilyCount and isClassified: + contrib.append([14, ONE_DAY*regularlyDays]) + + for rankReq, ttl: + serv_redis_db.set(keyname.format(org=orgName, orgCateg='CONTRIB_REQ_'+str(rankReq)), 1) + serv_redis_db.expire(keyname.format(org=orgName, orgCateg='CONTRIB_REQ_'+str(i)), ttl) + + ############## ## HANDLERS ## ############## From a9d5ca5657350ebd3b76fb26ffd1808c409d8bdd Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Wed, 8 Nov 2017 11:35:40 +0100 Subject: [PATCH 42/65] Updated configuration --- config/config.cfg.default | 6 ++-- config/ranking.cfg | 65 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 2 deletions(-) create mode 100644 config/ranking.cfg diff --git a/config/config.cfg.default b/config/config.cfg.default index 738b175..8bfee3e 100644 --- a/config/config.cfg.default +++ b/config/config.cfg.default @@ -38,13 +38,15 @@ char_separator=|| [RedisGlobal] host=localhost port=6251 +#misp_web_url = http://192.168.56.50 +misp_web_url = http://localhost +#zmq_url=tcp://192.168.56.50:50000 +zmq_url=tcp://localhost:50000 [RedisLog] db=0 channel=1 channelLastContributor = lastContributor -zmq_url=tcp://localhost:50000 -#zmq_url=tcp://192.168.56.50:50000 [RedisMap] db=1 diff --git a/config/ranking.cfg b/config/ranking.cfg new file mode 100644 index 0000000..32788aa --- /dev/null +++ b/config/ranking.cfg @@ -0,0 +1,65 @@ +[rankTitle] +maxLevel=16 +1=Ensign +2=Junior Lieutenant +3=Second Lieutenant +4=Lieutenant +5=Senior Lieutenant +6=Captain +7=Senior Captain +8=Major +9=Lieutenant Colonel +10=Colonel +11=Senior Colonel +12=Major General +13=Lieutenant General +14=General +15=Marshal +16=Grand General + +[rankRequirementsPnts] +1=2 +2=4 +3=8 +4=16 +5=32 +6=64 +7=128 +8=256 +9=512 +10=1024 +11=2048 +12=4096 +13=8192 +14=16384 +15=32768 +16=65536 + +[rankRequirementsText] +1=Contributing via sighting at least once a year +2=Contributing via attributes or objects to events at least once a year +3=Contributing via proposals or discussions at least once a year + +4=Contributing via sighting to recent events +5=Contributing proposals to recent events + +6=Contributing by creating new events at least once a year +7=Contributing by creating new events at least once a month +8=Regularly creating events +9=Regularly creating events with classification + +10=Heavily contributing via sighting to recent events +11=Heavily adding attributes or objects to recent events +12=Heavily contributing proposals to recent events +13=Heavily posting events +14=Heavily posting events with classification + +15=Heavily contributing quality content (delivered by instance administrators) +16=Honorable contributor (delivered by instance administrators) + +[HonorBadge] +1=Has made at least one pull request on the MISP project +2=Is a donator for the MISP project + +[additionalInfo] +textsArray=["Proposals means either edition, acceptation or rejection", "Recent events means event aged of one month at max", "Regularly means at least one per week" ,"Heavily means at least 10 per week", "Classification means correct tagging"] From 605538ce976c0124fa4588f5bdbb82458d9d8912 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Wed, 8 Nov 2017 15:34:02 +0100 Subject: [PATCH 43/65] Added display of conribution rank statys + acquired badges --- config/ranking.cfg | 5 ++ contributor_helper.py | 119 +++++++++++++++++++++++++++++++++++++++-- server.py | 18 ++++++- static/css/ranking.css | 4 ++ static/js/contrib.js | 35 ++++++++++++ templates/contrib.html | 9 ++-- zmq_subscriber.py | 12 +++-- 7 files changed, 191 insertions(+), 11 deletions(-) diff --git a/config/ranking.cfg b/config/ranking.cfg index 32788aa..b0119fb 100644 --- a/config/ranking.cfg +++ b/config/ranking.cfg @@ -35,6 +35,11 @@ maxLevel=16 15=32768 16=65536 +[rankRequirementsMisc] +heavilyCount=10 +recentDays=31 +regularlyDays=7 + [rankRequirementsText] 1=Contributing via sighting at least once a year 2=Contributing via attributes or objects to events at least once a year diff --git a/contributor_helper.py b/contributor_helper.py index 6252302..ee1cad4 100644 --- a/contributor_helper.py +++ b/contributor_helper.py @@ -13,8 +13,13 @@ class Contributor_helper: self.cfg_org_rank.read(os.path.join(os.environ['DASH_CONFIG'], 'ranking.cfg')) #honorBadge + self.honorBadgeNum = len(self.cfg_org_rank.options('HonorBadge')) + self.heavilyCount = self.cfg_org_rank.getint('rankRequirementsMisc', 'heavilyCount') + self.recentDays = self.cfg_org_rank.getint('rankRequirementsMisc', 'recentDays') + self.regularlyDays = self.cfg_org_rank.getint('rankRequirementsMisc', 'regularlyDays') + self.org_honor_badge_title = {} - for badgeNum in range(1, len(self.cfg_org_rank.options('HonorBadge'))+1): #get Num of honorBadge + for badgeNum in range(1, self.honorBadgeNum+1): #get Num of honorBadge self.org_honor_badge_title[badgeNum] = self.cfg_org_rank.get('HonorBadge', str(badgeNum)) #GLOBAL RANKING @@ -51,6 +56,11 @@ class Contributor_helper: self.rankMultiplier = cfg.getfloat('CONTRIB' ,'rankMultiplier') self.levelMax = 16 + + ''' HELPER ''' + def getOrgLogoFromRedis(self, org): + return "{}/img/orgs/{}.png".format(self.misp_web_url, org) + def getZrange(self, keyCateg, date, topNum, endSubkey=""): date_str = util.getDateStrFormat(date) keyname = "{}:{}{}".format(keyCateg, date_str, endSubkey) @@ -58,6 +68,64 @@ class Contributor_helper: data = [ [record[0].decode('utf8'), record[1]] for record in data ] return data + ''' CONTRIBUTION RANK ''' + # return: [final_rank, requirement_fulfilled, requirement_not_fulfilled] + def getOrgContributionRank(self, org): + keyname = 'CONTRIB_ORG:{org}:{orgCateg}' + final_rank = 0 + requirement_fulfilled = [] + requirement_not_fulfilled = [] + for i in range(1, self.org_rank_maxLevel+1): + key = keyname.format(org=org, orgCateg='CONTRIB_REQ_'+str(i)) + if self.serv_redis_db.get(key) is None: #non existing + requirement_not_fulfilled.append(i) + else: + requirement_fulfilled.append(i) + final_rank += 1 + num_of_previous_req_not_fulfilled = len([x for x in requirement_not_fulfilled if xContribution requirement - + {% for item in org_rank_list %} - + {{ item[1] }} {{ item[2] }} @@ -121,7 +121,7 @@ {% for item in org_honor_badge_title_list %} -
    +
    @@ -171,6 +171,7 @@
    -

    Organisation ranking

    -
    @@ -85,14 +81,37 @@
    -

    Global organisation ranking:

    +
    +
    +

    Organisation total points:

    +
    +
    +

    +
    +
    +
    +
    +

    Contribution ranking:

    +
    +
    + + + + + + + + +
    Current rankRequirement satisfiedRequirement not satisfied
    +
    +
    - + - + @@ -125,7 +144,7 @@ - + {% endfor %} From 10c59fc220c96de08d41cc4354a4781bc23edd33 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Wed, 8 Nov 2017 17:37:03 +0100 Subject: [PATCH 45/65] Added overtake remaining points + moved updateOrgContributionRank from zmq_subscribe to contributor_helper --- contributor_helper.py | 56 ++++++++++++++++++++++++++++++--- static/js/contrib.js | 22 ++++++++++++- templates/contrib.html | 8 +++++ templates/index.html | 11 ++++--- zmq_subscriber.py | 71 ++++++++++-------------------------------- 5 files changed, 103 insertions(+), 65 deletions(-) diff --git a/contributor_helper.py b/contributor_helper.py index 33f180c..ed675a3 100644 --- a/contributor_helper.py +++ b/contributor_helper.py @@ -88,11 +88,11 @@ class Contributor_helper: def giveContribRankToOrg(self, org, rankNum): keyname = 'CONTRIB_ORG:{org}:{orgCateg}' - serv_redis_db.set(keyname.format(org=orgName, orgCateg='CONTRIB_REQ_'+str(rankNum)), 1) + self.serv_redis_db.set(keyname.format(org=orgName, orgCateg='CONTRIB_REQ_'+str(rankNum)), 1) def removeContribRankFromOrg(self, org, rankNum): keyname = 'CONTRIB_ORG:{org}:{orgCateg}' - serv_redis_db.delete(keyname.format(org=orgName, orgCateg='CONTRIB_REQ_'+str(rankNum))) + self.serv_redis_db.delete(keyname.format(org=orgName, orgCateg='CONTRIB_REQ_'+str(rankNum))) # 1 for fulfilled, 0 for not fulfilled, -1 for not relevant def getCurrentContributionStatus(self, org): @@ -107,6 +107,52 @@ class Contributor_helper: to_ret[i] = -1 return to_ret + + def updateOrgContributionRank(self, orgName, pnts_to_add, contribType, eventTime, isLabeled): + keyname = 'CONTRIB_ORG:{org}:{orgCateg}' + #update total points + self.serv_redis_db.set(keyname.format(org=orgName, orgCateg='points'), pnts_to_add) + #update contribution Requirement + heavilyCount = self.heavilyCount + recentDays = self.recentDays + regularlyDays = self.regularlyDays + isRecent = (datetime.datetime.now() - eventTime).days > recentDays + + contrib = [] #[[contrib_level, contrib_ttl], [], ...] + if contribType == 'sighting': + #[contrib_level, contrib_ttl] + contrib.append([1, ONE_DAY*365]) + if contribType == 'attribute' or contribType == 'object': + contrib.append([2, ONE_DAY*365]) + if contribType == 'proposal' or contribType == 'discussion': + contrib.append([3, ONE_DAY*365]) + if contribType == 'sighting' and isRecent: + contrib.append([4, ONE_DAY*recentDays]) + if contribType == 'proposal' and isRecent: + contrib.append([5, ONE_DAY*recentDays]) + if contribType == 'event': + contrib.append([6, ONE_DAY*365]) + if contribType == 'event': + contrib.append([7, ONE_DAY*recentDays]) + if contribType == 'event': + contrib.append([8, ONE_DAY*regularlyDays]) + if contribType == 'event' and isLabeled: + contrib.append([9, ONE_DAY*regularlyDays]) + if contribType == 'sighting' and sightingWeekCount>heavilyCount: + contrib.append([10, ONE_DAY*regularlyDays]) + if (contribType == 'attribute' or contribType == 'object') and attributeWeekCount>heavilyCount: + contrib.append([11, ONE_DAY*regularlyDays]) + if contribType == 'proposal' and proposalWeekCount>heavilyCount: + contrib.append([12, ONE_DAY*regularlyDays]) + if contribType == 'event' and eventWeekCount>heavilyCount: + contrib.append([13, ONE_DAY*regularlyDays]) + if contribType == 'event' and eventWeekCount>heavilyCount and isLabeled: + contrib.append([14, ONE_DAY*regularlyDays]) + + for rankReq, ttl in contrib: + self.serv_redis_db.set(keyname.format(org=orgName, orgCateg='CONTRIB_REQ_'+str(rankReq)), 1) + self.serv_redis_db.expire(keyname.format(org=orgName, orgCateg='CONTRIB_REQ_'+str(i)), ttl) + ''' HONOR BADGES ''' def getOrgHonorBadges(self, org): keyname = 'CONTRIB_ORG:{org}:{orgCateg}' @@ -119,11 +165,11 @@ class Contributor_helper: def giveBadgeToOrg(self, org, badgeNum): keyname = 'CONTRIB_ORG:{org}:{orgCateg}' - serv_redis_db.set(keyname.format(org=orgName, orgCateg='BADGE_'+str(badgeNum)), 1) + self.serv_redis_db.set(keyname.format(org=orgName, orgCateg='BADGE_'+str(badgeNum)), 1) def removeBadgeFromOrg(self, org, badgeNum): keyname = 'CONTRIB_ORG:{org}:{orgCateg}' - serv_redis_db.delete(keyname.format(org=orgName, orgCateg='BADGE_'+str(badgeNum))) + self.serv_redis_db.delete(keyname.format(org=orgName, orgCateg='BADGE_'+str(badgeNum))) ''' MONTHLY CONTRIBUTION ''' def getOrgPntFromRedis(self, org, date): @@ -270,7 +316,9 @@ class Contributor_helper: return { 'remainingPts': 0, 'stepPts': self.rankMultiplier**self.levelMax } + ''' ''' ''' TEST DATA ''' + ''' ''' def TEST_getCategPerContribFromRedis(self, date): data2 = [] diff --git a/static/js/contrib.js b/static/js/contrib.js index b16c897..21aab6d 100644 --- a/static/js/contrib.js +++ b/static/js/contrib.js @@ -308,6 +308,7 @@ function addLastContributor(datatable, data, update) { } function updateProgressHeader(org) { + currOrg = org; // get Org rank $.getJSON( url_getOrgRank+'?org='+org, function( data ) { datatableTop.draw(); @@ -383,6 +384,25 @@ function updateProgressHeader(org) { } } }); + + //update overtake points + var prevOrg = ""; + var prevOrgPnts = 0; + datatableTop.rows().every( function() { + var row = this.node(); + if(this.data()[5] == currOrg) { + if(prevOrg == ""){ //already first + $('#orgToOverTake').text(this.data()[5]); + $('#pntsToOvertakeNext').text(0); + } else { + $('#orgToOverTake').text(prevOrg); + $('#pntsToOvertakeNext').text(parseInt(prevOrgPnts)-this.data()[0]); + } + } else { + prevOrg = this.data()[5]; + prevOrgPnts = this.data()[0]; + } + }); } function showOnlyOrg() { @@ -458,7 +478,7 @@ $(document).ready(function() { }); if(currOrg != "") // currOrg selected //FIXME: timeout used to wait that all datatables are draw. - setTimeout( function() { updateProgressHeader(currOrg); }, 200); + setTimeout( function() { updateProgressHeader(currOrg); }, 400); source_lastContrib = new EventSource(url_eventStreamLastContributor); source_lastContrib.onmessage = function(event) { diff --git a/templates/contrib.html b/templates/contrib.html index 6b84bdd..3567745 100644 --- a/templates/contrib.html +++ b/templates/contrib.html @@ -201,6 +201,14 @@
    +
    + + Points to overtake + + : + + +
    diff --git a/templates/index.html b/templates/index.html index 8e45eae..63b6f5a 100644 --- a/templates/index.html +++ b/templates/index.html @@ -107,7 +107,7 @@ table { } .textTopHeader { - height: 50px; + height: 50px; text-align: center; margin-left: 8px; float: left; @@ -132,6 +132,7 @@ small {
    @@ -146,7 +147,7 @@ small {
    - +
    No map @@ -191,7 +192,7 @@ small {
    - +
    @@ -231,7 +232,7 @@ small {
    - +
    @@ -247,7 +248,7 @@ small { - + @@ -225,7 +226,7 @@
    - Contributor Ranking + Contributor ranking (monthly)
    @@ -283,7 +284,7 @@
    - Contributors and Categories + Contributors and Categories (total)
    @@ -320,7 +321,7 @@
    - Hall Of Fame + Hall Of Fame (previous month)
    @@ -375,6 +376,7 @@ var url_getFameContributor = "{{ url_for('getFameContributor') }}"; var url_getCategPerContrib = "{{ url_for('getCategPerContrib') }}"; var url_getTop5Overtime = "{{ url_for('getTop5Overtime') }}"; + var url_getOrgOvertime = "{{ url_for('getOrgOvertime') }}"; var url_getLastContributor = "{{ url_for('getLastContributors') }}"; var url_eventStreamLastContributor = "{{ url_for('getLastContributor') }}"; var url_getAllOrg = "{{ url_for('getAllOrg') }}"; diff --git a/zmq_subscriber.py b/zmq_subscriber.py index eb6bfd9..f927d0b 100755 --- a/zmq_subscriber.py +++ b/zmq_subscriber.py @@ -86,13 +86,16 @@ def getCoordAndPublish(zmq_name, supposed_ip, categ): rep = ip_to_coord(supposed_ip) coord = rep['coord'] coord_dic = {'lat': coord['lat'], 'lon': coord['lon']} - ordDic = OrderedDict() + ordDic = OrderedDict() #keep fields with the same layout in redis ordDic['lat'] = coord_dic['lat'] ordDic['lon'] = coord_dic['lon'] coord_list = [coord['lat'], coord['lon']] push_to_redis_zset('GEO_COORD', json.dumps(ordDic)) push_to_redis_zset('GEO_COUNTRY', rep['full_rep'].country.iso_code) - push_to_redis_geo('GEO_RAD', coord['lon'], coord['lat'], json.dumps({ 'categ': categ, 'value': supposed_ip })) + ordDic = OrderedDict() #keep fields with the same layout in redis + ordDic['categ'] = categ + ordDic['value'] = supposed_ip + push_to_redis_geo('GEO_RAD', coord['lon'], coord['lat'], json.dumps(ordDic)) to_send = { "coord": coord, "categ": categ, @@ -170,13 +173,14 @@ def handler_keepalive(zmq_name, jsonevent): to_push = [ jsonevent['uptime'] ] publish_log(zmq_name, 'Keepalive', to_push) -def handler_sighting(zmq_name, jsonsight): +def handler_sighting(zmq_name, jsondata): print('sending' ,'sighting') - org = jsonsight['org'] - categ = jsonsight['categ'] - action = jsonsight['action'] + jsonsight = jsondata['Sighting'] + org = jsonsight['Event']['Orgc']['name'] + categ = jsonsight['Attribute']['category'] + action = jsondata['action'] handleContribution(zmq_name, org, categ, action, pntMultiplier=2) - return + handler_attribute(zmq_name, jsonsight, hasAlreadyBeenContributed=True) def handler_event(zmq_name, jsonobj): #fields: threat_level_id, id, info @@ -192,7 +196,7 @@ def handler_event(zmq_name, jsonobj): else: handler_attribute(zmq_name, attributes) -def handler_attribute(zmq_name, jsonobj): +def handler_attribute(zmq_name, jsonobj, hasAlreadyBeenContributed=False): # check if jsonattr is an attribute object if 'Attribute' in jsonobj: jsonattr = jsonobj['Attribute'] @@ -212,9 +216,10 @@ def handler_attribute(zmq_name, jsonobj): if jsonattr['category'] == "Network activity": getCoordAndPublish(zmq_name, jsonattr['value'], jsonattr['category']) - eventLabeled = False - #eventLabeled = len(jsonattr[]) > 0 - handleContribution(zmq_name, jsonobj['Event']['Orgc']['name'], jsonattr['category'], jsonobj['action'], isLabeled=eventLabeled) + if not hasAlreadyBeenContributed: + eventLabeled = False + #eventLabeled = len(jsonattr[]) > 0 + handleContribution(zmq_name, jsonobj['Event']['Orgc']['name'], jsonattr['category'], jsonobj['action'], isLabeled=eventLabeled) # Push to log publish_log(zmq_name, 'Attribute', to_push) @@ -227,6 +232,7 @@ def process_log(zmq_name, event): event = event.decode('utf8') topic, eventdata = event.split(' ', maxsplit=1) jsonevent = json.loads(eventdata) + print(event) dico_action[topic](zmq_name, jsonevent) From 882424be5216d0bf506c5739d76a2dd2e660e036 Mon Sep 17 00:00:00 2001 From: mokaddem Date: Thu, 9 Nov 2017 10:35:22 +0100 Subject: [PATCH 47/65] Update README.md --- README.md | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 22eb688..7eb5adb 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # MISP-Dashboard -A Dashboard showing live data and statistics from the MISP ZMQ +An experimental Dashboard showing live data and statistics from the MISP ZMQ ## Installation - Launch ```./install_dependencies.sh``` from the MISP-Dashboard directory @@ -27,3 +27,32 @@ optional arguments: -u ZMQURL, --url ZMQURL The URL to connect to ``` + +## License +Images and logos are handmade for: +- rankingMISPOrg/ +- rankingMISPMonthly/ +- MISPHonorableIcons/ + +Note that: +- Part of ```MISPHonorableIcons/1.svg``` comes from [octicons.github.com](https://octicons.github.com/icon/git-pull-request/) (CC0 - No Rights Reserved) +- Part of ```MISPHonorableIcons/2.svg``` comes from [Zeptozephyr](https://zeptozephyr.deviantart.com/art/Vectored-Portal-Icons-207347804) (CC0 - No Rights Reserved) + +``` +Copyright (C) 2017 CIRCL - Computer Incident Response Center Luxembourg (c/o smile, security made in Lëtzebuerg, Groupement d'Intérêt Economique) +Copyright (c) 2017 Sami Mokaddem + + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +``` From cc766d7b04f24397661ee87828a6ea1a8b09b363 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Thu, 9 Nov 2017 10:58:18 +0100 Subject: [PATCH 48/65] Updated non-test function + minor UI fix --- contributor_helper.py | 53 +++++++++++++++++++++++++++++-------------- server.py | 18 +++++++-------- static/js/contrib.js | 2 +- 3 files changed, 46 insertions(+), 27 deletions(-) diff --git a/contributor_helper.py b/contributor_helper.py index 670267b..7f6d621 100644 --- a/contributor_helper.py +++ b/contributor_helper.py @@ -58,7 +58,7 @@ class Contributor_helper: ''' HELPER ''' - def getOrgLogoFromRedis(self, org): + def getOrgLogoFromMISP(self, org): return "{}/img/orgs/{}.png".format(self.misp_web_url, org) def getZrange(self, keyCateg, date, topNum, endSubkey=""): @@ -84,7 +84,7 @@ class Contributor_helper: final_rank += 1 num_of_previous_req_not_fulfilled = len([x for x in requirement_not_fulfilled if x Date: Thu, 9 Nov 2017 13:43:02 +0100 Subject: [PATCH 49/65] fixed bugs if requesting data from non-existing org + refined draft updateOrgContributionRank + Fixed non existing org logo + try in zmq_subscribe --- config/ranking.cfg | 4 +- contributor_helper.py | 91 +++++++++++++++++++++++------------------- static/css/ranking.css | 4 ++ static/js/contrib.js | 26 +++++++----- util.py | 2 + zmq_subscriber.py | 29 ++++++++++---- 6 files changed, 94 insertions(+), 62 deletions(-) diff --git a/config/ranking.cfg b/config/ranking.cfg index 647fc90..8adeb93 100644 --- a/config/ranking.cfg +++ b/config/ranking.cfg @@ -32,8 +32,8 @@ maxLevel=16 12=4096 13=8192 14=16384 -15=32768 -16=65536 +15=0 +16=0 [rankRequirementsMisc] heavilyCount=10 diff --git a/contributor_helper.py b/contributor_helper.py index 7f6d621..7529e82 100644 --- a/contributor_helper.py +++ b/contributor_helper.py @@ -69,6 +69,15 @@ class Contributor_helper: return data ''' CONTRIBUTION RANK ''' + def getOrgContributionTotalPoints(self, org): + keyname = 'CONTRIB_ORG:{org}:{orgCateg}' + pnts = self.serv_redis_db.get(keyname.format(org=org, orgCateg='points')) + if pnts is None: + pnts = 0 + else: + pnts = int(pnts.decode('utf8')) + return pnts + # return: [final_rank, requirement_fulfilled, requirement_not_fulfilled] def getOrgContributionRank(self, org): keyname = 'CONTRIB_ORG:{org}:{orgCateg}' @@ -108,53 +117,55 @@ class Contributor_helper: to_ret[i] = 0 else: to_ret[i] = -1 - return to_ret - + return {'rank': final_rank, 'status': to_ret, 'totPoints': self.getOrgContributionTotalPoints(org)} def updateOrgContributionRank(self, orgName, pnts_to_add, contribType, eventTime, isLabeled): keyname = 'CONTRIB_ORG:{org}:{orgCateg}' #update total points - self.serv_redis_db.set(keyname.format(org=orgName, orgCateg='points'), pnts_to_add) - #update contribution Requirement + totOrgPnts = self.serv_redis_db.incrby(keyname.format(org=orgName, orgCateg='points'), pnts_to_add) + # getRequirement parameters heavilyCount = self.heavilyCount recentDays = self.recentDays regularlyDays = self.regularlyDays isRecent = (datetime.datetime.now() - eventTime).days > recentDays + print("isLabeled: {}, isRecent: {}, totOrgPnts".format(isLabeled, isRecent, totOrgPnts)) + #update contribution Requirement contrib = [] #[[contrib_level, contrib_ttl], [], ...] - if contribType == 'sighting': + if totOrgPnts >= self.org_rank_requirement_pnts[1] and contribType == 'Sighting': #[contrib_level, contrib_ttl] - contrib.append([1, ONE_DAY*365]) - if contribType == 'attribute' or contribType == 'object': - contrib.append([2, ONE_DAY*365]) - if contribType == 'proposal' or contribType == 'discussion': - contrib.append([3, ONE_DAY*365]) - if contribType == 'sighting' and isRecent: - contrib.append([4, ONE_DAY*recentDays]) - if contribType == 'proposal' and isRecent: - contrib.append([5, ONE_DAY*recentDays]) - if contribType == 'event': - contrib.append([6, ONE_DAY*365]) - if contribType == 'event': - contrib.append([7, ONE_DAY*recentDays]) - if contribType == 'event': - contrib.append([8, ONE_DAY*regularlyDays]) - if contribType == 'event' and isLabeled: - contrib.append([9, ONE_DAY*regularlyDays]) - if contribType == 'sighting' and sightingWeekCount>heavilyCount: - contrib.append([10, ONE_DAY*regularlyDays]) - if (contribType == 'attribute' or contribType == 'object') and attributeWeekCount>heavilyCount: - contrib.append([11, ONE_DAY*regularlyDays]) - if contribType == 'proposal' and proposalWeekCount>heavilyCount: - contrib.append([12, ONE_DAY*regularlyDays]) - if contribType == 'event' and eventWeekCount>heavilyCount: - contrib.append([13, ONE_DAY*regularlyDays]) - if contribType == 'event' and eventWeekCount>heavilyCount and isLabeled: - contrib.append([14, ONE_DAY*regularlyDays]) + contrib.append([1, util.ONE_DAY*365]) + if totOrgPnts >= self.org_rank_requirement_pnts[2] and contribType == 'Attribute' or contribType == 'Object': + contrib.append([2, util.ONE_DAY*365]) + if totOrgPnts >= self.org_rank_requirement_pnts[3] and contribType == 'Proposal' or contribType == 'Discussion': + contrib.append([3, util.ONE_DAY*365]) + if totOrgPnts >= self.org_rank_requirement_pnts[4] and contribType == 'Sighting' and isRecent: + contrib.append([4, util.ONE_DAY*recentDays]) + if totOrgPnts >= self.org_rank_requirement_pnts[5] and contribType == 'Proposal' and isRecent: + contrib.append([5, util.ONE_DAY*recentDays]) + if totOrgPnts >= self.org_rank_requirement_pnts[6] and contribType == 'Event': + contrib.append([6, util.ONE_DAY*365]) + if totOrgPnts >= self.org_rank_requirement_pnts[7] and contribType == 'Event': + contrib.append([7, util.ONE_DAY*recentDays]) + if totOrgPnts >= self.org_rank_requirement_pnts[8] and contribType == 'Event': + contrib.append([8, util.ONE_DAY*regularlyDays]) + if totOrgPnts >= self.org_rank_requirement_pnts[9] and contribType == 'Event' and isLabeled: + contrib.append([9, util.ONE_DAY*regularlyDays]) + if totOrgPnts >= self.org_rank_requirement_pnts[10] and contribType == 'Sighting' and sightingWeekCount>heavilyCount: + contrib.append([10, util.ONE_DAY*regularlyDays]) + if totOrgPnts >= self.org_rank_requirement_pnts[11] and (contribType == 'Attribute' or contribType == 'Object') and attributeWeekCount>heavilyCount: + contrib.append([11, util.ONE_DAY*regularlyDays]) + if totOrgPnts >= self.org_rank_requirement_pnts[12] and contribType == 'Proposal' and proposalWeekCount>heavilyCount: + contrib.append([12, util.ONE_DAY*regularlyDays]) + if totOrgPnts >= self.org_rank_requirement_pnts[13] and contribType == 'Event' and eventWeekCount>heavilyCount: + contrib.append([13, util.ONE_DAY*regularlyDays]) + if totOrgPnts >= self.org_rank_requirement_pnts[14] and contribType == 'Event' and eventWeekCount>heavilyCount and isLabeled: + contrib.append([14, util.ONE_DAY*regularlyDays]) + print([r for r, ttl in contrib]) for rankReq, ttl in contrib: self.serv_redis_db.set(keyname.format(org=orgName, orgCateg='CONTRIB_REQ_'+str(rankReq)), 1) - self.serv_redis_db.expire(keyname.format(org=orgName, orgCateg='CONTRIB_REQ_'+str(i)), ttl) + self.serv_redis_db.expire(keyname.format(org=orgName, orgCateg='CONTRIB_REQ_'+str(rankReq)), ttl) ''' HONOR BADGES ''' def getOrgHonorBadges(self, org): @@ -357,22 +368,20 @@ class Contributor_helper: def TEST_getTop5OvertimeFromRedis(self): import time now = time.time() - ONE_DAY = 60*60*24 data2 = [ - {'label': 'CIRCL', 'data': [[now, random.randint(1,50)], [now-ONE_DAY, random.randint(1,50)], [now-ONE_DAY*2, random.randint(1,50)], [now-ONE_DAY*3, random.randint(1,50)], [now-ONE_DAY*4, random.randint(1,50)]]}, - {'label': 'CASES', 'data': [[now, random.randint(1,50)], [now-ONE_DAY, random.randint(1,50)], [now-ONE_DAY*2, random.randint(1,50)], [now-ONE_DAY*3, random.randint(1,50)], [now-ONE_DAY*4, random.randint(1,50)]]}, - {'label': 'Org1', 'data': [[now, random.randint(1,50)], [now-ONE_DAY, random.randint(1,50)], [now-ONE_DAY*2, random.randint(1,50)], [now-ONE_DAY*3, random.randint(1,50)], [now-ONE_DAY*4, random.randint(1,50)]]}, - {'label': 'Org2', 'data': [[now, random.randint(1,50)], [now-ONE_DAY, random.randint(1,50)], [now-ONE_DAY*2, random.randint(1,50)], [now-ONE_DAY*3, random.randint(1,50)], [now-ONE_DAY*4, random.randint(1,50)]]}, - {'label': 'SMILE', 'data': [[now, random.randint(1,50)], [now-ONE_DAY, random.randint(1,50)], [now-ONE_DAY*2, random.randint(1,50)], [now-ONE_DAY*3, random.randint(1,50)], [now-ONE_DAY*4, random.randint(1,50)]]}, + {'label': 'CIRCL', 'data': [[now, random.randint(1,50)], [now-util.ONE_DAY, random.randint(1,50)], [now-util.ONE_DAY*2, random.randint(1,50)], [now-util.ONE_DAY*3, random.randint(1,50)], [now-util.ONE_DAY*4, random.randint(1,50)]]}, + {'label': 'CASES', 'data': [[now, random.randint(1,50)], [now-util.ONE_DAY, random.randint(1,50)], [now-util.ONE_DAY*2, random.randint(1,50)], [now-util.ONE_DAY*3, random.randint(1,50)], [now-util.ONE_DAY*4, random.randint(1,50)]]}, + {'label': 'Org1', 'data': [[now, random.randint(1,50)], [now-util.ONE_DAY, random.randint(1,50)], [now-util.ONE_DAY*2, random.randint(1,50)], [now-util.ONE_DAY*3, random.randint(1,50)], [now-util.ONE_DAY*4, random.randint(1,50)]]}, + {'label': 'Org2', 'data': [[now, random.randint(1,50)], [now-util.ONE_DAY, random.randint(1,50)], [now-util.ONE_DAY*2, random.randint(1,50)], [now-util.ONE_DAY*3, random.randint(1,50)], [now-util.ONE_DAY*4, random.randint(1,50)]]}, + {'label': 'SMILE', 'data': [[now, random.randint(1,50)], [now-util.ONE_DAY, random.randint(1,50)], [now-util.ONE_DAY*2, random.randint(1,50)], [now-util.ONE_DAY*3, random.randint(1,50)], [now-util.ONE_DAY*4, random.randint(1,50)]]}, ] return data2 def TEST_getOrgOvertime(self, org): import time now = time.time() - ONE_DAY = 60*60*24 data = [ - {'label': org, 'data': [[now, random.randint(1,30)], [now-ONE_DAY, random.randint(1,30)], [now-ONE_DAY*2, random.randint(1,30)], [now-ONE_DAY*3, random.randint(1,30)], [now-ONE_DAY*4, random.randint(1,40)]]} + {'label': org, 'data': [[now, random.randint(1,30)], [now-util.ONE_DAY, random.randint(1,30)], [now-util.ONE_DAY*2, random.randint(1,30)], [now-util.ONE_DAY*3, random.randint(1,30)], [now-util.ONE_DAY*4, random.randint(1,40)]]} ] return data diff --git a/static/css/ranking.css b/static/css/ranking.css index 7efaba1..f744290 100644 --- a/static/css/ranking.css +++ b/static/css/ranking.css @@ -3,6 +3,10 @@ height: auto; } +.successCell { + background-color: #dff0d8 !important +} + .circleBadge { width: 80px; height: 80px; diff --git a/static/js/contrib.js b/static/js/contrib.js index 2c3eaa8..d53e5a6 100644 --- a/static/js/contrib.js +++ b/static/js/contrib.js @@ -156,11 +156,13 @@ function getOrgRankIcon(rank, size) { } function createImg(source, size) { - var obj = document.createElement('object'); + var obj = document.createElement('img'); obj.height = size; obj.width = size; obj.style.margin = 'auto'; - obj.data = source; + obj.src = source; + obj.type = "image/png" + obj.alt = "" return obj.outerHTML; } @@ -372,19 +374,22 @@ function updateProgressHeader(org) { } for (var row of $('#bodyTablerankingModal')[0].children) { row = $(row); + var firstCell = $(row.children()[0]); var rank = row.data('rank'); //remove all classes row.removeClass("warning"); row.removeClass("danger"); row.removeClass("success"); + firstCell.removeClass("successCell"); //add correct class if(status[rank] == 0){ row.addClass("danger"); - } else if(status[rank] == 1 && rank == curContributionOrgRank) { - row.addClass("success"); } else if(status[rank] == 1) { row.addClass("warning"); } + if(rank == curContributionOrgRank) { + firstCell.addClass("successCell"); + } } }); @@ -424,14 +429,13 @@ function updateProgressHeader(org) { $.getJSON( url_getOrgOvertime+'?org='+org, function( data ) { var toPlot = dataTop5Overtime.slice(0); //cloning data // transform secs into date - for(i in data){ - var new_data = []; - for(list of data[i]['data']) { - new_data.push([new Date(list[0]*1000), list[1]]); - } - data[i]['data'] = new_data; - toPlot.push(data[i]); + var new_data = []; + for(list of data['data']) { + new_data.push([new Date(list[0]*1000), list[1]]); } + data['data'] = new_data; + toPlot.push(data); + plotLineChart.setData(toPlot); plotLineChart.setupGrid(); plotLineChart.draw(); diff --git a/util.py b/util.py index e419597..27969ee 100644 --- a/util.py +++ b/util.py @@ -1,5 +1,7 @@ import datetime +ONE_DAY = 60*60*24 + def getMonthSpan(date): ds = datetime.datetime(date.year, date.month, 1) dyear = 1 if ds.month+1 > 12 else 0 diff --git a/zmq_subscriber.py b/zmq_subscriber.py index f927d0b..bf6e798 100755 --- a/zmq_subscriber.py +++ b/zmq_subscriber.py @@ -130,8 +130,8 @@ def noSpaceLower(str): return str.lower().replace(' ', '_') #pntMultiplier if one contribution rewards more than others. (e.g. shighting may gives more points than editing) -def handleContribution(zmq_name, org, categ, action, pntMultiplier=1, eventTime=datetime.datetime.now(), isLabeled=False): - if action in ['edit']: +def handleContribution(zmq_name, org, contribType, categ, action, pntMultiplier=1, eventTime=datetime.datetime.now(), isLabeled=False): + if action in ['edit', None]: pass #return #not a contribution? # is a valid contribution @@ -151,7 +151,7 @@ def handleContribution(zmq_name, org, categ, action, pntMultiplier=1, eventTime= serv_redis_db.zadd('CONTRIB_LAST:'+util.getDateStrFormat(now), nowSec, org) serv_redis_db.expire('CONTRIB_LAST:'+util.getDateStrFormat(now), ONE_DAY) #expire after 1 day - contributor_helper.updateOrgContributionRank(org, pnts_to_add, eventTime, eventTime=datetime.datetime.now(), isLabeled=isLabeled) + contributor_helper.updateOrgContributionRank(org, pnts_to_add, contribType, eventTime=datetime.datetime.now(), isLabeled=isLabeled) publish_log(zmq_name, 'CONTRIBUTION', {'org': org, 'categ': categ, 'action': action, 'epoch': nowSec }, channel=CHANNEL_LASTCONTRIB) @@ -178,8 +178,11 @@ def handler_sighting(zmq_name, jsondata): jsonsight = jsondata['Sighting'] org = jsonsight['Event']['Orgc']['name'] categ = jsonsight['Attribute']['category'] - action = jsondata['action'] - handleContribution(zmq_name, org, categ, action, pntMultiplier=2) + try: + action = jsondata['action'] + except KeyError: + action = None + handleContribution(zmq_name, org, 'Sighting', categ, action, pntMultiplier=2) handler_attribute(zmq_name, jsonsight, hasAlreadyBeenContributed=True) def handler_event(zmq_name, jsonobj): @@ -217,9 +220,19 @@ def handler_attribute(zmq_name, jsonobj, hasAlreadyBeenContributed=False): getCoordAndPublish(zmq_name, jsonattr['value'], jsonattr['category']) if not hasAlreadyBeenContributed: - eventLabeled = False - #eventLabeled = len(jsonattr[]) > 0 - handleContribution(zmq_name, jsonobj['Event']['Orgc']['name'], jsonattr['category'], jsonobj['action'], isLabeled=eventLabeled) + try: + eventLabeled = len(jsonattr['Tag']) > 0 + except KeyError: + eventLabeled = False + try: + action = jsonobj['action'] + except KeyError: + action = None + handleContribution(zmq_name, jsonobj['Event']['Orgc']['name'], + 'Attribute', + jsonattr['category'], + action, + isLabeled=eventLabeled) # Push to log publish_log(zmq_name, 'Attribute', to_push) From a3c853e490b438eed7162aa06b6c6ee71faf210e Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Thu, 9 Nov 2017 16:31:04 +0100 Subject: [PATCH 50/65] Updated ranking level decision + updated support of message for zmq + Updated badge acquisition display to match data in DB --- contributor_helper.py | 43 ++++++++++++++++++------ static/js/contrib.js | 75 +++++++++++++++++++++++++++--------------- templates/contrib.html | 1 + zmq_subscriber.py | 60 ++++++++++++++++++++++++--------- 4 files changed, 127 insertions(+), 52 deletions(-) diff --git a/contributor_helper.py b/contributor_helper.py index 7529e82..d04cada 100644 --- a/contributor_helper.py +++ b/contributor_helper.py @@ -91,17 +91,18 @@ class Contributor_helper: else: requirement_fulfilled.append(i) final_rank += 1 - num_of_previous_req_not_fulfilled = len([x for x in requirement_not_fulfilled if x recentDays + print("contribType: {}, action: {}".format(contribType, action)) print("isLabeled: {}, isRecent: {}, totOrgPnts".format(isLabeled, isRecent, totOrgPnts)) #update contribution Requirement contrib = [] #[[contrib_level, contrib_ttl], [], ...] @@ -145,9 +168,9 @@ class Contributor_helper: contrib.append([5, util.ONE_DAY*recentDays]) if totOrgPnts >= self.org_rank_requirement_pnts[6] and contribType == 'Event': contrib.append([6, util.ONE_DAY*365]) - if totOrgPnts >= self.org_rank_requirement_pnts[7] and contribType == 'Event': + if totOrgPnts >= self.org_rank_requirement_pnts[7] and contribType == 'Event' and eventMonthCount>=1: contrib.append([7, util.ONE_DAY*recentDays]) - if totOrgPnts >= self.org_rank_requirement_pnts[8] and contribType == 'Event': + if totOrgPnts >= self.org_rank_requirement_pnts[8] and contribType == 'Event' and eventWeekCount>=1: contrib.append([8, util.ONE_DAY*regularlyDays]) if totOrgPnts >= self.org_rank_requirement_pnts[9] and contribType == 'Event' and isLabeled: contrib.append([9, util.ONE_DAY*regularlyDays]) @@ -179,11 +202,11 @@ class Contributor_helper: def giveBadgeToOrg(self, org, badgeNum): keyname = 'CONTRIB_ORG:{org}:{orgCateg}' - self.serv_redis_db.set(keyname.format(org=orgName, orgCateg='BADGE_'+str(badgeNum)), 1) + self.serv_redis_db.set(keyname.format(org=org, orgCateg='BADGE_'+str(badgeNum)), 1) def removeBadgeFromOrg(self, org, badgeNum): keyname = 'CONTRIB_ORG:{org}:{orgCateg}' - self.serv_redis_db.delete(keyname.format(org=orgName, orgCateg='BADGE_'+str(badgeNum))) + self.serv_redis_db.delete(keyname.format(org=org, orgCateg='BADGE_'+str(badgeNum))) ''' MONTHLY CONTRIBUTION ''' def getOrgPntFromRedis(self, org, date): diff --git a/static/js/contrib.js b/static/js/contrib.js index d53e5a6..b42ca12 100644 --- a/static/js/contrib.js +++ b/static/js/contrib.js @@ -324,6 +324,46 @@ function addLastContributor(datatable, data, update) { } } +function updateProgressBar(org) { + if(currOrg != org) + return; + $.getJSON( url_getOrgRank+'?org='+org, function( data ) { + var rank = Math.floor(data.rank); + var rankDec = data.rank-rank; + var popoverRank = $('#btnCurrRank').data('bs.popover'); + popoverRank.options.content = generateRankingSheet(rank, rankDec, data.stepPts, data.points, data.remainingPts); + $('#orgRankDiv').html(getMonthlyRankIcon(rank, 40, true)); + $('#orgNextRankDiv').html(getMonthlyRankIcon(rank+1, 40, true)); + if (data.rank > 16){ + $('#progressBarDiv').width(1*150); //150 is empty bar width + } else { + $('#progressBarDiv').width((data.rank - rank)*150); //150 is empty bar width + } + }); +} + +function updateOvertakePnts() { + var prevOrgName = ""; + var prevOrgPnts = 0; + datatableTop.rows().every( function() { + var row = this.node(); + var orgRowName = $(this.data()[5])[0].text; // contained in + var orgRowPnts = this.data()[0] + if(orgRowName == currOrg) { + if(prevOrgName == ""){ //already first + $('#orgToOverTake').text(orgRowName); + $('#pntsToOvertakeNext').text(0); + } else { + $('#orgToOverTake').text(prevOrgName); + $('#pntsToOvertakeNext').text(parseInt(prevOrgPnts)-orgRowPnts); + } + } else { + prevOrgName = orgRowName; + prevOrgPnts = orgRowPnts; + } + }); +} + function updateProgressHeader(org) { currOrg = org; // get Org rank @@ -395,36 +435,16 @@ function updateProgressHeader(org) { // colorize badge if acquired $.getJSON( url_getHonorBadges+'?org='+org, function( data ) { - for(var i=0; i - var orgRowPnts = this.data()[0] - if(orgRowName == currOrg) { - if(prevOrgName == ""){ //already first - $('#orgToOverTake').text(orgRowName); - $('#pntsToOvertakeNext').text(0); - } else { - $('#orgToOverTake').text(prevOrgName); - $('#pntsToOvertakeNext').text(parseInt(prevOrgPnts)-orgRowPnts); - } - } else { - prevOrgName = orgRowName; - prevOrgPnts = orgRowPnts; - } - }); - + updateOvertakePnts(); //Add new data to linechart $.getJSON( url_getOrgOvertime+'?org='+org, function( data ) { var toPlot = dataTop5Overtime.slice(0); //cloning data @@ -531,7 +551,8 @@ $(document).ready(function() { var json = jQuery.parseJSON( event.data ); addLastContributor(datatableLast, json, true); datatableLast.draw(); - updateProgressHeader(json.org) + updateProgressBar(json.org); + updateOvertakePnts(); sec_before_reload = refresh_speed; //reset timer at each contribution }; }); diff --git a/templates/contrib.html b/templates/contrib.html index 5a8a273..d380806 100644 --- a/templates/contrib.html +++ b/templates/contrib.html @@ -397,6 +397,7 @@ var categ_list = JSON.parse('{{ categ_list|safe }}'); var org_rank_obj = JSON.parse('{{ org_rank_json|safe }}'); var org_honor_badge_title = JSON.parse('{{ org_honor_badge_title|safe }}'); + var numberOfBadges = {{ org_honor_badge_title_list|length }}; diff --git a/zmq_subscriber.py b/zmq_subscriber.py index bf6e798..5b6ca3d 100755 --- a/zmq_subscriber.py +++ b/zmq_subscriber.py @@ -126,34 +126,39 @@ def getFields(obj, fields): except KeyError as e: return "" -def noSpaceLower(str): - return str.lower().replace(' ', '_') +def noSpaceLower(text): + return text.lower().replace(' ', '_') #pntMultiplier if one contribution rewards more than others. (e.g. shighting may gives more points than editing) def handleContribution(zmq_name, org, contribType, categ, action, pntMultiplier=1, eventTime=datetime.datetime.now(), isLabeled=False): if action in ['edit', None]: pass #return #not a contribution? - # is a valid contribution - try: - pnts_to_add = DICO_PNTS_REWARD[noSpaceLower(categ)] - except KeyError: - pnts_to_add = DEFAULT_PNTS_REWARD - pnts_to_add *= pntMultiplier - - push_to_redis_zset('CONTRIB_DAY', org, count=pnts_to_add) - #CONTRIB_CATEG retain the contribution per category, not the point earned in this categ - push_to_redis_zset('CONTRIB_CATEG', org, count=DEFAULT_PNTS_REWARD, endSubkey=':'+noSpaceLower(categ)) - serv_redis_db.sadd('CONTRIB_ALL_ORG', org) now = datetime.datetime.now() nowSec = int(time.time()) + pnts_to_add = DEFAULT_PNTS_REWARD + + # is a valid contribution + if categ is not None: + try: + pnts_to_add = DICO_PNTS_REWARD[noSpaceLower(categ)] + except KeyError: + pnts_to_add = DEFAULT_PNTS_REWARD + pnts_to_add *= pntMultiplier + + push_to_redis_zset('CONTRIB_DAY', org, count=pnts_to_add) + #CONTRIB_CATEG retain the contribution per category, not the point earned in this categ + push_to_redis_zset('CONTRIB_CATEG', org, count=DEFAULT_PNTS_REWARD, endSubkey=':'+noSpaceLower(categ)) + publish_log(zmq_name, 'CONTRIBUTION', {'org': org, 'categ': categ, 'action': action, 'epoch': nowSec }, channel=CHANNEL_LASTCONTRIB) + + serv_redis_db.sadd('CONTRIB_ALL_ORG', org) + serv_redis_db.zadd('CONTRIB_LAST:'+util.getDateStrFormat(now), nowSec, org) serv_redis_db.expire('CONTRIB_LAST:'+util.getDateStrFormat(now), ONE_DAY) #expire after 1 day - contributor_helper.updateOrgContributionRank(org, pnts_to_add, contribType, eventTime=datetime.datetime.now(), isLabeled=isLabeled) + contributor_helper.updateOrgContributionRank(org, pnts_to_add, action, contribType, eventTime=datetime.datetime.now(), isLabeled=isLabeled) - publish_log(zmq_name, 'CONTRIBUTION', {'org': org, 'categ': categ, 'action': action, 'epoch': nowSec }, channel=CHANNEL_LASTCONTRIB) ############## @@ -173,6 +178,10 @@ def handler_keepalive(zmq_name, jsonevent): to_push = [ jsonevent['uptime'] ] publish_log(zmq_name, 'Keepalive', to_push) +def handler_object(zmq_name, jsondata): + print('obj') + return + def handler_sighting(zmq_name, jsondata): print('sending' ,'sighting') jsonsight = jsondata['Sighting'] @@ -199,6 +208,26 @@ def handler_event(zmq_name, jsonobj): else: handler_attribute(zmq_name, attributes) + try: + action = jsonobj['action'] + except KeyError: + action = None + try: + eventLabeled = len(jsonobj['EventTag']) > 0 + except KeyError: + eventLabeled = False + try: + org = jsonobj['Orgc']['name'] + except KeyError: + org = None + + if org is not None: + handleContribution(zmq_name, org, + 'Event', + None, + action, + isLabeled=eventLabeled) + def handler_attribute(zmq_name, jsonobj, hasAlreadyBeenContributed=False): # check if jsonattr is an attribute object if 'Attribute' in jsonobj: @@ -267,6 +296,7 @@ dico_action = { "misp_json_event": handler_event, "misp_json_self": handler_keepalive, "misp_json_attribute": handler_attribute, + "misp_json_object": handler_object, "misp_json_sighting": handler_sighting, "misp_json_organisation": handler_log, "misp_json_user": handler_log, From 1725f0b519a509af43668e627b0c406b908d9f4c Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Thu, 9 Nov 2017 16:31:34 +0100 Subject: [PATCH 51/65] Added helper script to give ranks/badges --- give_honors_to_org.py | 111 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100755 give_honors_to_org.py diff --git a/give_honors_to_org.py b/give_honors_to_org.py new file mode 100755 index 0000000..87f7e11 --- /dev/null +++ b/give_honors_to_org.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3.5 + +import os, sys +import redis +import configparser + +import contributor_helper + +configfile = os.path.join(os.environ['DASH_CONFIG'], 'config.cfg') +cfg = configparser.ConfigParser() +cfg.read(configfile) +serv_redis_db = redis.StrictRedis( + host=cfg.get('RedisGlobal', 'host'), + port=cfg.getint('RedisGlobal', 'port'), + db=cfg.getint('RedisDB', 'db')) + +chelper = contributor_helper.Contributor_helper(serv_redis_db, cfg) + +def printOrgInfo(org): + org_pnts = chelper.getOrgContributionTotalPoints(org) + org_c_rank = chelper.getOrgContributionRank(org) + org_c_status = chelper.getCurrentContributionStatus(org) + org_honor_badge = chelper.getOrgHonorBadges(org) + + print() + print("Organisation points: {}".format(org_pnts)) + print("Organisation contribution rank: {}".format(org_c_status['rank'])) + print(''' +Organisation contribution rank: +-------------------------------''') + for rank in range(1, chelper.org_rank_maxLevel+1): + acq = 'x' if org_c_status['status'][rank] == 1 else ' ' + print("{}.\t[{}]\t{}\t{}".format(rank, acq, chelper.org_rank_requirement_pnts[rank], chelper.org_rank_requirement_text[rank])) + + print() + print(''' +Organisation honor badges: +--------------------------''') + for badgeNum, text in chelper.org_honor_badge_title.items(): + acq = 'x' if badgeNum in org_honor_badge else ' ' + print("{}.\t[{}]\t{}".format(badgeNum, acq, text)) + print() + + +def main(): + if len(sys.argv) > 1: + org = sys.argv[1] + else: + org = input('Enter the organisation name: ') + + printOrgInfo(org) + + # ranks + while True: + org_pnts = chelper.getOrgContributionTotalPoints(org) + org_c_rank = chelper.getOrgContributionRank(org) + org_c_status = chelper.getCurrentContributionStatus(org) + org_honor_badge = chelper.getOrgHonorBadges(org) + + userRep = input("Enter the organisation RANK to give/remove to {} ( to finish): ".format(org)) + if userRep == '': + break + else: + # validate input + try: #not int + rankNum = int(userRep) + except: + print('Not an integer') + continue + if rankNum < 1 and rankNum > chelper.org_rank_maxLevel: + print('Not a valid rank') + continue + + if org_c_status['status'][rankNum] == 1: #remove rank + chelper.removeContribRankFromOrg(org, rankNum) + else: + chelper.giveContribRankToOrg(org, rankNum) + + printOrgInfo(org) + + # badges + while True: + org_pnts = chelper.getOrgContributionTotalPoints(org) + org_c_rank = chelper.getOrgContributionRank(org) + org_c_status = chelper.getCurrentContributionStatus(org) + org_honor_badge = chelper.getOrgHonorBadges(org) + + userRep = input("Enter the organisation BADGE to give/remove to {} ( to finish): ".format(org)) + if userRep == '': + break + else: + # validate input + try: #not int + badgeNum = int(userRep) + except: + print('Not an integer') + continue + if badgeNum < 1 and badgeNum > chelper.honorBadgeNum: + print('Not a valid rank') + continue + + if badgeNum in org_honor_badge: #remove badge + chelper.removeBadgeFromOrg(org, badgeNum) + else: + chelper.giveBadgeToOrg(org, badgeNum) + + printOrgInfo(org) + +if __name__ == '__main__': + main() + From b0a83a9ebd3cb39fa638a80f6c032ef045a3a1c3 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Fri, 10 Nov 2017 09:00:50 +0100 Subject: [PATCH 52/65] Added support of previous date from getCategPerContrib --- contributor_helper.py | 28 ++++++++++++---------------- static/js/contrib.js | 1 + 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/contributor_helper.py b/contributor_helper.py index d04cada..474309b 100644 --- a/contributor_helper.py +++ b/contributor_helper.py @@ -310,24 +310,20 @@ class Contributor_helper: def getCategPerContribFromRedis(self, date): keyCateg = "CONTRIB_DAY" topNum = 0 # all - contrib_org = self.getZrange(keyCateg, date, topNum) - data = [] - for org, pnts in contrib_org: - dic = {} - dic['rank'] = self.getTrueRank(pnts) - dic['orgRank'] = self.getOrgContributionRank(org)['final_rank'] - dic['honorBadge'] = self.getOrgHonorBadges(org) - dic['logo_path'] = self.getOrgLogoFromMISP(org) - dic['org'] = org - dic['pnts'] = pnts + contrib_org = self.getTopContributorFromRedis(date) + for dic in contrib_org: + org = dic['org'] for categ in self.categories_in_datatable: - keyname = 'CONTRIB_CATEG:'+util.getDateStrFormat(date)+':'+categ - categ_score = self.serv_redis_db.zscore(keyname, org) - if categ_score is None: categ_score = 0 - dic[categ] = categ_score - data.append(dic) - return data + for curDate in util.getMonthSpan(date): + keyname = 'CONTRIB_CATEG:'+util.getDateStrFormat(curDate)+':'+categ + temp = self.serv_redis_db.zscore(keyname, org) + if temp is None: + temp = 0 + categ_score += temp + dic[categ] = categ_score + return contrib_org + def getAllOrgFromRedis(self): data = self.serv_redis_db.smembers('CONTRIB_ALL_ORG') diff --git a/static/js/contrib.js b/static/js/contrib.js index b42ca12..4a7a8e6 100644 --- a/static/js/contrib.js +++ b/static/js/contrib.js @@ -82,6 +82,7 @@ optionDatatable_fame.scrollY = '45vh'; var optionDatatable_Categ = { responsive: true, searching: true, + "order": [[ 0, "desc" ]], scrollY: '38vh', "scrollX": true, scrollCollapse: true, From cb53be42530d020fdbcad1d6e4eef0643ed874c9 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Fri, 10 Nov 2017 09:22:46 +0100 Subject: [PATCH 53/65] Prevent double display of org in line chart --- static/js/contrib.js | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/static/js/contrib.js b/static/js/contrib.js index 4a7a8e6..3de70dd 100644 --- a/static/js/contrib.js +++ b/static/js/contrib.js @@ -446,21 +446,31 @@ function updateProgressHeader(org) { //update overtake points updateOvertakePnts(); - //Add new data to linechart - $.getJSON( url_getOrgOvertime+'?org='+org, function( data ) { - var toPlot = dataTop5Overtime.slice(0); //cloning data - // transform secs into date - var new_data = []; - for(list of data['data']) { - new_data.push([new Date(list[0]*1000), list[1]]); - } - data['data'] = new_data; - toPlot.push(data); - plotLineChart.setData(toPlot); - plotLineChart.setupGrid(); - plotLineChart.draw(); - }); + //Add new data to linechart + var flag_already_displayed = false; + for(obj of dataTop5Overtime) { //check if already displayed + if (obj.label == currOrg) { + flag_already_displayed = true; + break; + } + } + if (!flag_already_displayed) { + $.getJSON( url_getOrgOvertime+'?org='+org, function( data ) { + var toPlot = dataTop5Overtime.slice(0); //cloning data + // transform secs into date + var new_data = []; + for(list of data['data']) { + new_data.push([new Date(list[0]*1000), list[1]]); + } + data['data'] = new_data; + toPlot.push(data); + + plotLineChart.setData(toPlot); + plotLineChart.setupGrid(); + plotLineChart.draw(); + }); + } } function showOnlyOrg() { From d8ad176b722d267d6a43ef1099bc759d8ea6f233 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Fri, 10 Nov 2017 09:28:59 +0100 Subject: [PATCH 54/65] Added support of previous date in getLastContributors --- contributor_helper.py | 25 ++++++++++++++----------- util.py | 10 ++++++++++ 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/contributor_helper.py b/contributor_helper.py index 474309b..51f2f95 100644 --- a/contributor_helper.py +++ b/contributor_helper.py @@ -229,21 +229,24 @@ class Contributor_helper: def getLastContributorsFromRedis(self): date = datetime.datetime.now() keyname = "CONTRIB_LAST" + prev_days = 7 topNum = self.MAX_NUMBER_OF_LAST_CONTRIBUTOR # default Num - last_contrib_org = self.getZrange(keyname, date, topNum) data = [] - for org, sec in last_contrib_org: - dic = {} - dic['rank'] = self.getOrgRankFromRedis(org, date) - dic['orgRank'] = self.getOrgContributionRank(org)['final_rank'] - dic['honorBadge'] = self.getOrgHonorBadges(org) - dic['logo_path'] = self.getOrgLogoFromMISP(org) - dic['org'] = org - dic['pnts'] = self.getOrgPntFromRedis(org, date) - dic['epoch'] = sec - data.append(dic) + for curDate in util.getXPrevDaysSpan(date, prev_days): + last_contrib_org = self.getZrange(keyname, curDate, topNum) + for org, sec in last_contrib_org: + dic = {} + dic['rank'] = self.getOrgRankFromRedis(org, date) + dic['orgRank'] = self.getOrgContributionRank(org)['final_rank'] + dic['honorBadge'] = self.getOrgHonorBadges(org) + dic['logo_path'] = self.getOrgLogoFromMISP(org) + dic['org'] = org + dic['pnts'] = self.getOrgPntFromRedis(org, date) + dic['epoch'] = sec + data.append(dic) return data + def getContributorFromRedis(self, org): date = datetime.datetime.now() epoch = self.serv_redis_db.zscore("CONTRIB_LAST", org) diff --git a/util.py b/util.py index 27969ee..ba794f1 100644 --- a/util.py +++ b/util.py @@ -14,5 +14,15 @@ def getMonthSpan(date): to_return.append(ds + datetime.timedelta(days=i)) return to_return +def getXPrevDaysSpan(date, days): + de = date + ds = de - datetime.timedelta(days=days) + + delta = de - ds + to_return = [] + for i in range(delta.days+1): + to_return.append(de - datetime.timedelta(days=i)) + return to_return + def getDateStrFormat(date): return str(date.year)+str(date.month).zfill(2)+str(date.day).zfill(2) From 46d389c2109cd2682f33fd23117292a9f7f740b4 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Fri, 10 Nov 2017 09:56:41 +0100 Subject: [PATCH 55/65] Added display of datetime in last contributors --- static/js/contrib.js | 6 +++--- templates/contrib.html | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/static/js/contrib.js b/static/js/contrib.js index 3de70dd..b1b183f 100644 --- a/static/js/contrib.js +++ b/static/js/contrib.js @@ -72,7 +72,6 @@ optionDatatable_last.columnDefs = [ { 'orderData':[6], 'targets': [0] }, { 'targets': [6], - 'visible': false, 'searchable': false }, ] @@ -305,14 +304,15 @@ function addLastFromJson(datatable, url) { } function addLastContributor(datatable, data, update) { + var date = new Date(data.epoch*1000); var to_add = [ + date.toTimeString().slice(0,-15) +' '+ date.toLocaleDateString(), data.pnts, getMonthlyRankIcon(data.rank), getOrgRankIcon(data.orgRank, 60), createHonorImg(data.honorBadge, 20), createImg(data.logo_path, 32), createOrgLink(data.org), - data.epoch ]; if (update == undefined || update == false) { datatable.row.add(to_add); @@ -398,7 +398,7 @@ function updateProgressHeader(org) { }); datatableLast.rows().every( function() { var row = this.node(); - if(this.data()[5] == data.org) { row.classList.add('selectedOrgInTable'); } else { row.classList.remove('selectedOrgInTable'); } + if(this.data()[6] == data.org) { row.classList.add('selectedOrgInTable'); } else { row.classList.remove('selectedOrgInTable'); } }); }); diff --git a/templates/contrib.html b/templates/contrib.html index d380806..d8c96bd 100644 --- a/templates/contrib.html +++ b/templates/contrib.html @@ -263,13 +263,13 @@
    Org. rankContribution rank TitlePoints requirementTotal points requirement Contribution requirement
    {{ item[1] }}{{ item[1] }}
    + - From 0080de3d8e1a9dbf555ad3340336dc7a063d50d4 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Fri, 10 Nov 2017 10:32:51 +0100 Subject: [PATCH 56/65] better display for badges and time --- static/css/ranking.css | 8 +++++--- static/js/contrib.js | 14 +++++++++++--- templates/contrib.html | 2 +- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/static/css/ranking.css b/static/css/ranking.css index f744290..a5a617a 100644 --- a/static/css/ranking.css +++ b/static/css/ranking.css @@ -84,14 +84,16 @@ .table > tbody > tr > td.centerCellPicOrgLogo { padding: 5px; - text-align: right; - min-width: 45px; + text-align: left; +} + +.table > tbody > tr > td.verticalAlign { + vertical-align: middle; } .table > tbody > tr > td.centerCellPicOrgRank { padding: 0px; text-align: left; - min-width: 45px; } .progress { diff --git a/static/js/contrib.js b/static/js/contrib.js index b1b183f..13013ce 100644 --- a/static/js/contrib.js +++ b/static/js/contrib.js @@ -66,9 +66,13 @@ var optionDatatable_light = { var optionDatatable_top = jQuery.extend({}, optionDatatable_light) var optionDatatable_last = jQuery.extend({}, optionDatatable_light) optionDatatable_last.columnDefs = [ - { className: "centerCellPicOrgRank", "targets": [ 2 ] }, + { className: "small", "targets": [ 0 ] }, + { className: "verticalAlign", "targets": [ 1 ] }, + { className: "centerCellPicOrgRank verticalAlign", "targets": [ 2 ] }, { className: "centerCellPicOrgLogo", "targets": [ 3 ] }, - { className: "centerCellPicOrgLogo", "targets": [ 4 ] }, + { className: "centerCellPicOrgLogo verticalAlign", "targets": [ 4 ] }, + { className: "centerCellPicOrgLogo verticalAlign", "targets": [ 5 ] }, + { className: "verticalAlign", "targets": [ 6 ] }, { 'orderData':[6], 'targets': [0] }, { 'targets': [6], @@ -169,6 +173,8 @@ function createImg(source, size) { function createHonorImg(array, size) { size = 32; var div = document.createElement('div'); + div.style.boxShadow = '0px 0px 5px #00000099'; + div.style.backgroundColor = '#e1e1e1'; for (badgeNum of array) { var obj = document.createElement('img'); obj.height = size; @@ -178,6 +184,8 @@ function createHonorImg(array, size) { obj.src = url_baseHonorLogo+badgeNum+'.svg'; div.appendChild(obj); } + div.style.width = 32*array.length+'px'; + div.style.borderRadius = '15px'; return div.outerHTML; } @@ -555,7 +563,7 @@ $(document).ready(function() { }); if(currOrg != "") // currOrg selected //FIXME: timeout used to wait that all datatables are draw. - setTimeout( function() { updateProgressHeader(currOrg); }, 400); + setTimeout( function() { updateProgressHeader(currOrg); }, 500); source_lastContrib = new EventSource(url_eventStreamLastContributor); source_lastContrib.onmessage = function(event) { diff --git a/templates/contrib.html b/templates/contrib.html index d8c96bd..333f427 100644 --- a/templates/contrib.html +++ b/templates/contrib.html @@ -263,7 +263,7 @@
    date/time Points Cur. rank Org. rank Organisationepoch
    - + From e554158467af70babbb4d33fc5626f7abe07f622 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Fri, 10 Nov 2017 10:40:56 +0100 Subject: [PATCH 57/65] Slight modification to point reward --- config/config.cfg.default | 2 +- zmq_subscriber.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/config.cfg.default b/config/config.cfg.default index 8bfee3e..51cf3bc 100644 --- a/config/config.cfg.default +++ b/config/config.cfg.default @@ -27,7 +27,7 @@ rankMultiplier = 2 categories_in_datatable = ["internal_reference", "targeting_data", "antivirus_detection", "payload_delivery", "artifacts_dropped", "payload_installation", "persistence_mechanism", "network_activity", "payload_type", "attribution", "external_analysis", "financial_fraud", "support_Tool", "social_network", "person", "other" ] default_pnts_per_contribution = 1 # array of the form [[category, pntsRcv], ...] -pnts_per_contribution = [["payload_delivery", 5], ["artifact_dropped", 20], ["network_activity", 5]] +pnts_per_contribution = [["payload_delivery", 1], ["artifact_dropped", 1], ["network_activity", 1]] additional_help_text = ["Sightings multiplies earned points by 2", "Editing an attribute earns you the same as creating one"] [Log] diff --git a/zmq_subscriber.py b/zmq_subscriber.py index 5b6ca3d..1947d7c 100755 --- a/zmq_subscriber.py +++ b/zmq_subscriber.py @@ -149,7 +149,7 @@ def handleContribution(zmq_name, org, contribType, categ, action, pntMultiplier= push_to_redis_zset('CONTRIB_DAY', org, count=pnts_to_add) #CONTRIB_CATEG retain the contribution per category, not the point earned in this categ - push_to_redis_zset('CONTRIB_CATEG', org, count=DEFAULT_PNTS_REWARD, endSubkey=':'+noSpaceLower(categ)) + push_to_redis_zset('CONTRIB_CATEG', org, count=1, endSubkey=':'+noSpaceLower(categ)) publish_log(zmq_name, 'CONTRIBUTION', {'org': org, 'categ': categ, 'action': action, 'epoch': nowSec }, channel=CHANNEL_LASTCONTRIB) serv_redis_db.sadd('CONTRIB_ALL_ORG', org) From 6b851441649372d4710ce42fd12f50e5c776ac4c Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Fri, 10 Nov 2017 11:15:31 +0100 Subject: [PATCH 58/65] Updated org contrib. overtime to display the date --- contributor_helper.py | 12 ++++++------ static/js/contrib.js | 4 ++-- templates/contrib.html | 2 +- util.py | 5 ++++- 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/contributor_helper.py b/contributor_helper.py index 51f2f95..86feef9 100644 --- a/contributor_helper.py +++ b/contributor_helper.py @@ -292,21 +292,21 @@ class Contributor_helper: # show current top 5 org points overtime (last 5 days) for dic in topSortedOrg[0:5]: org = dic['org'] - overtime = [] to_append = self.getOrgOvertime(org) data.append(to_append) return data def getOrgOvertime(self, org): overtime = [] - today = datetime.datetime.now() - for deltaD in range(1,6,1): - date = (datetime.datetime(today.year, today.month, today.day) - datetime.timedelta(days=deltaD)) - keyname = 'CONTRIB_DAY:'+util.getDateStrFormat(date) + today = datetime.datetime.today() + today = today.replace(hour=0, minute=0, second=0, microsecond=0) + for curDate in util.getXPrevDaysSpan(today, 7): + timestamp = util.getTimestamp(curDate) + keyname = 'CONTRIB_DAY:'+util.getDateStrFormat(curDate) org_score = self.serv_redis_db.zscore(keyname, org) if org_score is None: org_score = 0 - overtime.append([deltaD, org_score]) + overtime.append([timestamp, org_score]) to_return = {'label': org, 'data': overtime} return to_return diff --git a/static/js/contrib.js b/static/js/contrib.js index 13013ce..4f9ee03 100644 --- a/static/js/contrib.js +++ b/static/js/contrib.js @@ -469,7 +469,7 @@ function updateProgressHeader(org) { // transform secs into date var new_data = []; for(list of data['data']) { - new_data.push([new Date(list[0]*1000), list[1]]); + new_data.push([list[0]*1000, list[1]]); } data['data'] = new_data; toPlot.push(data); @@ -554,7 +554,7 @@ $(document).ready(function() { for(i in data){ var new_data = []; for(list of data[i]['data']) { - new_data.push([new Date(list[0]*1000), list[1]]); + new_data.push([list[0]*1000, list[1]]); } data[i]['data'] = new_data; } diff --git a/templates/contrib.html b/templates/contrib.html index 333f427..5d0f91d 100644 --- a/templates/contrib.html +++ b/templates/contrib.html @@ -284,7 +284,7 @@
    - Contributors and Categories (total) + Contributors and categories (total)
    diff --git a/util.py b/util.py index ba794f1..8518554 100644 --- a/util.py +++ b/util.py @@ -1,4 +1,4 @@ -import datetime +import datetime, time ONE_DAY = 60*60*24 @@ -26,3 +26,6 @@ def getXPrevDaysSpan(date, days): def getDateStrFormat(date): return str(date.year)+str(date.month).zfill(2)+str(date.day).zfill(2) + +def getTimestamp(date): + return time.mktime(date.timetuple()) From f8e1725a2257cffde666b21fe602e138bc372606 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Fri, 10 Nov 2017 11:50:32 +0100 Subject: [PATCH 59/65] Updated dependencies install script --- install_dependencies.sh | 22 ++++++++++++++++++---- templates/contrib.html | 7 ++++--- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/install_dependencies.sh b/install_dependencies.sh index b2d770a..a2e3e81 100755 --- a/install_dependencies.sh +++ b/install_dependencies.sh @@ -23,7 +23,7 @@ fi ## Web stuff pushd static/ -mkdir -p css fonts +mkdir -p css fonts js popd mkdir -p temp @@ -36,9 +36,9 @@ wget http://code.jquery.com/jquery-${JQVERSION}.min.js -O ./static/js/jquery.min FLOTVERSION="0.8.3" wget http://www.flotcharts.org/downloads/flot-${FLOTVERSION}.zip -O ./temp/flot-${FLOTVERSION}.zip unzip -o temp/flot-${FLOTVERSION}.zip -d temp/ -mv temp/flot/jquery.flot.js ./static/js -mv temp/flot/jquery.flot.pie.min.js ./static/js -mv temp/flot/jquery.flot.resize.js ./static/js +mv temp/flot/jquery.flot.js ./static/js +mv temp/flot/jquery.flot.pie.min.js ./static/js +mv temp/flot/jquery.flot.resize.js ./static/js JQUERYUIVERSION="1.12.1" @@ -82,9 +82,23 @@ mv temp/jquery-jvectormap-2.0.3.css ./static/css mv temp/jquery-jvectormap-2.0.3.min.js ./static/js wget http://jvectormap.com/js/jquery-jvectormap-world-mill.js -O ./static/js/jquery-jvectormap-world-mill.js +# maxmind DB mkdir data pushd data wget http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz -O GeoLite2-City.tar.gz tar xvfz GeoLite2-City.tar.gz rm -rf GeoLite2-City.tar.gz + +# DataTable +DATATABLE_VERSION="1.10.16" +wget https://datatables.net/download/builder?dt/dt-${DATATABLE_VERSION}.zip -O temp/DataTables.zip +unzip -o temp/DataTables.zip -d temp/ +mv temp/DataTables/${DATATABLE_VERSION}/css/dataTables.bootstrap.css ./static/css +mv temp/DataTables/${DATATABLE_VERSION}/js/jquery.dataTables.min.js ./static/js +mv temp/DataTables/${DATATABLE_VERSION}/js/dataTables.bootstrap.js ./static/js + +#typeahead +git clone https://github.com/bassjobsen/Bootstrap-3-Typeahead.git temp/ +mv temp/Bootstrap-3-Typeahead/bootstrap3-typeahead.min.js ./static/js + rm -rf ./temp diff --git a/templates/contrib.html b/templates/contrib.html index 5d0f91d..949806a 100644 --- a/templates/contrib.html +++ b/templates/contrib.html @@ -29,17 +29,18 @@ - + + + + - - From 254141ecb1b5c636e671a29a9952408cb81e93c4 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Fri, 10 Nov 2017 13:25:38 +0100 Subject: [PATCH 60/65] Updating install script --- install_dependencies.sh | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/install_dependencies.sh b/install_dependencies.sh index a2e3e81..c9d598c 100755 --- a/install_dependencies.sh +++ b/install_dependencies.sh @@ -21,6 +21,8 @@ if [ ! -f config/config.cfg ]; then cp config/config.cfg.sample config/config.cfg fi +cp -i config/config.cfg.default config/config.cfg + ## Web stuff pushd static/ mkdir -p css fonts js @@ -33,14 +35,16 @@ wget http://www.misp-project.org/assets/images/misp-small.png -O static/pics/MIS JQVERSION="3.2.1" wget http://code.jquery.com/jquery-${JQVERSION}.min.js -O ./static/js/jquery.min.js +# jquery flot FLOTVERSION="0.8.3" wget http://www.flotcharts.org/downloads/flot-${FLOTVERSION}.zip -O ./temp/flot-${FLOTVERSION}.zip unzip -o temp/flot-${FLOTVERSION}.zip -d temp/ mv temp/flot/jquery.flot.js ./static/js mv temp/flot/jquery.flot.pie.min.js ./static/js mv temp/flot/jquery.flot.resize.js ./static/js +mv temp/flot/jquery.flot.time.js ./static/js - +# jquery UI JQUERYUIVERSION="1.12.1" wget https://jqueryui.com/resources/download/jquery-ui-${JQUERYUIVERSION}.zip -O temp/jquery-ui.zip unzip -o temp/jquery-ui.zip -d temp/ @@ -64,6 +68,8 @@ unzip -o temp/${SBADMIN_VERSION}-2.zip -d temp/ mv temp/startbootstrap-sb-admin-2-${SBADMIN_VERSION}/dist/js/* ./static/js/ mv temp/startbootstrap-sb-admin-2-${SBADMIN_VERSION}/dist/css/* ./static/css/ mv temp/startbootstrap-sb-admin-2-${SBADMIN_VERSION}/bower_components/font-awesome/fonts/* ./static/fonts +mv temp/startbootstrap-sb-admin-2-${SBADMIN_VERSION}/bower_components/font-awesome/css/* ./static/css +mv temp/startbootstrap-sb-admin-2-${SBADMIN_VERSION}/bower_components/bootstrap/fonts/* ./static/fonts # leaflet LEAFLET_VERSION="1.2.0" @@ -88,17 +94,16 @@ pushd data wget http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz -O GeoLite2-City.tar.gz tar xvfz GeoLite2-City.tar.gz rm -rf GeoLite2-City.tar.gz +popd # DataTable DATATABLE_VERSION="1.10.16" -wget https://datatables.net/download/builder?dt/dt-${DATATABLE_VERSION}.zip -O temp/DataTables.zip -unzip -o temp/DataTables.zip -d temp/ -mv temp/DataTables/${DATATABLE_VERSION}/css/dataTables.bootstrap.css ./static/css -mv temp/DataTables/${DATATABLE_VERSION}/js/jquery.dataTables.min.js ./static/js -mv temp/DataTables/${DATATABLE_VERSION}/js/dataTables.bootstrap.js ./static/js +wget https://cdn.datatables.net/${DATATABLE_VERSION}/js/jquery.dataTables.min.js -O ./static/js/jquery.dataTables.min.js +wget https://cdn.datatables.net/${DATATABLE_VERSION}/css/dataTables.bootstrap.css -O ./static/css/dataTables.bootstrap.css +wget https://cdn.datatables.net/${DATATABLE_VERSION}/js/dataTables.bootstrap.js -O ./static/js/dataTables.bootstrap.js #typeahead -git clone https://github.com/bassjobsen/Bootstrap-3-Typeahead.git temp/ +git clone https://github.com/bassjobsen/Bootstrap-3-Typeahead.git temp/Bootstrap-3-Typeahead mv temp/Bootstrap-3-Typeahead/bootstrap3-typeahead.min.js ./static/js rm -rf ./temp From df53a51b66580cc873d45007c29167d7a51876f4 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Fri, 10 Nov 2017 13:39:59 +0100 Subject: [PATCH 61/65] removed initial config.cfg --- config/config.cfg | 44 -------------------------------------------- 1 file changed, 44 deletions(-) delete mode 100644 config/config.cfg diff --git a/config/config.cfg b/config/config.cfg deleted file mode 100644 index 4d140cd..0000000 --- a/config/config.cfg +++ /dev/null @@ -1,44 +0,0 @@ -[Dashboard] -#hours -graph_log_refresh_rate = 1 -#sec -rotation_wait_time = 30 -max_img_rotation = 10 -hours_spanned = 48 -zoomlevel = 15 -item_to_plot = Attribute.category -# [1->12] -size_dashboard_left_width = 5 -size_openStreet_pannel_perc = 55 -size_world_pannel_perc = 35 - -[GEO] -#min -updateFrequency = 60 -zoomlevel = 11 -# ~meter -clusteringDistance = 10 - -[Log] -field_to_plot = Attribute.category -fieldname_order=["Event.id", "Attribute.Tag", "Attribute.category", "Attribute.type", ["Attribute.value", "Attribute.comment"]] -char_separator=|| - -[RedisGlobal] -host=localhost -port=6251 - -[RedisLog] -db=0 -channel=1 -#zmq_url=tcp://localhost:50000 -zmq_url=tcp://192.168.56.50:50000 - -[RedisMap] -db=1 -channelProc=CoordToProcess -channelDisp=PicToDisplay -pathMaxMindDB=./data/GeoLite2-City_20171003/GeoLite2-City.mmdb - -[RedisDB] -db=2 From 98e849f39e329ac8d6e0185acecffab095889245 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Fri, 10 Nov 2017 13:46:23 +0100 Subject: [PATCH 62/65] Added CP interactivity --- install_dependencies.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/install_dependencies.sh b/install_dependencies.sh index c9d598c..525dca8 100755 --- a/install_dependencies.sh +++ b/install_dependencies.sh @@ -17,10 +17,6 @@ fi pip3 install -U pip argparse redis zmq geoip2 flask ## config -if [ ! -f config/config.cfg ]; then - cp config/config.cfg.sample config/config.cfg -fi - cp -i config/config.cfg.default config/config.cfg ## Web stuff From 158532aa863f3d8ebe55b0c5f7465d10ea1935c8 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Fri, 10 Nov 2017 14:10:48 +0100 Subject: [PATCH 63/65] Prevent duplication of org in Lsat contributors --- contributor_helper.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/contributor_helper.py b/contributor_helper.py index 86feef9..f1e4a83 100644 --- a/contributor_helper.py +++ b/contributor_helper.py @@ -231,10 +231,13 @@ class Contributor_helper: keyname = "CONTRIB_LAST" prev_days = 7 topNum = self.MAX_NUMBER_OF_LAST_CONTRIBUTOR # default Num + addedOrg = [] data = [] for curDate in util.getXPrevDaysSpan(date, prev_days): last_contrib_org = self.getZrange(keyname, curDate, topNum) for org, sec in last_contrib_org: + if org in addedOrg: + continue dic = {} dic['rank'] = self.getOrgRankFromRedis(org, date) dic['orgRank'] = self.getOrgContributionRank(org)['final_rank'] @@ -244,6 +247,7 @@ class Contributor_helper: dic['pnts'] = self.getOrgPntFromRedis(org, date) dic['epoch'] = sec data.append(dic) + addedOrg.append(org) return data From 9291c346999338bb7b3a5307aec4f23d6006bcf4 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Fri, 10 Nov 2017 14:33:10 +0100 Subject: [PATCH 64/65] Added support of Discusison --- zmq_subscriber.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/zmq_subscriber.py b/zmq_subscriber.py index 1947d7c..35adc2e 100755 --- a/zmq_subscriber.py +++ b/zmq_subscriber.py @@ -178,6 +178,21 @@ def handler_keepalive(zmq_name, jsonevent): to_push = [ jsonevent['uptime'] ] publish_log(zmq_name, 'Keepalive', to_push) +def handler_conversation(zmq_name, jsonevent): + try: #only consider POST, not THREAD + jsonpost = jsonevent['Post'] + except KeyError: + return + print('sending' ,'Post') + org = jsonpost['org_name'] + categ = None + action = 'add' + handleContribution(zmq_name, org, + 'Discussion', + None, + action, + isLabeled=False) + def handler_object(zmq_name, jsondata): print('obj') return @@ -300,7 +315,7 @@ dico_action = { "misp_json_sighting": handler_sighting, "misp_json_organisation": handler_log, "misp_json_user": handler_log, - "misp_json_conversation": handler_log + "misp_json_conversation": handler_conversation } From e071975effb90a8801e75f67a851131c9d1f72e4 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Fri, 10 Nov 2017 15:03:33 +0100 Subject: [PATCH 65/65] Added count for proposal and sighting --- contributor_helper.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/contributor_helper.py b/contributor_helper.py index f1e4a83..c8009dc 100644 --- a/contributor_helper.py +++ b/contributor_helper.py @@ -68,6 +68,11 @@ class Contributor_helper: data = [ [record[0].decode('utf8'), record[1]] for record in data ] return data + def addContributionToCateg(self, date, categ, org, count=1): + today_str = util.getDateStrFormat(date) + keyname = "CONTRIB_CATEG:{}:{}".format(today_str, categ) + self.serv_redis_db.zincrby(keyname, org, count) + ''' CONTRIBUTION RANK ''' def getOrgContributionTotalPoints(self, org): keyname = 'CONTRIB_ORG:{org}:{orgCateg}' @@ -133,10 +138,15 @@ class Contributor_helper: if contribType == 'Proposal': proposalWeekCount = self.serv_redis_db.incrby(keyname.format(org=orgName, orgCateg='PROP_WEEK_COUNT'), 1) self.serv_redis_db.expire(keyname.format(org=orgName, orgCateg='PROP_WEEK_COUNT'), util.ONE_DAY*7) + addContributionToCateg(datetime.datetime.now(), 'proposal') if contribType == 'Sighting': sightingWeekCount = self.serv_redis_db.incrby(keyname.format(org=orgName, orgCateg='SIGHT_WEEK_COUNT'), 1) self.serv_redis_db.expire(keyname.format(org=orgName, orgCateg='SIGHT_WEEK_COUNT'), util.ONE_DAY*7) + self.addContributionToCateg(datetime.datetime.now(), 'sighting', orgName) + + if contribType == 'Discussion': + self.addContributionToCateg(datetime.datetime.now(), 'discussion', orgName) if contribType == 'Event': eventWeekCount = self.serv_redis_db.incrby(keyname.format(org=orgName, orgCateg='EVENT_WEEK_COUNT'), 1)
    date/timeDate Points Cur. rank Org. rank