From 97a90207160af2fbd8f14bda3d62a50395ce3c0d Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Thu, 9 Nov 2017 10:32:39 +0100 Subject: [PATCH] Added draft support of org overtime + Added clickable orgName + minor fix in live dashbaord + draft support of sighting in zmq_sub --- contributor_helper.py | 19 +++++++++-- server.py | 8 +++++ static/js/contrib.js | 66 +++++++++++++++++++++++++++++------- static/js/index/index_map.js | 10 +++--- templates/contrib.html | 8 +++-- zmq_subscriber.py | 28 +++++++++------ 6 files changed, 106 insertions(+), 33 deletions(-) diff --git a/contributor_helper.py b/contributor_helper.py index ed675a3..670267b 100644 --- a/contributor_helper.py +++ b/contributor_helper.py @@ -336,12 +336,27 @@ class Contributor_helper: return data2 def TEST_getTop5OvertimeFromRedis(self): + import time + now = time.time() + ONE_DAY = 60*60*24 data2 = [ - {'label': 'CIRCL', 'data': [[0, 4], [1, 7], [2,14]]}, - {'label': 'CASES', 'data': [[0, 1], [1, 5], [2,2]]} + {'label': 'CIRCL', 'data': [[now, random.randint(1,50)], [now-ONE_DAY, random.randint(1,50)], [now-ONE_DAY*2, random.randint(1,50)], [now-ONE_DAY*3, random.randint(1,50)], [now-ONE_DAY*4, random.randint(1,50)]]}, + {'label': 'CASES', 'data': [[now, random.randint(1,50)], [now-ONE_DAY, random.randint(1,50)], [now-ONE_DAY*2, random.randint(1,50)], [now-ONE_DAY*3, random.randint(1,50)], [now-ONE_DAY*4, random.randint(1,50)]]}, + {'label': 'Org1', 'data': [[now, random.randint(1,50)], [now-ONE_DAY, random.randint(1,50)], [now-ONE_DAY*2, random.randint(1,50)], [now-ONE_DAY*3, random.randint(1,50)], [now-ONE_DAY*4, random.randint(1,50)]]}, + {'label': 'Org2', 'data': [[now, random.randint(1,50)], [now-ONE_DAY, random.randint(1,50)], [now-ONE_DAY*2, random.randint(1,50)], [now-ONE_DAY*3, random.randint(1,50)], [now-ONE_DAY*4, random.randint(1,50)]]}, + {'label': 'SMILE', 'data': [[now, random.randint(1,50)], [now-ONE_DAY, random.randint(1,50)], [now-ONE_DAY*2, random.randint(1,50)], [now-ONE_DAY*3, random.randint(1,50)], [now-ONE_DAY*4, random.randint(1,50)]]}, ] return data2 + def TEST_getOrgOvertime(self, org): + import time + now = time.time() + ONE_DAY = 60*60*24 + data = [ + {'label': org, 'data': [[now, random.randint(1,30)], [now-ONE_DAY, random.randint(1,30)], [now-ONE_DAY*2, random.randint(1,30)], [now-ONE_DAY*3, random.randint(1,30)], [now-ONE_DAY*4, random.randint(1,40)]]} + ] + return data + def TEST_getTopContributorFromRedis(self, date): data2 = [ { diff --git a/server.py b/server.py index 042b83b..0460040 100755 --- a/server.py +++ b/server.py @@ -331,6 +331,14 @@ def getFameContributor(): def getTop5Overtime(): return jsonify(contributor_helper.TEST_getTop5OvertimeFromRedis()) +@app.route("/_getOrgOvertime") +def getOrgOvertime(): + try: + org = request.args.get('org') + except: + org = '' + return jsonify(contributor_helper.TEST_getOrgOvertime(org)) + @app.route("/_getCategPerContrib") def getCategPerContrib(): try: diff --git a/static/js/contrib.js b/static/js/contrib.js index 21aab6d..79299a2 100644 --- a/static/js/contrib.js +++ b/static/js/contrib.js @@ -5,6 +5,8 @@ var datatableFame; var refresh_speed = min_between_reload*60; var will_reload = $("#reloadCheckbox").is(':checked'); var sec_before_reload = refresh_speed; +var dataTop5Overtime; +var plotLineChart /* CONFIG */ var maxRank = 16; @@ -24,6 +26,7 @@ var optionsLineChart = { } } }, + colors: ["#2F4F4F", "#778899", "#696969", "#A9A9A9", "#D3D3D3", "#337ab7"], points: { show: true }, lines: { show: true, fill: true }, grid: { @@ -33,6 +36,11 @@ var optionsLineChart = { legend: { show: true, position: "nw" + }, + xaxis: { + mode: "time", + timeformat: "%m/%d", + minTickSize: [1, "day"] } }; var optionDatatable_light = { @@ -171,6 +179,13 @@ function createHonorImg(array, size) { return div.outerHTML; } +function createOrgLink(org) { + var a = document.createElement('a'); + a.innerHTML = org; + a.href = "?org="+org; + return a.outerHTML; +} + function generateRankingSheet(rank, rankDec, stepPnt, pnt, Rpnt) { var Cpnt = pnt - stepPnt; var Tpnt = Cpnt + Rpnt; @@ -267,7 +282,7 @@ function addToTableFromJson(datatable, url) { getOrgRankIcon(row.orgRank, 60), createHonorImg(row.honorBadge, 20), createImg(row.logo_path, 32), - row.org + createOrgLink(row.org) ]; datatable.row.add(to_add); } @@ -293,7 +308,7 @@ function addLastContributor(datatable, data, update) { getOrgRankIcon(data.orgRank, 60), createHonorImg(data.honorBadge, 20), createImg(data.logo_path, 32), - data.org, + createOrgLink(data.org), data.epoch ]; if (update == undefined || update == false) { @@ -347,7 +362,6 @@ function updateProgressHeader(org) { // colorize row contribution rank help $.getJSON( url_getContributionOrgStatus+'?org='+org, function( data ) { var status = data['status']; - console.log(data); var curContributionOrgRank = data['rank']; var totNumPoints = data['totPoints'] $('#orgTotNumOfPoint').text(totNumPoints); @@ -386,23 +400,42 @@ function updateProgressHeader(org) { }); //update overtake points - var prevOrg = ""; + var prevOrgName = ""; var prevOrgPnts = 0; datatableTop.rows().every( function() { var row = this.node(); - if(this.data()[5] == currOrg) { - if(prevOrg == ""){ //already first + var orgRowName = $(this.data()[5])[0].text; // contained in + var orgRowPnts = this.data()[0] + if(orgRowName == currOrg) { + if(prevOrgName == ""){ //already first $('#orgToOverTake').text(this.data()[5]); $('#pntsToOvertakeNext').text(0); } else { - $('#orgToOverTake').text(prevOrg); - $('#pntsToOvertakeNext').text(parseInt(prevOrgPnts)-this.data()[0]); + $('#orgToOverTake').text(prevOrgName); + $('#pntsToOvertakeNext').text(parseInt(prevOrgPnts)-orgRowPnts); } } else { - prevOrg = this.data()[5]; - prevOrgPnts = this.data()[0]; + prevOrgName = orgRowName; + prevOrgPnts = orgRowPnts; } }); + + //Add new data to linechart + $.getJSON( url_getOrgOvertime+'?org='+org, function( data ) { + var toPlot = dataTop5Overtime.slice(0); //cloning data + // transform secs into date + for(i in data){ + var new_data = []; + for(list of data[i]['data']) { + new_data.push([new Date(list[0]*1000), list[1]]); + } + data[i]['data'] = new_data; + toPlot.push(data[i]); + } + plotLineChart.setData(toPlot); + plotLineChart.setupGrid(); + plotLineChart.draw(); + }); } function showOnlyOrg() { @@ -462,7 +495,7 @@ $(document).ready(function() { getOrgRankIcon(row.orgRank, 44), createHonorImg(row.honorBadge, 20), createImg(row.logo_path, 32), - row.org, + createOrgLink(row.org), ]; for (categ of categ_list) { to_add.push(row[categ]); @@ -474,7 +507,16 @@ $(document).ready(function() { }); // top 5 contrib overtime $.getJSON( url_getTop5Overtime, function( data ) { - var plotLineChart = $.plot("#divTop5Overtime", data, optionsLineChart); + // transform secs into date + for(i in data){ + var new_data = []; + for(list of data[i]['data']) { + new_data.push([new Date(list[0]*1000), list[1]]); + } + data[i]['data'] = new_data; + } + dataTop5Overtime = data; + plotLineChart = $.plot("#divTop5Overtime", data, optionsLineChart); }); if(currOrg != "") // currOrg selected //FIXME: timeout used to wait that all datatables are draw. diff --git a/static/js/index/index_map.js b/static/js/index/index_map.js index ae83f0e..c5740bf 100644 --- a/static/js/index/index_map.js +++ b/static/js/index/index_map.js @@ -125,7 +125,7 @@ class MapEventManager { // Add and Manage markers on the map + make Animation popupCoord(coord, regionCode) { var coord = [coord.lat, coord.lon]; - var color = Math.random()*180; + var color = 0.5*180; var pnts = openStreetMapObj.latLngToPoint(coord[0], coord[1]) if (pnts != false) { //sometimes latLngToPoint return false var addedMarker = openStreetMapObj.addMarker(this._curMarkerNum, coord, [color]); @@ -178,7 +178,8 @@ $(function(){ series: { markers: [{ attribute: 'fill', - scale: ['#1A0DAB', '#e50000', '#62ff41'], + //scale: ['#1A0DAB', '#e50000', '#62ff41'], + scale: ['#ffff66'], values: [], min: 0, max: 180 @@ -207,10 +208,10 @@ function connect_source_map() { var marker = L.marker([json.coord.lat, json.coord.lon]).addTo(myOpenStreetMap); var mapEvent = new MapEvent(json, marker); mapEventManager.addMapEvent(mapEvent); - + }; source_map.onopen = function(){ - console.log('connection is opened. '+source_map.readyState); + console.log('connection is opened. '+source_map.readyState); }; source_map.onerror = function(){ console.log('error: '+source_map.readyState); @@ -240,4 +241,3 @@ $(document).ready(function () { mapEventManager.directZoom(); }); }); - diff --git a/templates/contrib.html b/templates/contrib.html index 3567745..5a8a273 100644 --- a/templates/contrib.html +++ b/templates/contrib.html @@ -24,6 +24,7 @@ + @@ -225,7 +226,7 @@
- Contributor Ranking + Contributor ranking (monthly)
@@ -283,7 +284,7 @@
- Contributors and Categories + Contributors and Categories (total)
@@ -320,7 +321,7 @@
- Hall Of Fame + Hall Of Fame (previous month)
@@ -375,6 +376,7 @@ var url_getFameContributor = "{{ url_for('getFameContributor') }}"; var url_getCategPerContrib = "{{ url_for('getCategPerContrib') }}"; var url_getTop5Overtime = "{{ url_for('getTop5Overtime') }}"; + var url_getOrgOvertime = "{{ url_for('getOrgOvertime') }}"; var url_getLastContributor = "{{ url_for('getLastContributors') }}"; var url_eventStreamLastContributor = "{{ url_for('getLastContributor') }}"; var url_getAllOrg = "{{ url_for('getAllOrg') }}"; diff --git a/zmq_subscriber.py b/zmq_subscriber.py index eb6bfd9..f927d0b 100755 --- a/zmq_subscriber.py +++ b/zmq_subscriber.py @@ -86,13 +86,16 @@ def getCoordAndPublish(zmq_name, supposed_ip, categ): rep = ip_to_coord(supposed_ip) coord = rep['coord'] coord_dic = {'lat': coord['lat'], 'lon': coord['lon']} - ordDic = OrderedDict() + ordDic = OrderedDict() #keep fields with the same layout in redis ordDic['lat'] = coord_dic['lat'] ordDic['lon'] = coord_dic['lon'] coord_list = [coord['lat'], coord['lon']] push_to_redis_zset('GEO_COORD', json.dumps(ordDic)) push_to_redis_zset('GEO_COUNTRY', rep['full_rep'].country.iso_code) - push_to_redis_geo('GEO_RAD', coord['lon'], coord['lat'], json.dumps({ 'categ': categ, 'value': supposed_ip })) + ordDic = OrderedDict() #keep fields with the same layout in redis + ordDic['categ'] = categ + ordDic['value'] = supposed_ip + push_to_redis_geo('GEO_RAD', coord['lon'], coord['lat'], json.dumps(ordDic)) to_send = { "coord": coord, "categ": categ, @@ -170,13 +173,14 @@ def handler_keepalive(zmq_name, jsonevent): to_push = [ jsonevent['uptime'] ] publish_log(zmq_name, 'Keepalive', to_push) -def handler_sighting(zmq_name, jsonsight): +def handler_sighting(zmq_name, jsondata): print('sending' ,'sighting') - org = jsonsight['org'] - categ = jsonsight['categ'] - action = jsonsight['action'] + jsonsight = jsondata['Sighting'] + org = jsonsight['Event']['Orgc']['name'] + categ = jsonsight['Attribute']['category'] + action = jsondata['action'] handleContribution(zmq_name, org, categ, action, pntMultiplier=2) - return + handler_attribute(zmq_name, jsonsight, hasAlreadyBeenContributed=True) def handler_event(zmq_name, jsonobj): #fields: threat_level_id, id, info @@ -192,7 +196,7 @@ def handler_event(zmq_name, jsonobj): else: handler_attribute(zmq_name, attributes) -def handler_attribute(zmq_name, jsonobj): +def handler_attribute(zmq_name, jsonobj, hasAlreadyBeenContributed=False): # check if jsonattr is an attribute object if 'Attribute' in jsonobj: jsonattr = jsonobj['Attribute'] @@ -212,9 +216,10 @@ def handler_attribute(zmq_name, jsonobj): if jsonattr['category'] == "Network activity": getCoordAndPublish(zmq_name, jsonattr['value'], jsonattr['category']) - eventLabeled = False - #eventLabeled = len(jsonattr[]) > 0 - handleContribution(zmq_name, jsonobj['Event']['Orgc']['name'], jsonattr['category'], jsonobj['action'], isLabeled=eventLabeled) + if not hasAlreadyBeenContributed: + eventLabeled = False + #eventLabeled = len(jsonattr[]) > 0 + handleContribution(zmq_name, jsonobj['Event']['Orgc']['name'], jsonattr['category'], jsonobj['action'], isLabeled=eventLabeled) # Push to log publish_log(zmq_name, 'Attribute', to_push) @@ -227,6 +232,7 @@ def process_log(zmq_name, event): event = event.decode('utf8') topic, eventdata = event.split(' ', maxsplit=1) jsonevent = json.loads(eventdata) + print(event) dico_action[topic](zmq_name, jsonevent)