From cba6d260ff163aa036817cb9a35c570a310ad5ae Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Mon, 6 Nov 2017 13:43:55 +0100 Subject: [PATCH] Added draft of automatic update for lastContributor --- config/config.cfg.default | 1 + contributor_helper.py | 20 ++++++++++++--- server.py | 21 ++++++++++++++-- static/js/contrib.js | 52 +++++++++++++++++++++++++++++++++++++-- templates/contrib.html | 4 ++- zmq_subscriber.py | 13 ++++++---- 6 files changed, 97 insertions(+), 14 deletions(-) diff --git a/config/config.cfg.default b/config/config.cfg.default index 4a568fb..a20434b 100644 --- a/config/config.cfg.default +++ b/config/config.cfg.default @@ -40,6 +40,7 @@ port=6251 [RedisLog] db=0 channel=1 +channelLastContributor = lastContributor zmq_url=tcp://localhost:50000 #zmq_url=tcp://192.168.56.50:50000 diff --git a/contributor_helper.py b/contributor_helper.py index 26c890a..5039ba9 100644 --- a/contributor_helper.py +++ b/contributor_helper.py @@ -40,11 +40,11 @@ class Contributor_helper: def getOrgLogoFromRedis(self, org): return 'logo_'+org - def getLastContributorFromRedis(self): + def getLastContributorsFromRedis(self): date = datetime.datetime.now() - keyCateg = "CONTRIB_LAST" + keyname = "CONTRIB_LAST" topNum = self.MAX_NUMBER_OF_LAST_CONTRIBUTOR # default Num - last_contrib_org = self.getZrange(keyCateg, date, topNum) + last_contrib_org = self.getZrange(keyname, date, topNum) data = [] for org, sec in last_contrib_org: dic = {} @@ -52,9 +52,21 @@ class Contributor_helper: dic['logo_path'] = self.getOrgLogoFromRedis(org) dic['org'] = org dic['pnts'] = self.getOrgPntFromRedis(org, date) + dic['epoch'] = sec data.append(dic) return data + def getContributorFromRedis(self, org): + date = datetime.datetime.now() + epoch = self.serv_redis_db.zscore("CONTRIB_LAST", org) + dic = {} + dic['rank'] = self.getOrgRankFromRedis(org, date) + dic['logo_path'] = self.getOrgLogoFromRedis(org) + dic['org'] = org + dic['pnts'] = self.getOrgPntFromRedis(org, date) + dic['epoch'] = epoch + return dic + def getTopContributorFromRedis(self, date): orgDicoPnts = {} for curDate in util.getMonthSpan(date): @@ -206,7 +218,7 @@ class Contributor_helper: ] return data2 - def TEST_getLastContributorFromRedis(self): + def TEST_getLastContributorsFromRedis(self): data2 = [ { 'rank': random.randint(1,self.levelMax), diff --git a/server.py b/server.py index fd464fe..cc666d8 100755 --- a/server.py +++ b/server.py @@ -37,6 +37,8 @@ subscriber_log = redis_server_log.pubsub(ignore_subscribe_messages=True) subscriber_log.psubscribe(cfg.get('RedisLog', 'channel')) subscriber_map = redis_server_map.pubsub(ignore_subscribe_messages=True) subscriber_map.psubscribe(cfg.get('RedisMap', 'channelDisp')) +subscriber_lastContrib = redis_server_log.pubsub(ignore_subscribe_messages=True) +subscriber_lastContrib.psubscribe(cfg.get('RedisLog', 'channelLastContributor')) eventNumber = 0 ########## @@ -260,9 +262,24 @@ def getCoordsByRadius(): ''' CONTRIB ''' -@app.route("/_getLastContributor") +@app.route("/_getLastContributors") +def getLastContributors(): + return jsonify(contributor_helper.getLastContributorsFromRedis()) + +@app.route("/_eventStreamLastContributor") def getLastContributor(): - return jsonify(contributor_helper.getLastContributorFromRedis()) + return Response(eventStreamLastContributor(), mimetype="text/event-stream") + +def eventStreamLastContributor(): + for msg in subscriber_lastContrib.listen(): + content = msg['data'].decode('utf8') + contentJson = json.loads(content) + lastContribJson = json.loads(contentJson['log']) + org = lastContribJson['org'] + to_return = contributor_helper.getContributorFromRedis(org) + epoch = lastContribJson['epoch'] + to_return['epoch'] = epoch + yield 'data: {}\n\n'.format(json.dumps(to_return)) @app.route("/_getTopContributor") def getTopContributor(suppliedDate=None): diff --git a/static/js/contrib.js b/static/js/contrib.js index 14fa12b..e5a8ac4 100644 --- a/static/js/contrib.js +++ b/static/js/contrib.js @@ -49,6 +49,14 @@ var optionDatatable_light = { }; var optionDatatable_top = jQuery.extend({}, optionDatatable_light) var optionDatatable_last = jQuery.extend({}, optionDatatable_light) +optionDatatable_last.columnDefs = [ + { 'orderData':[4], 'targets': [0] }, + { + 'targets': [4], + 'visible': false, + 'searchable': false + }, +] var optionDatatable_fame = jQuery.extend({}, optionDatatable_light) optionDatatable_fame.scrollY = '50vh'; @@ -208,6 +216,38 @@ function addToTableFromJson(datatable, url) { }); } +function addLastFromJson(datatable, url) { + $.getJSON( url, function( data ) { + for (i in data) { + var row = data[i]; + i = parseInt(i); + addLastContributor(datatable, row); + } + datatable.draw(); + }); +} + +function addLastContributor(datatable, data, update) { + var to_add = [ + data.pnts, + getRankIcon(data.rank), + data.logo_path, + data.org, + data.epoch + ]; + if (update == undefined || update == false) { + datatable.row.add(to_add); + } else if(update == true) { + datatable.rows().every( function() { + if(this.data()[3] == data.org) { + datatable.row( this ).data( to_add ); + } + }); + } +} + + + function updateProgressHeader(org) { // get Org rank $.getJSON( url_getOrgRank+'?org='+org, function( data ) { @@ -261,7 +301,7 @@ $(document).ready(function() { // hall of fame addToTableFromJson(datatableFame, url_getFameContributor); // last contributors - addToTableFromJson(datatableLast, url_getLastContributor); + addLastFromJson(datatableLast, url_getLastContributor); // category per contributors $.getJSON( url_getCategPerContrib, function( data ) { for (i in data) { @@ -286,5 +326,13 @@ $(document).ready(function() { var plotLineChart = $.plot("#divTop5Overtime", data, optionsLineChart); }); if(currOrg != "") // currOrg selected - updateProgressHeader(currOrg) + //FIXME: timeout used to wait that all datatables are draw. + setTimeout( function() { updateProgressHeader(currOrg); }, 200); + + source_lastContrib = new EventSource(url_eventStreamLastContributor); + source_lastContrib.onmessage = function(event) { + var json = jQuery.parseJSON( event.data ); + addLastContributor(datatableLast, json, true); + datatableLast.draw(); + }; }); diff --git a/templates/contrib.html b/templates/contrib.html index ad36e6b..a885910 100644 --- a/templates/contrib.html +++ b/templates/contrib.html @@ -130,6 +130,7 @@ Rank Logo Organisation + epoch @@ -232,7 +233,8 @@ var url_getFameContributor = "{{ url_for('getFameContributor') }}"; var url_getCategPerContrib = "{{ url_for('getCategPerContrib') }}"; var url_getTop5Overtime = "{{ url_for('getTop5Overtime') }}"; - var url_getLastContributor = "{{ url_for('getLastContributor') }}"; + var url_getLastContributor = "{{ url_for('getLastContributors') }}"; + var url_eventStreamLastContributor = "{{ url_for('getLastContributor') }}"; var url_getAllOrg = "{{ url_for('getAllOrg') }}"; var url_getOrgRank = "{{ url_for('getOrgRank') }}"; var url_baseRankLogo = "{{ url_for('static', filename='pics/rankingMISP/1.png') }}"; diff --git a/zmq_subscriber.py b/zmq_subscriber.py index a5001c0..53c6806 100755 --- a/zmq_subscriber.py +++ b/zmq_subscriber.py @@ -20,6 +20,7 @@ cfg.read(configfile) ZMQ_URL = cfg.get('RedisLog', 'zmq_url') CHANNEL = cfg.get('RedisLog', 'channel') +CHANNEL_LASTCONTRIB = cfg.get('RedisLog', 'channelLastContributor') CHANNELDISP = cfg.get('RedisMap', 'channelDisp') CHANNEL_PROC = cfg.get('RedisMap', 'channelProc') PATH_TO_DB = cfg.get('RedisMap', 'pathMaxMindDB') @@ -50,9 +51,9 @@ reader = geoip2.database.Reader(PATH_TO_DB) def getDateStrFormat(date): return str(date.year)+str(date.month).zfill(2)+str(date.day).zfill(2) -def publish_log(zmq_name, name, content): +def publish_log(zmq_name, name, content, channel=CHANNEL): to_send = { 'name': name, 'log': json.dumps(content), 'zmqName': zmq_name } - serv_log.publish(CHANNEL, json.dumps(to_send)) + serv_log.publish(channel, json.dumps(to_send)) def push_to_redis_zset(keyCateg, toAdd, endSubkey="", count=1): now = datetime.datetime.now() @@ -122,7 +123,7 @@ def noSpaceLower(str): return str.lower().replace(' ', '_') #pntMultiplier if one contribution rewards more than others. (e.g. shighting may gives more points than editing) -def handleContribution(org, categ, action, pntMultiplier=1): +def handleContribution(zmq_name, org, categ, action, pntMultiplier=1): if action in ['edit']: pass #return #not a contribution? @@ -143,6 +144,8 @@ def handleContribution(org, categ, action, pntMultiplier=1): serv_redis_db.zadd('CONTRIB_LAST:'+getDateStrFormat(now), nowSec, org) serv_redis_db.expire('CONTRIB_LAST:'+getDateStrFormat(now), 60*60*24) #expire after 1 day + publish_log(zmq_name, 'CONTRIBUTION', {'org': org, 'categ': categ, 'action': action, 'epoch': nowSec }, channel=CHANNEL_LASTCONTRIB) + ############## ## HANDLERS ## ############## @@ -165,7 +168,7 @@ def handler_sighting(zmq_name, jsonsight): org = jsonsight['org'] categ = jsonsight['categ'] action = jsonsight['action'] - handleContribution(org, categ, action) + handleContribution(zmq_name, org, categ, action) return def handler_event(zmq_name, jsonobj): @@ -202,7 +205,7 @@ def handler_attribute(zmq_name, jsonobj): if jsonattr['category'] == "Network activity": getCoordAndPublish(zmq_name, jsonattr['value'], jsonattr['category']) - handleContribution(jsonobj['Event']['Orgc']['name'], jsonattr['category'], jsonobj['action']) + handleContribution(zmq_name, jsonobj['Event']['Orgc']['name'], jsonattr['category'], jsonobj['action']) # Push to log publish_log(zmq_name, 'Attribute', to_push)