Added draft of automatic update for lastContributor

config/config.cfg.default

@@ -40,6 +40,7 @@ port=6251
 [RedisLog]
 db=0
 channel=1
+channelLastContributor = lastContributor
 zmq_url=tcp://localhost:50000
 #zmq_url=tcp://192.168.56.50:50000
 

contributor_helper.py

@@ -40,11 +40,11 @@ class Contributor_helper:
     def getOrgLogoFromRedis(self, org):
         return 'logo_'+org
 
-    def getLastContributorFromRedis(self):
+    def getLastContributorsFromRedis(self):
         date = datetime.datetime.now()
-        keyCateg = "CONTRIB_LAST"
+        keyname = "CONTRIB_LAST"
         topNum = self.MAX_NUMBER_OF_LAST_CONTRIBUTOR # default Num
-        last_contrib_org = self.getZrange(keyCateg, date, topNum)
+        last_contrib_org = self.getZrange(keyname, date, topNum)
         data = []
         for org, sec in last_contrib_org:
             dic = {}
@@ -52,9 +52,21 @@ class Contributor_helper:
             dic['logo_path'] = self.getOrgLogoFromRedis(org)
             dic['org'] = org
             dic['pnts'] = self.getOrgPntFromRedis(org, date)
+            dic['epoch'] = sec
             data.append(dic)
         return data
 
+    def getContributorFromRedis(self, org):
+        date = datetime.datetime.now()
+        epoch = self.serv_redis_db.zscore("CONTRIB_LAST", org)
+        dic = {}
+        dic['rank'] = self.getOrgRankFromRedis(org, date)
+        dic['logo_path'] = self.getOrgLogoFromRedis(org)
+        dic['org'] = org
+        dic['pnts'] = self.getOrgPntFromRedis(org, date)
+        dic['epoch'] = epoch
+        return dic
+
     def getTopContributorFromRedis(self, date):
         orgDicoPnts = {}
         for curDate in util.getMonthSpan(date):
@@ -206,7 +218,7 @@ class Contributor_helper:
         ]
         return data2
 
-    def TEST_getLastContributorFromRedis(self):
+    def TEST_getLastContributorsFromRedis(self):
         data2 = [
             {
                 'rank': random.randint(1,self.levelMax),

server.py

@@ -37,6 +37,8 @@ subscriber_log = redis_server_log.pubsub(ignore_subscribe_messages=True)
 subscriber_log.psubscribe(cfg.get('RedisLog', 'channel'))
 subscriber_map = redis_server_map.pubsub(ignore_subscribe_messages=True)
 subscriber_map.psubscribe(cfg.get('RedisMap', 'channelDisp'))
+subscriber_lastContrib = redis_server_log.pubsub(ignore_subscribe_messages=True)
+subscriber_lastContrib.psubscribe(cfg.get('RedisLog', 'channelLastContributor'))
 eventNumber = 0
 
 ##########
@@ -260,9 +262,24 @@ def getCoordsByRadius():
 
 ''' CONTRIB '''
 
-@app.route("/_getLastContributor")
+@app.route("/_getLastContributors")
+def getLastContributors():
+    return jsonify(contributor_helper.getLastContributorsFromRedis())
+
+@app.route("/_eventStreamLastContributor")
 def getLastContributor():
-    return jsonify(contributor_helper.getLastContributorFromRedis())
+    return Response(eventStreamLastContributor(), mimetype="text/event-stream")
+
+def eventStreamLastContributor():
+    for msg in subscriber_lastContrib.listen():
+        content = msg['data'].decode('utf8')
+        contentJson = json.loads(content)
+        lastContribJson = json.loads(contentJson['log'])
+        org = lastContribJson['org']
+        to_return = contributor_helper.getContributorFromRedis(org)
+        epoch = lastContribJson['epoch']
+        to_return['epoch'] = epoch
+        yield 'data: {}\n\n'.format(json.dumps(to_return))
 
 @app.route("/_getTopContributor")
 def getTopContributor(suppliedDate=None):

static/js/contrib.js

@@ -49,6 +49,14 @@ var optionDatatable_light = {
 };
 var optionDatatable_top = jQuery.extend({}, optionDatatable_light)
 var optionDatatable_last = jQuery.extend({}, optionDatatable_light)
+optionDatatable_last.columnDefs = [
+    { 'orderData':[4], 'targets': [0] },
+    {
+        'targets': [4],
+        'visible': false,
+        'searchable': false
+    },
+]
 var optionDatatable_fame = jQuery.extend({}, optionDatatable_light)
 optionDatatable_fame.scrollY = '50vh';
 
@@ -208,6 +216,38 @@ function addToTableFromJson(datatable, url) {
     });
 }
 
+function addLastFromJson(datatable, url) {
+    $.getJSON( url, function( data ) {
+        for (i in data) {
+            var row = data[i];
+            i = parseInt(i);
+            addLastContributor(datatable, row);
+        }
+        datatable.draw();
+    });
+}
+
+function addLastContributor(datatable, data, update) {
+    var to_add = [
+        data.pnts,
+        getRankIcon(data.rank),
+        data.logo_path,
+        data.org,
+        data.epoch
+    ];
+    if (update == undefined || update == false) {
+        datatable.row.add(to_add);
+    } else if(update == true) {
+        datatable.rows().every( function() {
+            if(this.data()[3] == data.org) {
+                datatable.row( this ).data( to_add );
+            }
+        });
+    }
+}
+
+
+
 function updateProgressHeader(org) {
     // get Org rank
     $.getJSON( url_getOrgRank+'?org='+org, function( data ) {
@@ -261,7 +301,7 @@ $(document).ready(function() {
     // hall of fame
     addToTableFromJson(datatableFame, url_getFameContributor);
     // last contributors
-    addToTableFromJson(datatableLast, url_getLastContributor);
+    addLastFromJson(datatableLast, url_getLastContributor);
     // category per contributors
     $.getJSON( url_getCategPerContrib, function( data ) {
         for (i in data) {
@@ -286,5 +326,13 @@ $(document).ready(function() {
         var plotLineChart = $.plot("#divTop5Overtime", data, optionsLineChart);
     });
     if(currOrg != "") // currOrg selected
-        updateProgressHeader(currOrg)
+        //FIXME: timeout used to wait that all datatables are draw.
+        setTimeout( function() { updateProgressHeader(currOrg); }, 200);
+
+    source_lastContrib = new EventSource(url_eventStreamLastContributor);
+    source_lastContrib.onmessage = function(event) {
+        var json = jQuery.parseJSON( event.data );
+        addLastContributor(datatableLast, json, true);
+        datatableLast.draw();
+    };
 });

templates/contrib.html

@@ -130,6 +130,7 @@
                                                         <th>Rank</th>
                                                         <th>Logo</th>
                                                         <th>Organisation</th>
+                                                        <th>epoch</th>
                                                     </tr>
                                                 </thead>
                                                 <tbody>
@@ -232,7 +233,8 @@
         var url_getFameContributor = "{{ url_for('getFameContributor') }}";
         var url_getCategPerContrib = "{{ url_for('getCategPerContrib') }}";
         var url_getTop5Overtime = "{{ url_for('getTop5Overtime') }}";
-        var url_getLastContributor = "{{ url_for('getLastContributor') }}";
+        var url_getLastContributor = "{{ url_for('getLastContributors') }}";
+        var url_eventStreamLastContributor = "{{ url_for('getLastContributor') }}";
         var url_getAllOrg = "{{ url_for('getAllOrg') }}";
         var url_getOrgRank = "{{ url_for('getOrgRank') }}";
         var url_baseRankLogo =     "{{ url_for('static', filename='pics/rankingMISP/1.png') }}";

zmq_subscriber.py

@@ -20,6 +20,7 @@ cfg.read(configfile)
 
 ZMQ_URL = cfg.get('RedisLog', 'zmq_url')
 CHANNEL = cfg.get('RedisLog', 'channel')
+CHANNEL_LASTCONTRIB = cfg.get('RedisLog', 'channelLastContributor')
 CHANNELDISP = cfg.get('RedisMap', 'channelDisp')
 CHANNEL_PROC = cfg.get('RedisMap', 'channelProc')
 PATH_TO_DB = cfg.get('RedisMap', 'pathMaxMindDB')
@@ -50,9 +51,9 @@ reader = geoip2.database.Reader(PATH_TO_DB)
 def getDateStrFormat(date):
     return str(date.year)+str(date.month).zfill(2)+str(date.day).zfill(2)
 
-def publish_log(zmq_name, name, content):
+def publish_log(zmq_name, name, content, channel=CHANNEL):
     to_send = { 'name': name, 'log': json.dumps(content), 'zmqName': zmq_name }
-    serv_log.publish(CHANNEL, json.dumps(to_send))
+    serv_log.publish(channel, json.dumps(to_send))
 
 def push_to_redis_zset(keyCateg, toAdd, endSubkey="", count=1):
     now = datetime.datetime.now()
@@ -122,7 +123,7 @@ def noSpaceLower(str):
     return str.lower().replace(' ', '_')
 
 #pntMultiplier if one contribution rewards more than others. (e.g. shighting may gives more points than editing)
-def handleContribution(org, categ, action, pntMultiplier=1):
+def handleContribution(zmq_name, org, categ, action, pntMultiplier=1):
     if action in ['edit']:
         pass
         #return #not a contribution?
@@ -143,6 +144,8 @@ def handleContribution(org, categ, action, pntMultiplier=1):
     serv_redis_db.zadd('CONTRIB_LAST:'+getDateStrFormat(now), nowSec, org)
     serv_redis_db.expire('CONTRIB_LAST:'+getDateStrFormat(now), 60*60*24) #expire after 1 day
 
+    publish_log(zmq_name, 'CONTRIBUTION', {'org': org, 'categ': categ, 'action': action, 'epoch': nowSec }, channel=CHANNEL_LASTCONTRIB)
+
 ##############
 ## HANDLERS ##
 ##############
@@ -165,7 +168,7 @@ def handler_sighting(zmq_name, jsonsight):
     org = jsonsight['org']
     categ = jsonsight['categ']
     action = jsonsight['action']
-    handleContribution(org, categ, action)
+    handleContribution(zmq_name, org, categ, action)
     return
 
 def handler_event(zmq_name, jsonobj):
@@ -202,7 +205,7 @@ def handler_attribute(zmq_name, jsonobj):
     if jsonattr['category'] == "Network activity":
         getCoordAndPublish(zmq_name, jsonattr['value'], jsonattr['category'])
 
-    handleContribution(jsonobj['Event']['Orgc']['name'], jsonattr['category'], jsonobj['action'])
+    handleContribution(zmq_name, jsonobj['Event']['Orgc']['name'], jsonattr['category'], jsonobj['action'])
     # Push to log
     publish_log(zmq_name, 'Attribute', to_push)