Added overtake remaining points + moved updateOrgContributionRank from zmq_subscribe to contributor_helper

contributor_helper.py

@@ -88,11 +88,11 @@ class Contributor_helper:
 
     def giveContribRankToOrg(self, org, rankNum):
         keyname = 'CONTRIB_ORG:{org}:{orgCateg}'
-        serv_redis_db.set(keyname.format(org=orgName, orgCateg='CONTRIB_REQ_'+str(rankNum)), 1)
+        self.serv_redis_db.set(keyname.format(org=orgName, orgCateg='CONTRIB_REQ_'+str(rankNum)), 1)
 
     def removeContribRankFromOrg(self, org, rankNum):
         keyname = 'CONTRIB_ORG:{org}:{orgCateg}'
-        serv_redis_db.delete(keyname.format(org=orgName, orgCateg='CONTRIB_REQ_'+str(rankNum)))
+        self.serv_redis_db.delete(keyname.format(org=orgName, orgCateg='CONTRIB_REQ_'+str(rankNum)))
 
     # 1 for fulfilled, 0 for not fulfilled, -1 for not relevant
     def getCurrentContributionStatus(self, org):
@@ -107,6 +107,52 @@ class Contributor_helper:
                 to_ret[i] = -1
         return to_ret
 
+
+    def updateOrgContributionRank(self, orgName, pnts_to_add, contribType, eventTime, isLabeled):
+        keyname = 'CONTRIB_ORG:{org}:{orgCateg}'
+        #update total points
+        self.serv_redis_db.set(keyname.format(org=orgName, orgCateg='points'), pnts_to_add)
+        #update contribution Requirement
+        heavilyCount = self.heavilyCount
+        recentDays = self.recentDays
+        regularlyDays = self.regularlyDays
+        isRecent = (datetime.datetime.now() - eventTime).days > recentDays
+
+        contrib = [] #[[contrib_level, contrib_ttl], [], ...]
+        if contribType == 'sighting':
+            #[contrib_level, contrib_ttl]
+            contrib.append([1, ONE_DAY*365])
+        if contribType == 'attribute' or contribType == 'object':
+            contrib.append([2, ONE_DAY*365])
+        if contribType == 'proposal' or contribType == 'discussion':
+            contrib.append([3, ONE_DAY*365])
+        if contribType == 'sighting' and isRecent:
+            contrib.append([4, ONE_DAY*recentDays])
+        if contribType == 'proposal' and isRecent:
+            contrib.append([5, ONE_DAY*recentDays])
+        if contribType == 'event':
+            contrib.append([6, ONE_DAY*365])
+        if contribType == 'event':
+            contrib.append([7, ONE_DAY*recentDays])
+        if contribType == 'event':
+            contrib.append([8, ONE_DAY*regularlyDays])
+        if contribType == 'event' and isLabeled:
+            contrib.append([9, ONE_DAY*regularlyDays])
+        if contribType == 'sighting' and sightingWeekCount>heavilyCount:
+            contrib.append([10, ONE_DAY*regularlyDays])
+        if (contribType == 'attribute' or contribType == 'object') and attributeWeekCount>heavilyCount:
+            contrib.append([11, ONE_DAY*regularlyDays])
+        if contribType == 'proposal' and proposalWeekCount>heavilyCount:
+            contrib.append([12, ONE_DAY*regularlyDays])
+        if contribType == 'event' and eventWeekCount>heavilyCount:
+            contrib.append([13, ONE_DAY*regularlyDays])
+        if contribType == 'event' and eventWeekCount>heavilyCount  and isLabeled:
+            contrib.append([14, ONE_DAY*regularlyDays])
+
+        for rankReq, ttl in contrib:
+            self.serv_redis_db.set(keyname.format(org=orgName, orgCateg='CONTRIB_REQ_'+str(rankReq)), 1)
+            self.serv_redis_db.expire(keyname.format(org=orgName, orgCateg='CONTRIB_REQ_'+str(i)), ttl)
+
     ''' HONOR BADGES '''
     def getOrgHonorBadges(self, org):
         keyname = 'CONTRIB_ORG:{org}:{orgCateg}'
@@ -119,11 +165,11 @@ class Contributor_helper:
 
     def giveBadgeToOrg(self, org, badgeNum):
         keyname = 'CONTRIB_ORG:{org}:{orgCateg}'
-        serv_redis_db.set(keyname.format(org=orgName, orgCateg='BADGE_'+str(badgeNum)), 1)
+        self.serv_redis_db.set(keyname.format(org=orgName, orgCateg='BADGE_'+str(badgeNum)), 1)
 
     def removeBadgeFromOrg(self, org, badgeNum):
         keyname = 'CONTRIB_ORG:{org}:{orgCateg}'
-        serv_redis_db.delete(keyname.format(org=orgName, orgCateg='BADGE_'+str(badgeNum)))
+        self.serv_redis_db.delete(keyname.format(org=orgName, orgCateg='BADGE_'+str(badgeNum)))
 
     ''' MONTHLY CONTRIBUTION '''
     def getOrgPntFromRedis(self, org, date):
@@ -270,7 +316,9 @@ class Contributor_helper:
         return { 'remainingPts': 0, 'stepPts': self.rankMultiplier**self.levelMax }
 
 
+    '''           '''
     ''' TEST DATA '''
+    '''           '''
 
     def TEST_getCategPerContribFromRedis(self, date):
         data2 = []

static/js/contrib.js

@@ -308,6 +308,7 @@ function addLastContributor(datatable, data, update) {
 }
 
 function updateProgressHeader(org) {
+    currOrg = org;
     // get Org rank
     $.getJSON( url_getOrgRank+'?org='+org, function( data ) {
         datatableTop.draw();
@@ -383,6 +384,25 @@ function updateProgressHeader(org) {
             }
         }
     });
+
+    //update overtake points
+    var prevOrg = "";
+    var prevOrgPnts = 0;
+    datatableTop.rows().every( function() {
+        var row = this.node();
+        if(this.data()[5] == currOrg) {
+            if(prevOrg == ""){ //already first
+                $('#orgToOverTake').text(this.data()[5]);
+                $('#pntsToOvertakeNext').text(0);
+            } else {
+                $('#orgToOverTake').text(prevOrg);
+                $('#pntsToOvertakeNext').text(parseInt(prevOrgPnts)-this.data()[0]);
+            }
+        } else {
+            prevOrg = this.data()[5];
+            prevOrgPnts = this.data()[0];
+        }
+    });
 }
 
 function showOnlyOrg() {
@@ -458,7 +478,7 @@ $(document).ready(function() {
     });
     if(currOrg != "") // currOrg selected
         //FIXME: timeout used to wait that all datatables are draw.
-        setTimeout( function() { updateProgressHeader(currOrg); }, 200);
+        setTimeout( function() { updateProgressHeader(currOrg); }, 400);
 
     source_lastContrib = new EventSource(url_eventStreamLastContributor);
     source_lastContrib.onmessage = function(event) {

templates/contrib.html

@@ -201,6 +201,14 @@
                         </div>
                         <div id="orgNextRankDiv" class='textTopHeader' style="padding-top: 0px; position: relative; width: 40px; height: 40px;"></div>
                     </button>
+                    <div class='leftSepa textTopHeader'>
+                        <span class="label label-primary">
+                        <strong style="">Points to overtake</strong>
+                        <strong style="font-size: medium;" id='orgToOverTake'></strong>
+                        <strong style="">: </strong>
+                        <strong style="font-size: large;" id='pntsToOvertakeNext'></strong>
+                        </span>
+                    </div>
                 </div>
             </div>
             <!-- /.navbar-header -->

templates/index.html

@@ -132,6 +132,7 @@ small {
                     <ul class="dropdown-menu">
                       <li><a href="{{ url_for('index') }}">MISP Live Dashboard</a></li>
                       <li><a href="{{ url_for('geo') }}">MISP Geolocalisation</a></li>
+                      <li><a href="{{ url_for('contrib') }}">MISP Contributors</a></li>
                     </ul>
 
                 <div id="ledsHolder" style="float: right; height: 50px;"></div>

zmq_subscriber.py

@@ -14,6 +14,9 @@ import sys
 import json
 import geoip2.database
 
+import util
+import contributor_helper
+
 configfile = os.path.join(os.environ['DASH_CONFIG'], 'config.cfg')
 cfg = configparser.ConfigParser()
 cfg.read(configfile)
@@ -52,22 +55,19 @@ contributor_helper = contributor_helper.Contributor_helper(serv_redis_db, cfg)
 
 reader = geoip2.database.Reader(PATH_TO_DB)
 
-def getDateStrFormat(date):
-    return str(date.year)+str(date.month).zfill(2)+str(date.day).zfill(2)
-
 def publish_log(zmq_name, name, content, channel=CHANNEL):
     to_send = { 'name': name, 'log': json.dumps(content), 'zmqName': zmq_name }
     serv_log.publish(channel, json.dumps(to_send))
 
 def push_to_redis_zset(keyCateg, toAdd, endSubkey="", count=1):
     now = datetime.datetime.now()
-    today_str = getDateStrFormat(now)
+    today_str = util.getDateStrFormat(now)
     keyname = "{}:{}{}".format(keyCateg, today_str, endSubkey)
     serv_redis_db.zincrby(keyname, toAdd, count)
 
 def push_to_redis_geo(keyCateg, lon, lat, content):
     now = datetime.datetime.now()
-    today_str = getDateStrFormat(now)
+    today_str = util.getDateStrFormat(now)
     keyname = "{}:{}".format(keyCateg, today_str)
     serv_redis_db.geoadd(keyname, lon, lat, content)
 
@@ -127,7 +127,7 @@ def noSpaceLower(str):
     return str.lower().replace(' ', '_')
 
 #pntMultiplier if one contribution rewards more than others. (e.g. shighting may gives more points than editing)
-def handleContribution(zmq_name, org, categ, action, pntMultiplier=1):
+def handleContribution(zmq_name, org, categ, action, pntMultiplier=1, eventTime=datetime.datetime.now(), isLabeled=False):
     if action in ['edit']:
         pass
         #return #not a contribution?
@@ -145,58 +145,13 @@ def handleContribution(zmq_name, org, categ, action, pntMultiplier=1):
 
     now = datetime.datetime.now()
     nowSec = int(time.time())
-    serv_redis_db.zadd('CONTRIB_LAST:'+getDateStrFormat(now), nowSec, org)
+    serv_redis_db.zadd('CONTRIB_LAST:'+util.getDateStrFormat(now), nowSec, org)
-    serv_redis_db.expire('CONTRIB_LAST:'+getDateStrFormat(now), ONE_DAY) #expire after 1 day
+    serv_redis_db.expire('CONTRIB_LAST:'+util.getDateStrFormat(now), ONE_DAY) #expire after 1 day
 
-    updateOrgRank(org, pnts_to_add, eventTime, eventClassification)
+    contributor_helper.updateOrgContributionRank(org, pnts_to_add, eventTime, eventTime=datetime.datetime.now(), isLabeled=isLabeled)
 
     publish_log(zmq_name, 'CONTRIBUTION', {'org': org, 'categ': categ, 'action': action, 'epoch': nowSec }, channel=CHANNEL_LASTCONTRIB)
 
-def updateOrgRank(orgName, pnts_to_add, contribType, eventTime, isClassified):
-    keyname = 'CONTRIB_ORG:{org}:{orgCateg}'
-    #update total points
-    serv_redis_db.set(keyname.format(org=orgName, orgCateg='points'), pnts_to_add)
-    #update contribution Requirement
-    heavilyCount = contributor_helper.heavilyCount
-    recentDays = contributor_helper.recentDays
-    regularlyDays = contributor_helper.regularlyDays
-    isRecent = True if (datetime.datetime.now() - eventTime).days > recentDays
-
-    contrib = [] #[[contrib_level, contrib_ttl], [], ...]
-    if contribType == 'sighting':
-        #[contrib_level, contrib_ttl]
-        contrib.append([1, ONE_DAY*365]])
-    if contribType == 'attribute' or contribType == 'object':
-        contrib.append([2, ONE_DAY*365])
-    if contribType == 'proposal' or contribType == 'discussion':
-        contrib.append([3, ONE_DAY*365])
-    if contribType == 'sighting' and isRecent:
-        contrib.append([4, ONE_DAY*recentDays])
-    if contribType == 'proposal' and isRecent:
-        contrib.append([5, ONE_DAY*recentDays])
-    if contribType == 'event':
-        contrib.append([6, ONE_DAY*365])
-    if contribType == 'event':
-        contrib.append([7, ONE_DAY*recentDays])
-    if contribType == 'event':
-        contrib.append([8, ONE_DAY*regularlyDays])
-    if contribType == 'event' and isClassified:
-        contrib.append([9, ONE_DAY*regularlyDays])
-    if contribType == 'sighting' and sightingWeekCount>heavilyCount:
-        contrib.append([10, ONE_DAY*regularlyDays])
-    if (contribType == 'attribute' or contribType == 'object') and attributeWeekCount>heavilyCount:
-        contrib.append([11, ONE_DAY*regularlyDays])
-    if contribType == 'proposal' and proposalWeekCount>heavilyCount:
-        contrib.append([12, ONE_DAY*regularlyDays])
-    if contribType == 'event' and eventWeekCount>heavilyCount:
-        contrib.append([13, ONE_DAY*regularlyDays])
-    if contribType == 'event' and eventWeekCount>heavilyCount  and isClassified:
-        contrib.append([14, ONE_DAY*regularlyDays])
-
-    for rankReq, ttl in contrib:
-        serv_redis_db.set(keyname.format(org=orgName, orgCateg='CONTRIB_REQ_'+str(rankReq)), 1)
-        serv_redis_db.expire(keyname.format(org=orgName, orgCateg='CONTRIB_REQ_'+str(i)), ttl)
-
 
 ##############
 ## HANDLERS ##
@@ -257,11 +212,17 @@ def handler_attribute(zmq_name, jsonobj):
     if jsonattr['category'] == "Network activity":
         getCoordAndPublish(zmq_name, jsonattr['value'], jsonattr['category'])
 
-    handleContribution(zmq_name, jsonobj['Event']['Orgc']['name'], jsonattr['category'], jsonobj['action'])
+    eventLabeled = False
+    #eventLabeled = len(jsonattr[]) > 0
+    handleContribution(zmq_name, jsonobj['Event']['Orgc']['name'], jsonattr['category'], jsonobj['action'], isLabeled=eventLabeled)
     # Push to log
     publish_log(zmq_name, 'Attribute', to_push)
 
 
+###############
+## MAIN LOOP ##
+###############
+
 def process_log(zmq_name, event):
     event = event.decode('utf8')
     topic, eventdata = event.split(' ', maxsplit=1)