Merge pull request #12 from mokaddem/codeCleanUp

Code clean up

config/config.cfg.default

@@ -22,12 +22,6 @@ clusteringDistance = 10
 [CONTRIB]
 max_number_of_last_contributor = 10
 min_between_reload = 5
-#How much harder it gets to rank up (exponential multiplier) [1.5 -> +inf]
-rankMultiplier = 2
-categories_in_datatable = ["internal_reference", "targeting_data", "antivirus_detection", "payload_delivery", "artifacts_dropped", "payload_installation", "persistence_mechanism", "network_activity", "payload_type", "attribution", "external_analysis", "financial_fraud", "support_Tool", "social_network", "person", "other" ]
-default_pnts_per_contribution = 1
-# array of the form [[category, pntsRcv], ...]
-pnts_per_contribution = [["payload_delivery", 1], ["artifacts_dropped", 1], ["network_activity", 1]]
 additional_help_text = ["Sightings multiplies earned points by 2", "Editing an attribute earns you the same as creating one"]
 
 [Log]

config/ranking.cfg

@@ -1,3 +1,12 @@
+[monthlyRanking]
+#How much harder it gets to rank up (exponential multiplier) [1.5 -> +inf]
+rankMultiplier = 2
+levelMax=16
+categories_in_datatable = ["internal_reference", "targeting_data", "antivirus_detection", "payload_delivery", "artifacts_dropped", "payload_installation", "persistence_mechanism", "network_activity", "payload_type", "attribution", "external_analysis", "financial_fraud", "support_Tool", "social_network", "person", "other" ]
+default_pnts_per_contribution = 1
+# array of the form [[category, pntsRcv], ...]
+pnts_per_contribution = [["payload_delivery", 1], ["artifacts_dropped", 1], ["network_activity", 1]]
+
 [rankTitle]
 maxLevel=16
 1=Ensign

contributor_helper.py

@@ -1,4 +1,5 @@
 import util
+from util import getZrange
 import math, random
 import os
 import configparser
@@ -11,6 +12,7 @@ class Contributor_helper:
         self.cfg = cfg
         self.cfg_org_rank = configparser.ConfigParser()
         self.cfg_org_rank.read(os.path.join(os.environ['DASH_CONFIG'], 'ranking.cfg'))
+        self.CHANNEL_LASTAWARDS = cfg.get('RedisLog', 'channelLastAwards')
 
         #honorBadge
         self.honorBadgeNum = len(self.cfg_org_rank.options('HonorBadge'))
@@ -45,11 +47,11 @@ class Contributor_helper:
         #WEB STUFF
         self.misp_web_url = cfg.get('RedisGlobal', 'misp_web_url')
         self.MAX_NUMBER_OF_LAST_CONTRIBUTOR = cfg.getint('CONTRIB', 'max_number_of_last_contributor')
-        self.categories_in_datatable = json.loads(cfg.get('CONTRIB', 'categories_in_datatable'))
+        self.categories_in_datatable = json.loads(self.cfg_org_rank.get('monthlyRanking', 'categories_in_datatable'))
 
         #MONTHLY RANKING
-        self.default_pnts_per_contribution = json.loads(cfg.get('CONTRIB', 'default_pnts_per_contribution'))
+        self.default_pnts_per_contribution = json.loads(self.cfg_org_rank.get('monthlyRanking', 'default_pnts_per_contribution'))
-        temp = json.loads(cfg.get('CONTRIB', 'pnts_per_contribution'))
+        temp = json.loads(self.cfg_org_rank.get('monthlyRanking', 'pnts_per_contribution'))
         self.DICO_PNTS_REWARD = {}
         for categ, pnts in temp:
             self.DICO_PNTS_REWARD[categ] = pnts
@@ -60,30 +62,81 @@ class Contributor_helper:
             else:
                 self.DICO_PNTS_REWARD[categ] = self.default_pnts_per_contribution
 
-        self.rankMultiplier = cfg.getfloat('CONTRIB' ,'rankMultiplier')
+        self.rankMultiplier = self.cfg_org_rank.getfloat('monthlyRanking' ,'rankMultiplier')
-        self.levelMax = 16
+        self.levelMax = self.cfg_org_rank.getfloat('monthlyRanking' ,'levelMax')
+
+        # REDIS KEYS
+        self.keyDay         = "CONTRIB_DAY"
+        self.keyCateg       = "CONTRIB_CATEG"
+        self.keyLastContrib = "CONTRIB_LAST"
+        self.keyAllOrg      = "CONTRIB_ALL_ORG"
+        self.keyContribReq  = "CONTRIB_ORG"
+        self.keyTrophy      = "CONTRIB_TROPHY"
+        self.keyLastAward   = "CONTRIB_LAST_AWARDS"
 
 
     ''' HELPER '''
     def getOrgLogoFromMISP(self, org):
         return "{}/img/orgs/{}.png".format(self.misp_web_url, org)
 
-    def getZrange(self, keyCateg, date, topNum, endSubkey=""):
-        date_str = util.getDateStrFormat(date)
-        keyname = "{}:{}{}".format(keyCateg, date_str, endSubkey)
-        data = self.serv_redis_db.zrange(keyname, 0, topNum-1, desc=True, withscores=True)
-        data = [ [record[0].decode('utf8'), record[1]] for record in data ]
-        return data
-
     def addContributionToCateg(self, date, categ, org, count=1):
         today_str = util.getDateStrFormat(date)
-        keyname = "CONTRIB_CATEG:{}:{}".format(today_str, categ)
+        keyname = "{}:{}:{}".format(self.keyCateg, today_str, categ)
         self.serv_redis_db.zincrby(keyname, org, count)
 
+    def publish_log(self, zmq_name, name, content, channel=""):
+        to_send = { 'name': name, 'log': json.dumps(content), 'zmqName': zmq_name }
+        serv_log.publish(channel, json.dumps(to_send))
+
+    ''' HANDLER '''
+    #pntMultiplier if one contribution rewards more than others. (e.g. shighting may gives more points than editing)
+    def handleContribution(self, zmq_name, org, contribType, categ, action, pntMultiplier=1, eventTime=datetime.datetime.now(), isLabeled=False):
+        if action in ['edit', None]:
+            pass
+            #return #not a contribution?
+    
+        now = datetime.datetime.now()
+        nowSec = int(time.time())
+        pnts_to_add = self.default_pnts_per_contribution
+    
+        # if there is a contribution, there is a login (even if ti comes from the API)
+        users_helper.add_user_login(nowSec, org)
+    
+        # is a valid contribution
+        if categ is not None:
+            try:
+                pnts_to_add = self.DICO_PNTS_REWARD[noSpaceLower(categ)]
+            except KeyError:
+                pnts_to_add = self.default_pnts_per_contribution
+            pnts_to_add *= pntMultiplier
+    
+            util.push_to_redis_zset(self.serv_redis_db, self.keyDay, org, count=pnts_to_add)
+            #CONTRIB_CATEG retain the contribution per category, not the point earned in this categ
+            util.push_to_redis_zset(self.serv_redis_db, self.keyCateg, org, count=1, endSubkey=':'+noSpaceLower(categ))
+            self.publish_log(zmq_name, 'CONTRIBUTION', {'org': org, 'categ': categ, 'action': action, 'epoch': nowSec }, channel=CHANNEL_LASTCONTRIB)
+        else:
+            categ = ""
+    
+        serv_redis_db.sadd(self.keyAllOrg, org)
+    
+        keyname = "{}:{}".format(self.keyLastContrib, util.getDateStrFormat(now))
+        serv_redis_db.zadd(keyname, nowSec, org)
+        serv_redis_db.expire(keyname, ONE_DAY*7) #expire after 7 day
+    
+        awards_given = self.updateOrgContributionRank(org, pnts_to_add, action, contribType, eventTime=datetime.datetime.now(), isLabeled=isLabeled, categ=noSpaceLower(categ))
+    
+        for award in awards_given:
+            # update awards given
+            keyname = "{}:{}".format(self.keyLastAward, util.getDateStrFormat(now))
+            serv_redis_db.zadd(keyname, nowSec, json.dumps({'org': org, 'award': award, 'epoch': nowSec }))
+            serv_redis_db.expire(keyname, ONE_DAY*7) #expire after 7 day
+            # publish
+            self.publish_log(zmq_name, 'CONTRIBUTION', {'org': org, 'award': award, 'epoch': nowSec }, channel=self.CHANNEL_LASTAWARDS)
+
     ''' CONTRIBUTION RANK '''
     def getOrgContributionTotalPoints(self, org):
-        keyname = 'CONTRIB_ORG:{org}:{orgCateg}'
+        keyname = '{mainKey}:{org}:{orgCateg}'
-        pnts = self.serv_redis_db.get(keyname.format(org=org, orgCateg='points'))
+        pnts = self.serv_redis_db.get(keyname.format(mainKey=self.keyContribReq, org=org, orgCateg='points'))
         if pnts is None:
             pnts = 0
         else:
@@ -92,12 +145,12 @@ class Contributor_helper:
 
     # return: [final_rank, requirement_fulfilled, requirement_not_fulfilled]
     def getOrgContributionRank(self, org):
-        keyname = 'CONTRIB_ORG:{org}:{orgCateg}'
+        keyname = '{mainKey}:{org}:{orgCateg}'
         final_rank = 0
         requirement_fulfilled = []
         requirement_not_fulfilled = []
         for i in range(1, self.org_rank_maxLevel+1):
-            key = keyname.format(org=org, orgCateg='CONTRIB_REQ_'+str(i))
+            key = keyname.format(mainKey=self.keyContribReq, org=org, orgCateg='CONTRIB_REQ_'+str(i))
             if self.serv_redis_db.get(key) is None: #non existing
                 requirement_not_fulfilled.append(i)
             else:
@@ -109,12 +162,12 @@ class Contributor_helper:
         return {'final_rank': final_rank, 'req_fulfilled': requirement_fulfilled, 'req_not_fulfilled': requirement_not_fulfilled}
 
     def giveContribRankToOrg(self, org, rankNum):
-        keyname = 'CONTRIB_ORG:{org}:{orgCateg}'
+        keyname = '{mainKey}:{org}:{orgCateg}'
-        self.serv_redis_db.set(keyname.format(org=org, orgCateg='CONTRIB_REQ_'+str(rankNum)), 1)
+        self.serv_redis_db.set(keyname.format(mainKey=self.keyContribReq, org=org, orgCateg='CONTRIB_REQ_'+str(rankNum)), 1)
 
     def removeContribRankFromOrg(self, org, rankNum):
-        keyname = 'CONTRIB_ORG:{org}:{orgCateg}'
+        keyname = '{mainKey}:{org}:{orgCateg}'
-        self.serv_redis_db.delete(keyname.format(org=org, orgCateg='CONTRIB_REQ_'+str(rankNum)))
+        self.serv_redis_db.delete(keyname.format(mainKey=self.keyContribReq, org=org, orgCateg='CONTRIB_REQ_'+str(rankNum)))
 
     # 1 for fulfilled, 0 for not fulfilled, -1 for not relevant
     def getCurrentContributionStatus(self, org):
@@ -138,7 +191,7 @@ class Contributor_helper:
         oldContributionStatus = ContributionStatus['status']
         oldHonorBadges = self.getOrgHonorBadges(orgName)
         oldTrophy = self.getOrgTrophies(orgName)
-        keyname = 'CONTRIB_ORG:{org}:{orgCateg}'
+        keyname = self.keyContribReq+':{org}:{orgCateg}'
         # update total points
         totOrgPnts = self.serv_redis_db.incrby(keyname.format(org=orgName, orgCateg='points'), pnts_to_add)
 
@@ -246,29 +299,29 @@ class Contributor_helper:
 
     ''' HONOR BADGES '''
     def getOrgHonorBadges(self, org):
-        keyname = 'CONTRIB_ORG:{org}:{orgCateg}'
+        keyname = '{mainKey}:{org}:{orgCateg}'
         honorBadge = []
         for i in range(1, self.honorBadgeNum+1):
-            key = keyname.format(org=org, orgCateg='BADGE_'+str(i))
+            key = keyname.format(mainKey=self.keyContribReq, org=org, orgCateg='BADGE_'+str(i))
             if self.serv_redis_db.get(key) is not None: #existing
                 honorBadge.append(i)
         return honorBadge
 
     def giveBadgeToOrg(self, org, badgeNum):
-        keyname = 'CONTRIB_ORG:{org}:{orgCateg}'
+        keyname = '{mainKey}:{org}:{orgCateg}'
-        self.serv_redis_db.set(keyname.format(org=org, orgCateg='BADGE_'+str(badgeNum)), 1)
+        self.serv_redis_db.set(keyname.format(mainKey=self.keyContribReq, org=org, orgCateg='BADGE_'+str(badgeNum)), 1)
 
     def removeBadgeFromOrg(self, org, badgeNum):
-        keyname = 'CONTRIB_ORG:{org}:{orgCateg}'
+        keyname = '{mainKey}:{org}:{orgCateg}'
-        self.serv_redis_db.delete(keyname.format(org=org, orgCateg='BADGE_'+str(badgeNum)))
+        self.serv_redis_db.delete(keyname.format(mainKey=self.keyContribReq, org=org, orgCateg='BADGE_'+str(badgeNum)))
 
     ''' TROPHIES '''
     def getOrgTrophies(self, org):
         self.getAllOrgsTrophyRanking()
-        keyname = 'CONTRIB_TROPHY:{orgCateg}'
+        keyname = '{mainKey}:{orgCateg}'
         trophy = []
         for categ in self.categories_in_trophy:
-            key = keyname.format(orgCateg=categ)
+            key = keyname.format(mainKey=self.keyTrophy, orgCateg=categ)
             totNum = self.serv_redis_db.zcard(key)
             if totNum == 0:
                 continue
@@ -281,8 +334,8 @@ class Contributor_helper:
         return trophy
 
     def getOrgsTrophyRanking(self, categ):
-        keyname = 'CONTRIB_TROPHY:{orgCateg}'
+        keyname = '{mainKey}:{orgCateg}'
-        res = self.serv_redis_db.zrange(keyname.format(orgCateg=categ), 0, -1, withscores=True, desc=True)
+        res = self.serv_redis_db.zrange(keyname.format(mainKey=self.keyTrophy, orgCateg=categ), 0, -1, withscores=True, desc=True)
         res = [[org.decode('utf8'), score] for org, score in res]
         return res
 
@@ -310,23 +363,23 @@ class Contributor_helper:
         return self.trophyNum+1 - position
 
     def giveTrophyPointsToOrg(self, org, categ, points):
-        keyname = 'CONTRIB_TROPHY:{orgCateg}'
+        keyname = '{mainKey}:{orgCateg}'
-        self.serv_redis_db.zincrby(keyname.format(orgCateg=categ), org, points)
+        self.serv_redis_db.zincrby(keyname.format(mainKey=self.keyTrophy, orgCateg=categ), org, points)
 
     def removeTrophyPointsFromOrg(self, org, categ, points):
-        keyname = 'CONTRIB_TROPHY:{orgCateg}'
+        keyname = '{mainKey}:{orgCateg}'
-        self.serv_redis_db.zincrby(keyname.format(orgCateg=categ), org, -points)
+        self.serv_redis_db.zincrby(keyname.format(mainKey=self.keyTrophy, orgCateg=categ), org, -points)
 
     ''' AWARDS HELPER '''
     def getLastAwardsFromRedis(self):
         date = datetime.datetime.now()
-        keyname = "CONTRIB_LAST_AWARDS"
+        keyname = self.keyLastAward
         prev_days = 7
         topNum = self.MAX_NUMBER_OF_LAST_CONTRIBUTOR # default Num
         addedOrg = []
         data = []
         for curDate in util.getXPrevDaysSpan(date, prev_days):
-            last_awards = self.getZrange(keyname, curDate, topNum)
+            last_awards = getZrange(self.serv_redis_db, keyname, curDate, topNum)
             for dico_award, sec in last_awards:
                 dico_award = json.loads(dico_award)
                 org = dico_award['org']
@@ -341,11 +394,10 @@ class Contributor_helper:
 
     ''' MONTHLY CONTRIBUTION '''
     def getOrgPntFromRedis(self, org, date):
-        keyCateg = 'CONTRIB_DAY'
         scoreSum = 0
         for curDate in util.getMonthSpan(date):
             date_str = util.getDateStrFormat(curDate)
-            keyname = "{}:{}".format(keyCateg, date_str)
+            keyname = "{}:{}".format(self.keyDay, date_str)
             data = self.serv_redis_db.zscore(keyname, org)
             if data is None:
                 data = 0
@@ -359,13 +411,12 @@ class Contributor_helper:
 
     def getLastContributorsFromRedis(self):
         date = datetime.datetime.now()
-        keyname = "CONTRIB_LAST"
         prev_days = 7
         topNum = self.MAX_NUMBER_OF_LAST_CONTRIBUTOR # default Num
         addedOrg = []
         data = []
         for curDate in util.getXPrevDaysSpan(date, prev_days):
-            last_contrib_org = self.getZrange(keyname, curDate, topNum)
+            last_contrib_org = getZrange(self.serv_redis_db, self.keyLastContrib, curDate, topNum)
             for org, sec in last_contrib_org:
                 if org in addedOrg:
                     continue
@@ -384,7 +435,7 @@ class Contributor_helper:
 
     def getContributorFromRedis(self, org):
         date = datetime.datetime.now()
-        epoch = self.serv_redis_db.zscore("CONTRIB_LAST", org)
+        epoch = self.serv_redis_db.zscore(self.keyLastContrib, org)
         dic = {}
         dic['rank'] = self.getOrgRankFromRedis(org, date)
         dic['orgRank'] = self.getOrgContributionRank(org)['final_rank']
@@ -398,9 +449,8 @@ class Contributor_helper:
     def getTopContributorFromRedis(self, date):
         orgDicoPnts = {}
         for curDate in util.getMonthSpan(date):
-            keyCateg = "CONTRIB_DAY"
             topNum = 0 # all
-            contrib_org = self.getZrange(keyCateg, curDate, topNum)
+            contrib_org = getZrange(self.serv_redis_db, self.keyDay, curDate, topNum)
             for org, pnts in contrib_org:
                 if org not in orgDicoPnts:
                     orgDicoPnts[org] = 0
@@ -437,7 +487,7 @@ class Contributor_helper:
         today = today.replace(hour=0, minute=0, second=0, microsecond=0)
         for curDate in util.getXPrevDaysSpan(today, 7):
             timestamp = util.getTimestamp(curDate)
-            keyname = 'CONTRIB_DAY:'+util.getDateStrFormat(curDate)
+            keyname = "{}:{}".format(self.keyDay, util.getDateStrFormat(curDate))
             org_score =  self.serv_redis_db.zscore(keyname, org)
             if org_score is None:
                 org_score = 0
@@ -446,7 +496,6 @@ class Contributor_helper:
         return to_return
 
     def getCategPerContribFromRedis(self, date):
-        keyCateg = "CONTRIB_DAY"
         topNum = 0 # all
         contrib_org = self.getTopContributorFromRedis(date)
         for dic in contrib_org:
@@ -454,7 +503,7 @@ class Contributor_helper:
             for categ in self.categories_in_datatable:
                     categ_score = 0
                     for curDate in util.getMonthSpan(date):
-                        keyname = 'CONTRIB_CATEG:'+util.getDateStrFormat(curDate)+':'+categ
+                        keyname = "{}:{}:{}".format(self.keyCateg, util.getDateStrFormat(curDate), categ)
                         temp = self.serv_redis_db.zscore(keyname, org)
                         if temp is None:
                             temp = 0
@@ -464,7 +513,7 @@ class Contributor_helper:
 
 
     def getAllOrgFromRedis(self):
-        data = self.serv_redis_db.smembers('CONTRIB_ALL_ORG')
+        data = self.serv_redis_db.smembers(self.keyAllOrg)
         data = [x.decode('utf8') for x in data]
         return data
 
@@ -502,213 +551,3 @@ class Contributor_helper:
             prev = i
         return { 'remainingPts': 0, 'stepPts': self.rankMultiplier**self.levelMax }
 
-    '''           '''
-    ''' TEST DATA '''
-    '''           '''
-
-    def TEST_getCategPerContribFromRedis(self, date):
-        data2 = []
-        for d in range(15):
-            dic = {}
-            dic['rank'] = random.randint(1,self.levelMax)
-            dic['orgRank'] = random.randint(1,self.levelMax),
-            dic['honorBadge'] = [random.randint(1,2)],
-            dic['logo_path'] = 'logo'
-            dic['org'] = 'Org'+str(d)
-            dic['pnts'] = random.randint(1,2**self.levelMax)
-            for f in self.categories_in_datatable:
-                dic[f] = random.randint(0,1600)
-            data2.append(dic)
-        return data2
-
-    def TEST_getTop5OvertimeFromRedis(self):
-        import time
-        now = time.time()
-        data2 = [
-            {'label': 'CIRCL', 'data': [[now, random.randint(1,50)], [now-util.ONE_DAY, random.randint(1,50)], [now-util.ONE_DAY*2, random.randint(1,50)], [now-util.ONE_DAY*3, random.randint(1,50)], [now-util.ONE_DAY*4, random.randint(1,50)]]},
-            {'label': 'CASES', 'data': [[now, random.randint(1,50)], [now-util.ONE_DAY, random.randint(1,50)], [now-util.ONE_DAY*2, random.randint(1,50)], [now-util.ONE_DAY*3, random.randint(1,50)], [now-util.ONE_DAY*4, random.randint(1,50)]]},
-            {'label': 'Org1', 'data': [[now, random.randint(1,50)], [now-util.ONE_DAY, random.randint(1,50)], [now-util.ONE_DAY*2, random.randint(1,50)], [now-util.ONE_DAY*3, random.randint(1,50)], [now-util.ONE_DAY*4, random.randint(1,50)]]},
-            {'label': 'Org2', 'data': [[now, random.randint(1,50)], [now-util.ONE_DAY, random.randint(1,50)], [now-util.ONE_DAY*2, random.randint(1,50)], [now-util.ONE_DAY*3, random.randint(1,50)], [now-util.ONE_DAY*4, random.randint(1,50)]]},
-            {'label': 'SMILE', 'data': [[now, random.randint(1,50)], [now-util.ONE_DAY, random.randint(1,50)], [now-util.ONE_DAY*2, random.randint(1,50)], [now-util.ONE_DAY*3, random.randint(1,50)], [now-util.ONE_DAY*4, random.randint(1,50)]]},
-        ]
-        return data2
-
-    def TEST_getOrgOvertime(self, org):
-        import time
-        now = time.time()
-        data = [
-            {'label': org, 'data': [[now, random.randint(1,30)], [now-util.ONE_DAY, random.randint(1,30)], [now-util.ONE_DAY*2, random.randint(1,30)], [now-util.ONE_DAY*3, random.randint(1,30)], [now-util.ONE_DAY*4, random.randint(1,40)]]}
-        ]
-        return data
-
-    def TEST_getTopContributorFromRedis(self, date):
-        data2 = [
-            {
-                'rank': random.randint(1,self.levelMax),
-                'orgRank': random.randint(1,self.levelMax),
-                'honorBadge': [1,2],
-                'logo_path': self.getOrgLogoFromMISP('MISP'),
-                'org': 'MISP',
-                'pnts': random.randint(1,2**self.levelMax)
-            },
-            {
-                'rank': random.randint(1,self.levelMax),
-                'orgRank': random.randint(1,self.levelMax),
-                'honorBadge': [1],
-                'logo_path': 'logo1',
-                'org': 'CIRCL',
-                'pnts': random.randint(1,2**self.levelMax)
-            },
-            {
-                'rank': random.randint(1,self.levelMax),
-                'orgRank': random.randint(1,self.levelMax),
-                'honorBadge': [2],
-                'logo_path': 'logo2',
-                'org': 'CASES',
-                'pnts': random.randint(1,2**self.levelMax)
-            },
-            {
-                'rank': random.randint(1,self.levelMax),
-                'orgRank': random.randint(1,self.levelMax),
-                'honorBadge': [],
-                'logo_path': 'logo3',
-                'org': 'SMILE',
-                'pnts': random.randint(1,2**self.levelMax)
-            },
-            {
-                'rank': random.randint(1,self.levelMax),
-                'orgRank': random.randint(1,self.levelMax),
-                'honorBadge': [],
-                'logo_path': 'logo4',
-                'org': 'ORG4',
-                'pnts': random.randint(1,2**self.levelMax)
-            },
-            {
-                'rank': random.randint(1,self.levelMax),
-                'orgRank': random.randint(1,self.levelMax),
-                'honorBadge': [],
-                'logo_path': 'logo5',
-                'org': 'ORG5',
-                'pnts': random.randint(1,2**self.levelMax)
-            },
-        ]
-        return data2*2
-
-    def TEST_getLastContributorsFromRedis(self):
-        import time
-        data2 = [
-            {
-                'rank': random.randint(1,self.levelMax),
-                'orgRank': random.randint(1,self.levelMax),
-                'honorBadge': [1,2],
-                'logo_path': self.getOrgLogoFromMISP('MISP'),
-                'org': 'MISP',
-                'pnts': random.randint(1,2**self.levelMax),
-                'epoch': time.time() - random.randint(0, 10000)
-            },
-            {
-                'rank': random.randint(1,self.levelMax),
-                'orgRank': random.randint(1,self.levelMax),
-                'honorBadge': [1],
-                'logo_path': 'logo1',
-                'org': 'CIRCL',
-                'pnts': random.randint(1,2**self.levelMax),
-                'epoch': time.time() - random.randint(0, 10000)
-            },
-            {
-                'rank': random.randint(1,self.levelMax),
-                'orgRank': random.randint(1,self.levelMax),
-                'honorBadge': [2],
-                'logo_path': 'logo2',
-                'org': 'CASES',
-                'pnts': random.randint(1,2**self.levelMax),
-                'epoch': time.time() - random.randint(0, 10000)
-            },
-            {
-                'rank': random.randint(1,self.levelMax),
-                'orgRank': random.randint(1,self.levelMax),
-                'honorBadge': [],
-                'logo_path': 'logo3',
-                'org': 'SMILE',
-                'pnts': random.randint(1,2**self.levelMax),
-                'epoch': time.time() - random.randint(0, 10000)
-            },
-            {
-                'rank': random.randint(1,self.levelMax),
-                'orgRank': random.randint(1,self.levelMax),
-                'honorBadge': [],
-                'logo_path': 'logo4',
-                'org': 'ORG4',
-                'pnts': random.randint(1,2**self.levelMax),
-                'epoch': time.time() - random.randint(0, 10000)
-            },
-            {
-                'rank': random.randint(1,self.levelMax),
-                'orgRank': random.randint(1,self.levelMax),
-                'honorBadge': [],
-                'logo_path': 'logo5',
-                'org': 'ORG5',
-                'pnts': random.randint(1,2**self.levelMax),
-                'epoch': time.time() - random.randint(0, 10000)
-            },
-        ]
-        return data2*2
-
-    def TEST_getAllOrgFromRedis(self):
-        data2 = ['CIRCL', 'CASES', 'SMILE' ,'ORG4' ,'ORG5', 'SUPER HYPER LONG ORGINZATION NAME', 'Org3', 'MISP']
-        return data2
-
-    def TEST_getCurrentOrgRankFromRedis(self, org):
-        date = datetime.datetime.now()
-        points = random.randint(1,2**self.levelMax)
-        remainingPts = self.getRemainingPoints(points)
-        data = {
-            'org': org,
-            'points': points,
-            'rank': self.getRankLevel(points),
-            'remainingPts': remainingPts['remainingPts'],
-            'stepPts': remainingPts['stepPts'],
-        }
-        return data
-
-    def TEST_getCurrentContributionStatus(self, org):
-        num = random.randint(1, self.org_rank_maxLevel)
-        requirement_fulfilled = [x for x in range(1,num+1)]
-        requirement_not_fulfilled = [x for x in range(num,self.org_rank_maxLevel+1-num)]
-
-        num2 = random.randint(1, self.org_rank_maxLevel)
-        if num2 < num-1:
-            to_swap = requirement_fulfilled[num2]
-            del requirement_fulfilled[num2]
-            requirement_not_fulfilled = [to_swap] + requirement_not_fulfilled
-
-        final_rank = len(requirement_fulfilled)
-        to_ret = {}
-        for i in range(1, self.org_rank_maxLevel+1):
-            if i in requirement_fulfilled:
-                to_ret[i] = 1
-            elif i in requirement_not_fulfilled and i<=final_rank:
-                to_ret[i] = 0
-            else:
-                to_ret[i] = -1
-        return {'rank': final_rank, 'status': to_ret, 'totPoints': random.randint(2**final_rank, 2**self.org_rank_maxLevel*4)}
-
-    def TEST_getOrgHonorBadges(self, org):
-        keyname = 'CONTRIB_ORG:{org}:{orgCateg}'
-        honorBadge = []
-        for i in range(1, self.honorBadgeNum+1):
-            key = keyname.format(org=org, orgCateg='BADGE_'+str(i))
-            if random.randint(0,1) == 1: #existing
-                honorBadge.append(1)
-            else:
-                honorBadge.append(0)
-        return honorBadge
-
-    def TEST_getOrgTrophies(self, org):
-        keyname = 'CONTRIB_ORG:{org}:{orgCateg}'
-        trophy = []
-        for categ in self.categories_in_trophy:
-            key = keyname.format(org=org, orgCateg='TROPHY_'+categ)
-            trophy_Pnts = random.randint(0,10)
-            trophy.append({'categName': categ, 'rank': trophy_Pnts})
-        return trophy

trendings_helper.py

@@ -11,6 +11,14 @@ class Trendings_helper:
         self.serv_redis_db = serv_redis_db
         self.cfg = cfg
 
+        # REDIS keys
+        self.keyEvent   = "TRENDINGS_EVENTS"
+        self.keyCateg   = "TRENDINGS_CATEGS"
+        self.keyTag     = "TRENDINGS_TAGS"
+        self.keyDisc    = "TRENDINGS_DISC"
+        self.keySigh    = "TRENDINGS_SIGHT_sightings"
+        self.keyFalse   = "TRENDINGS_SIGHT_false_positive"
+
     ''' SETTER '''
 
     def addGenericTrending(self, trendingType, data, timestamp):
@@ -24,13 +32,13 @@ class Trendings_helper:
         self.serv_redis_db.zincrby(keyname, to_save, 1)
 
     def addTrendingEvent(self, eventName, timestamp):
-        self.addGenericTrending('TRENDINGS_EVENTS', eventName, timestamp)
+        self.addGenericTrending(self.keyEvent, eventName, timestamp)
 
     def addTrendingCateg(self, categName, timestamp):
-        self.addGenericTrending('TRENDINGS_CATEGS', categName, timestamp)
+        self.addGenericTrending(self.keyCateg, categName, timestamp)
 
     def addTrendingDisc(self, eventName, timestamp):
-        self.addGenericTrending('TRENDINGS_DISC', eventName, timestamp)
+        self.addGenericTrending(self.keyDisc, eventName, timestamp)
 
     def addTrendingTags(self, tags, timestamp):
         for tag in tags:
@@ -38,18 +46,18 @@ class Trendings_helper:
             ordDic['id'] = tag['id']
             ordDic['name'] = tag['name']
             ordDic['colour'] = tag['colour']
-            self.addGenericTrending('TRENDINGS_TAGS', ordDic, timestamp)
+            self.addGenericTrending(self.keyTag, ordDic, timestamp)
 
     def addSightings(self, timestamp):
         timestampDate = datetime.datetime.fromtimestamp(float(timestamp))
         timestampDate_str = util.getDateStrFormat(timestampDate)
-        keyname = "{}:{}".format("TRENDINGS_SIGHT_sightings", timestampDate_str)
+        keyname = "{}:{}".format(self.keySigh, timestampDate_str)
         self.serv_redis_db.incrby(keyname, 1)
 
     def addFalsePositive(self, timestamp):
         timestampDate = datetime.datetime.fromtimestamp(float(timestamp))
         timestampDate_str = util.getDateStrFormat(timestampDate)
-        keyname = "{}:{}".format("TRENDINGS_SIGHT_false_positive", timestampDate_str)
+        keyname = "{}:{}".format(self.keyFalse, timestampDate_str)
         self.serv_redis_db.incrby(keyname, 1)
 
     ''' GETTER '''
@@ -77,20 +85,20 @@ class Trendings_helper:
 
     def getTrendingEvents(self, dateS, dateE, specificLabel=None):
         if specificLabel is None:
-            return self.getGenericTrending('TRENDINGS_EVENTS', dateS, dateE)
+            return self.getGenericTrending(self.keyEvent, dateS, dateE)
         else:
             specificLabel = specificLabel.replace('\\n', '\n'); # reset correctly label with their \n (CR) instead of their char value
-            return self.getSpecificTrending('TRENDINGS_EVENTS', dateS, dateE, specificLabel)
+            return self.getSpecificTrending(self.keyEvent, dateS, dateE, specificLabel)
 
     def getTrendingCategs(self, dateS, dateE):
-        return self.getGenericTrending('TRENDINGS_CATEGS', dateS, dateE)
+        return self.getGenericTrending(self.keyCateg, dateS, dateE)
 
     # FIXME: Construct this when getting data
     def getTrendingTags(self, dateS, dateE, topNum=12):
         to_ret = []
         prev_days = (dateE - dateS).days
         for curDate in util.getXPrevDaysSpan(dateE, prev_days):
-            keyname = "{}:{}".format('TRENDINGS_TAGS', util.getDateStrFormat(curDate))
+            keyname = "{}:{}".format(self.keyTag, util.getDateStrFormat(curDate))
             data = self.serv_redis_db.zrange(keyname, 0, topNum-1, desc=True, withscores=True)
             data = [ [record[0].decode('utf8'), record[1]] for record in data ]
             data = data if data is not None else []
@@ -105,21 +113,21 @@ class Trendings_helper:
         to_ret = []
         prev_days = (dateE - dateS).days
         for curDate in util.getXPrevDaysSpan(dateE, prev_days):
-            keyname = "{}:{}".format("TRENDINGS_SIGHT_sightings", util.getDateStrFormat(curDate))
+            keyname = "{}:{}".format(self.keySigh, util.getDateStrFormat(curDate))
             sight = self.serv_redis_db.get(keyname)
             sight = 0 if sight is None else int(sight.decode('utf8'))
-            keyname = "{}:{}".format("TRENDINGS_SIGHT_false_positive", util.getDateStrFormat(curDate))
+            keyname = "{}:{}".format(self.keyFalse, util.getDateStrFormat(curDate))
             fp = self.serv_redis_db.get(keyname)
             fp = 0 if fp is None else int(fp.decode('utf8'))
             to_ret.append([util.getTimestamp(curDate), { 'sightings': sight, 'false_positive': fp}])
         return to_ret
 
     def getTrendingDisc(self, dateS, dateE):
-        return self.getGenericTrending('TRENDINGS_DISC', dateS, dateE)
+        return self.getGenericTrending(self.keyDisc, dateS, dateE)
 
     def getTypeaheadData(self, dateS, dateE):
         to_ret = {}
-        for trendingType in ['TRENDINGS_EVENTS', 'TRENDINGS_CATEGS']:
+        for trendingType in [self.keyEvent, self.keyCateg]:
             allSet = set()
             prev_days = (dateE - dateS).days
             for curDate in util.getXPrevDaysSpan(dateE, prev_days):
@@ -135,5 +143,5 @@ class Trendings_helper:
             for tag in tagList:
                 tag = tag[0]
                 tagSet.add(tag['name'])
-        to_ret['TRENDINGS_TAGS'] = list(tagSet)
+        to_ret[self.keyTag] = list(tagSet)
         return to_ret

users_helper.py

@@ -4,23 +4,30 @@ import json
 import datetime, time
 
 import util
+import contributor_helper
 
 class Users_helper:
     def __init__(self, serv_redis_db, cfg):
         self.serv_redis_db = serv_redis_db
         self.cfg = cfg
+        # REDIS keys
+        self.keyTimestamp    = "LOGIN_TIMESTAMP"
+        self.keyTimestampSet = "LOGIN_TIMESTAMPSET"
+        self.keyOrgLog       = "LOGIN_ORG"
+        contrib_helper   = contributor_helper.Contributor_helper(serv_redis_db, cfg)
+        self.keyContribDay   = contrib_helper.keyDay # Key to get monthly contribution
 
     def addTemporary(self, org, timestamp):
         timestampDate = datetime.datetime.fromtimestamp(float(timestamp))
         timestampDate_str = util.getDateHoursStrFormat(timestampDate)
-        keyname_timestamp = "{}:{}".format('LOGIN_TIMESTAMPSET', timestampDate_str)
+        keyname_timestamp = "{}:{}".format(self.keyTimestampSet, timestampDate_str)
         self.serv_redis_db.sadd(keyname_timestamp, org)
         self.serv_redis_db.expire(keyname_timestamp, 60*60)
 
     def hasAlreadyBeenAdded(self, org, timestamp):
         timestampDate = datetime.datetime.fromtimestamp(float(timestamp))
         timestampDate_str = util.getDateHoursStrFormat(timestampDate)
-        keyname_timestamp = "{}:{}".format('LOGIN_TIMESTAMPSET', timestampDate_str)
+        keyname_timestamp = "{}:{}".format(self.keyTimestampSet, timestampDate_str)
         orgs = [ org.decode('utf8') for org in self.serv_redis_db.smembers(keyname_timestamp) ]
         if orgs is None:
             return False
@@ -31,21 +38,21 @@ class Users_helper:
         timestampDate_str = util.getDateStrFormat(timestampDate)
 
         if not self.hasAlreadyBeenAdded(org, timestamp):
-            keyname_timestamp = "{}:{}".format('LOGIN_TIMESTAMP', timestampDate_str)
+            keyname_timestamp = "{}:{}".format(self.keyTimestamp, timestampDate_str)
             self.serv_redis_db.sadd(keyname_timestamp, timestamp)
             self.addTemporary(org, timestamp)
 
-        keyname_org = "{}:{}".format('LOGIN_ORG', timestampDate_str)
+        keyname_org = "{}:{}".format(self.keyOrgLog, timestampDate_str)
         self.serv_redis_db.zincrby(keyname_org, org, 1)
 
     def getUserLogins(self, date):
-        keyname = "LOGIN_TIMESTAMP:{}".format(util.getDateStrFormat(date))
+        keyname = "{}:{}".format(self.keyTimestamp, util.getDateStrFormat(date))
         timestamps = self.serv_redis_db.smembers(keyname)
         timestamps = [int(timestamp.decode('utf8')) for timestamp in timestamps]
         return timestamps
 
     def getOrgslogin(self, date, topNum=12):
-        keyname = "LOGIN_ORG:{}".format(util.getDateStrFormat(date))
+        keyname = "{}:{}".format(self.keyOrgLog, util.getDateStrFormat(date))
         data = self.serv_redis_db.zrange(keyname, 0, topNum-1, desc=True, withscores=True)
         data = [ [record[0].decode('utf8'), record[1]] for record in data ]
         return data
@@ -53,20 +60,20 @@ class Users_helper:
     def getAllLoggedInOrgs(self, date, prev_days=31):
         orgs = set()
         for curDate in util.getXPrevDaysSpan(date, prev_days):
-            keyname = "LOGIN_ORG:{}".format(util.getDateStrFormat(curDate))
+            keyname = "{}:{}".format(self.keyOrgLog, util.getDateStrFormat(curDate))
             data = self.serv_redis_db.zrange(keyname, 0, -1, desc=True, withscores=True)
             for org in data:
                 orgs.add(org[0].decode('utf8'))
         return list(orgs)
 
     def getOrgContribAndLogin(self, date, org, prev_days=31):
-        keyname_log = "LOGIN_ORG:{}"
+        keyname_log = "{}:{}"
-        keyname_contrib = "CONTRIB_DAY:{}"
+        keyname_contrib = "{}:{}"
         data = []
         for curDate in util.getXPrevDaysSpan(date, prev_days):
-            log = self.serv_redis_db.zscore(keyname_log.format(util.getDateStrFormat(curDate)), org)
+            log = self.serv_redis_db.zscore(keyname_log.format(self.keyOrgLog, util.getDateStrFormat(curDate)), org)
             log = 0 if log is None else 1
-            contrib = self.serv_redis_db.zscore(keyname_contrib.format(util.getDateStrFormat(curDate)), org)
+            contrib = self.serv_redis_db.zscore(keyname_contrib.format(keyContribDay, util.getDateStrFormat(curDate)), org)
             contrib = 0 if contrib is None else 1
             data.append([log, contrib])
         return data
@@ -93,7 +100,7 @@ class Users_helper:
 
 
     def getLoginVSCOntribution(self, date):
-        keyname = "CONTRIB_DAY:{}".format(util.getDateStrFormat(date))
+        keyname = "{}:{}".format(keyContribDay, util.getDateStrFormat(date))
         orgs_contri = self.serv_redis_db.zrange(keyname, 0, -1, desc=True, withscores=False)
         orgs_contri = [ org.decode('utf8') for org in orgs_contri ]
         orgs_login = [ org[0] for org in self.getOrgslogin(date, topNum=0) ]
@@ -145,7 +152,7 @@ class Users_helper:
 
         for curDate in util.getXPrevDaysSpan(date, prev_days):
             timestamps = self.getUserLogins(curDate)
-            keyname = "CONTRIB_DAY:{}".format(util.getDateStrFormat(curDate))
+            keyname = "{}:{}".format(keyContribDay, util.getDateStrFormat(curDate))
 
             orgs_contri = self.serv_redis_db.zrange(keyname, 0, -1, desc=True, withscores=False)
             orgs_contri_num = len(orgs_contri)

util.py

@@ -9,6 +9,12 @@ def getZrange(serv_redis_db, keyCateg, date, topNum, endSubkey=""):
     data = [ [record[0].decode('utf8'), record[1]] for record in data ]
     return data
 
+def push_to_redis_zset(serv_redis_db, mainKey, toAdd, endSubkey="", count=1):
+    now = datetime.datetime.now()
+    today_str = util.getDateStrFormat(now)
+    keyname = "{}:{}{}".format(mainKey, today_str, endSubkey)
+    serv_redis_db.zincrby(keyname, toAdd, count)
+
 def getMonthSpan(date):
     ds = datetime.datetime(date.year, date.month, 1)
     dyear = 1 if ds.month+1 > 12 else 0

zmq_subscriber.py

@@ -22,18 +22,8 @@ configfile = os.path.join(os.environ['DASH_CONFIG'], 'config.cfg')
 cfg = configparser.ConfigParser()
 cfg.read(configfile)
 
-ONE_DAY = 60*60*24
 ZMQ_URL = cfg.get('RedisGlobal', 'zmq_url')
 CHANNEL = cfg.get('RedisLog', 'channel')
-CHANNEL_LASTCONTRIB = cfg.get('RedisLog', 'channelLastContributor')
-CHANNEL_LASTAWARDS = cfg.get('RedisLog', 'channelLastAwards')
-
-DEFAULT_PNTS_REWARD = cfg.get('CONTRIB', 'default_pnts_per_contribution')
-categories_in_datatable = json.loads(cfg.get('CONTRIB', 'categories_in_datatable'))
-DICO_PNTS_REWARD = {}
-temp = json.loads(cfg.get('CONTRIB', 'pnts_per_contribution'))
-for categ, pnts in temp:
-    DICO_PNTS_REWARD[categ] = pnts
 
 serv_log = redis.StrictRedis(
         host=cfg.get('RedisGlobal', 'host'),
@@ -54,12 +44,6 @@ def publish_log(zmq_name, name, content, channel=CHANNEL):
     to_send = { 'name': name, 'log': json.dumps(content), 'zmqName': zmq_name }
     serv_log.publish(channel, json.dumps(to_send))
 
-def push_to_redis_zset(keyCateg, toAdd, endSubkey="", count=1):
-    now = datetime.datetime.now()
-    today_str = util.getDateStrFormat(now)
-    keyname = "{}:{}{}".format(keyCateg, today_str, endSubkey)
-    serv_redis_db.zincrby(keyname, toAdd, count)
-
 def getFields(obj, fields):
     jsonWalker = fields.split('.')
     itemToExplore = obj
@@ -78,49 +62,6 @@ def getFields(obj, fields):
 def noSpaceLower(text):
     return text.lower().replace(' ', '_')
 
-#pntMultiplier if one contribution rewards more than others. (e.g. shighting may gives more points than editing)
-def handleContribution(zmq_name, org, contribType, categ, action, pntMultiplier=1, eventTime=datetime.datetime.now(), isLabeled=False):
-    if action in ['edit', None]:
-        pass
-        #return #not a contribution?
-
-    now = datetime.datetime.now()
-    nowSec = int(time.time())
-    pnts_to_add = DEFAULT_PNTS_REWARD
-
-    # if there is a contribution, there is a login (even if ti comes from the API)
-    users_helper.add_user_login(nowSec, org)
-
-    # is a valid contribution
-    if categ is not None:
-        try:
-            pnts_to_add = DICO_PNTS_REWARD[noSpaceLower(categ)]
-        except KeyError:
-            pnts_to_add = DEFAULT_PNTS_REWARD
-        pnts_to_add *= pntMultiplier
-
-        push_to_redis_zset('CONTRIB_DAY', org, count=pnts_to_add)
-        #CONTRIB_CATEG retain the contribution per category, not the point earned in this categ
-        push_to_redis_zset('CONTRIB_CATEG', org, count=1, endSubkey=':'+noSpaceLower(categ))
-        publish_log(zmq_name, 'CONTRIBUTION', {'org': org, 'categ': categ, 'action': action, 'epoch': nowSec }, channel=CHANNEL_LASTCONTRIB)
-    else:
-        categ = ""
-
-    serv_redis_db.sadd('CONTRIB_ALL_ORG', org)
-
-    serv_redis_db.zadd('CONTRIB_LAST:'+util.getDateStrFormat(now), nowSec, org)
-    serv_redis_db.expire('CONTRIB_LAST:'+util.getDateStrFormat(now), ONE_DAY*7) #expire after 7 day
-
-    awards_given = contributor_helper.updateOrgContributionRank(org, pnts_to_add, action, contribType, eventTime=datetime.datetime.now(), isLabeled=isLabeled, categ=noSpaceLower(categ))
-
-    for award in awards_given:
-        # update awards given
-        serv_redis_db.zadd('CONTRIB_LAST_AWARDS:'+util.getDateStrFormat(now), nowSec, json.dumps({'org': org, 'award': award, 'epoch': nowSec }))
-        serv_redis_db.expire('CONTRIB_LAST_AWARDS:'+util.getDateStrFormat(now), ONE_DAY*7) #expire after 7 day
-        # publish
-        publish_log(zmq_name, 'CONTRIBUTION', {'org': org, 'award': award, 'epoch': nowSec }, channel=CHANNEL_LASTAWARDS)
-
-
 ##############
 ## HANDLERS ##
 ##############
@@ -159,7 +100,7 @@ def handler_conversation(zmq_name, jsonevent):
     categ = None
     action = 'add'
     eventName = 'no name or id yet...'
-    handleContribution(zmq_name, org,
+    contributor_helper.handleContribution(zmq_name, org,
                     'Discussion',
                     None,
                     action,
@@ -177,17 +118,11 @@ def handler_sighting(zmq_name, jsondata):
     jsonsight = jsondata['Sighting']
     org = jsonsight['Event']['Orgc']['name']
     categ = jsonsight['Attribute']['category']
-    try:
+    action = jsondata.get('action', None)
-        action = jsondata['action']
+    contributor_helper.handleContribution(zmq_name, org, 'Sighting', categ, action, pntMultiplier=2)
-    except KeyError:
-        action = None
-    handleContribution(zmq_name, org, 'Sighting', categ, action, pntMultiplier=2)
     handler_attribute(zmq_name, jsonsight, hasAlreadyBeenContributed=True)
 
-    try:
+    timestamp = jsonsight.get('date_sighting', None)
-        timestamp = jsonsight['date_sighting']
-    except KeyError:
-        pass
 
     if jsonsight['type'] == "0": # sightings
         trendings_helper.addSightings(timestamp)
@@ -202,13 +137,12 @@ def handler_event(zmq_name, jsonobj):
     eventName = jsonevent['info']
     timestamp = jsonevent['timestamp']
     trendings_helper.addTrendingEvent(eventName, timestamp)
-    try:
+    tags = []
-        temp = jsonobj['EventTag']
+    for tag in jsonobj.get('EventTag', []):
-        tags = []
+        try:
-        for tag in temp:
             tags.append(tag['Tag'])
-    except KeyError:
+        except KeyError:
-        tags = []
+            pass
     trendings_helper.addTrendingTags(tags, timestamp)
 
     #redirect to handler_attribute
@@ -222,21 +156,12 @@ def handler_event(zmq_name, jsonobj):
         else:
             handler_attribute(zmq_name, attributes)
 
-    try:
+    action = jsonobj.get('action', None)
-        action = jsonobj['action']
+    eventLabeled = len(jsonobj.get('EventTag', [])) > 0
-    except KeyError:
+    org = jsonobj.get('Orgc', {}).get('name', None)
-        action = None
-    try:
-        eventLabeled = len(jsonobj['EventTag']) > 0
-    except KeyError:
-        eventLabeled = False
-    try:
-        org = jsonobj['Orgc']['name']
-    except KeyError:
-        org = None
 
     if org is not None:
-        handleContribution(zmq_name, org,
+        contributor_helper.handleContribution(zmq_name, org,
                         'Event',
                         None,
                         action,
@@ -249,18 +174,14 @@ def handler_attribute(zmq_name, jsonobj, hasAlreadyBeenContributed=False):
 
     #Add trending
     categName = jsonattr['category']
-    try:
+    timestamp = jsonattr.get('timestamp', int(time.time()))
-        timestamp = jsonattr['timestamp']
-    except KeyError:
-        timestamp = int(time.time())
     trendings_helper.addTrendingCateg(categName, timestamp)
-    try:
+    tags = []
-        temp = jsonattr['Tag']
+    for tag in jsonattr.get('Tag', []):
-        tags = []
+        try:
-        for tag in temp:
             tags.append(tag['Tag'])
-    except KeyError:
+        except KeyError:
-        tags = []
+            pass
     trendings_helper.addTrendingTags(tags, timestamp)
 
     to_push = []
@@ -283,15 +204,9 @@ def handler_attribute(zmq_name, jsonobj, hasAlreadyBeenContributed=False):
         geo_helper.getCoordFromPhoneAndPublish(jsonattr['value'], jsonattr['category'])
 
     if not hasAlreadyBeenContributed:
-        try:
+        eventLabeled = len(jsonobj.get('EventTag', [])) > 0
-            eventLabeled = len(jsonattr['Tag']) > 0
+        action = jsonobj.get('action', None)
-        except KeyError:
+        contributor_helper.handleContribution(zmq_name, jsonobj['Event']['Orgc']['name'],
-            eventLabeled = False
-        try:
-            action = jsonobj['action']
-        except KeyError:
-            action = None
-        handleContribution(zmq_name, jsonobj['Event']['Orgc']['name'],
                             'Attribute',
                             jsonattr['category'],
                             action,