diff --git a/config/config.cfg.default b/config/config.cfg.default index ad88f5f..eabe996 100644 --- a/config/config.cfg.default +++ b/config/config.cfg.default @@ -22,12 +22,6 @@ clusteringDistance = 10 [CONTRIB] max_number_of_last_contributor = 10 min_between_reload = 5 -#How much harder it gets to rank up (exponential multiplier) [1.5 -> +inf] -rankMultiplier = 2 -categories_in_datatable = ["internal_reference", "targeting_data", "antivirus_detection", "payload_delivery", "artifacts_dropped", "payload_installation", "persistence_mechanism", "network_activity", "payload_type", "attribution", "external_analysis", "financial_fraud", "support_Tool", "social_network", "person", "other" ] -default_pnts_per_contribution = 1 -# array of the form [[category, pntsRcv], ...] -pnts_per_contribution = [["payload_delivery", 1], ["artifacts_dropped", 1], ["network_activity", 1]] additional_help_text = ["Sightings multiplies earned points by 2", "Editing an attribute earns you the same as creating one"] [Log] diff --git a/config/ranking.cfg b/config/ranking.cfg index 5413363..6962a70 100644 --- a/config/ranking.cfg +++ b/config/ranking.cfg @@ -1,3 +1,12 @@ +[monthlyRanking] +#How much harder it gets to rank up (exponential multiplier) [1.5 -> +inf] +rankMultiplier = 2 +levelMax=16 +categories_in_datatable = ["internal_reference", "targeting_data", "antivirus_detection", "payload_delivery", "artifacts_dropped", "payload_installation", "persistence_mechanism", "network_activity", "payload_type", "attribution", "external_analysis", "financial_fraud", "support_Tool", "social_network", "person", "other" ] +default_pnts_per_contribution = 1 +# array of the form [[category, pntsRcv], ...] +pnts_per_contribution = [["payload_delivery", 1], ["artifacts_dropped", 1], ["network_activity", 1]] + [rankTitle] maxLevel=16 1=Ensign diff --git a/contributor_helper.py b/contributor_helper.py index 43be8a9..0f47d43 100644 --- a/contributor_helper.py +++ b/contributor_helper.py @@ -1,4 +1,5 @@ import util +from util import getZrange import math, random import os import configparser @@ -11,6 +12,7 @@ class Contributor_helper: self.cfg = cfg self.cfg_org_rank = configparser.ConfigParser() self.cfg_org_rank.read(os.path.join(os.environ['DASH_CONFIG'], 'ranking.cfg')) + self.CHANNEL_LASTAWARDS = cfg.get('RedisLog', 'channelLastAwards') #honorBadge self.honorBadgeNum = len(self.cfg_org_rank.options('HonorBadge')) @@ -45,11 +47,11 @@ class Contributor_helper: #WEB STUFF self.misp_web_url = cfg.get('RedisGlobal', 'misp_web_url') self.MAX_NUMBER_OF_LAST_CONTRIBUTOR = cfg.getint('CONTRIB', 'max_number_of_last_contributor') - self.categories_in_datatable = json.loads(cfg.get('CONTRIB', 'categories_in_datatable')) + self.categories_in_datatable = json.loads(self.cfg_org_rank.get('monthlyRanking', 'categories_in_datatable')) #MONTHLY RANKING - self.default_pnts_per_contribution = json.loads(cfg.get('CONTRIB', 'default_pnts_per_contribution')) - temp = json.loads(cfg.get('CONTRIB', 'pnts_per_contribution')) + self.default_pnts_per_contribution = json.loads(self.cfg_org_rank.get('monthlyRanking', 'default_pnts_per_contribution')) + temp = json.loads(self.cfg_org_rank.get('monthlyRanking', 'pnts_per_contribution')) self.DICO_PNTS_REWARD = {} for categ, pnts in temp: self.DICO_PNTS_REWARD[categ] = pnts @@ -60,30 +62,81 @@ class Contributor_helper: else: self.DICO_PNTS_REWARD[categ] = self.default_pnts_per_contribution - self.rankMultiplier = cfg.getfloat('CONTRIB' ,'rankMultiplier') - self.levelMax = 16 + self.rankMultiplier = self.cfg_org_rank.getfloat('monthlyRanking' ,'rankMultiplier') + self.levelMax = self.cfg_org_rank.getfloat('monthlyRanking' ,'levelMax') + + # REDIS KEYS + self.keyDay = "CONTRIB_DAY" + self.keyCateg = "CONTRIB_CATEG" + self.keyLastContrib = "CONTRIB_LAST" + self.keyAllOrg = "CONTRIB_ALL_ORG" + self.keyContribReq = "CONTRIB_ORG" + self.keyTrophy = "CONTRIB_TROPHY" + self.keyLastAward = "CONTRIB_LAST_AWARDS" ''' HELPER ''' def getOrgLogoFromMISP(self, org): return "{}/img/orgs/{}.png".format(self.misp_web_url, org) - def getZrange(self, keyCateg, date, topNum, endSubkey=""): - date_str = util.getDateStrFormat(date) - keyname = "{}:{}{}".format(keyCateg, date_str, endSubkey) - data = self.serv_redis_db.zrange(keyname, 0, topNum-1, desc=True, withscores=True) - data = [ [record[0].decode('utf8'), record[1]] for record in data ] - return data - def addContributionToCateg(self, date, categ, org, count=1): today_str = util.getDateStrFormat(date) - keyname = "CONTRIB_CATEG:{}:{}".format(today_str, categ) + keyname = "{}:{}:{}".format(self.keyCateg, today_str, categ) self.serv_redis_db.zincrby(keyname, org, count) + def publish_log(self, zmq_name, name, content, channel=""): + to_send = { 'name': name, 'log': json.dumps(content), 'zmqName': zmq_name } + serv_log.publish(channel, json.dumps(to_send)) + + ''' HANDLER ''' + #pntMultiplier if one contribution rewards more than others. (e.g. shighting may gives more points than editing) + def handleContribution(self, zmq_name, org, contribType, categ, action, pntMultiplier=1, eventTime=datetime.datetime.now(), isLabeled=False): + if action in ['edit', None]: + pass + #return #not a contribution? + + now = datetime.datetime.now() + nowSec = int(time.time()) + pnts_to_add = self.default_pnts_per_contribution + + # if there is a contribution, there is a login (even if ti comes from the API) + users_helper.add_user_login(nowSec, org) + + # is a valid contribution + if categ is not None: + try: + pnts_to_add = self.DICO_PNTS_REWARD[noSpaceLower(categ)] + except KeyError: + pnts_to_add = self.default_pnts_per_contribution + pnts_to_add *= pntMultiplier + + util.push_to_redis_zset(self.serv_redis_db, self.keyDay, org, count=pnts_to_add) + #CONTRIB_CATEG retain the contribution per category, not the point earned in this categ + util.push_to_redis_zset(self.serv_redis_db, self.keyCateg, org, count=1, endSubkey=':'+noSpaceLower(categ)) + self.publish_log(zmq_name, 'CONTRIBUTION', {'org': org, 'categ': categ, 'action': action, 'epoch': nowSec }, channel=CHANNEL_LASTCONTRIB) + else: + categ = "" + + serv_redis_db.sadd(self.keyAllOrg, org) + + keyname = "{}:{}".format(self.keyLastContrib, util.getDateStrFormat(now)) + serv_redis_db.zadd(keyname, nowSec, org) + serv_redis_db.expire(keyname, ONE_DAY*7) #expire after 7 day + + awards_given = self.updateOrgContributionRank(org, pnts_to_add, action, contribType, eventTime=datetime.datetime.now(), isLabeled=isLabeled, categ=noSpaceLower(categ)) + + for award in awards_given: + # update awards given + keyname = "{}:{}".format(self.keyLastAward, util.getDateStrFormat(now)) + serv_redis_db.zadd(keyname, nowSec, json.dumps({'org': org, 'award': award, 'epoch': nowSec })) + serv_redis_db.expire(keyname, ONE_DAY*7) #expire after 7 day + # publish + self.publish_log(zmq_name, 'CONTRIBUTION', {'org': org, 'award': award, 'epoch': nowSec }, channel=self.CHANNEL_LASTAWARDS) + ''' CONTRIBUTION RANK ''' def getOrgContributionTotalPoints(self, org): - keyname = 'CONTRIB_ORG:{org}:{orgCateg}' - pnts = self.serv_redis_db.get(keyname.format(org=org, orgCateg='points')) + keyname = '{mainKey}:{org}:{orgCateg}' + pnts = self.serv_redis_db.get(keyname.format(mainKey=self.keyContribReq, org=org, orgCateg='points')) if pnts is None: pnts = 0 else: @@ -92,12 +145,12 @@ class Contributor_helper: # return: [final_rank, requirement_fulfilled, requirement_not_fulfilled] def getOrgContributionRank(self, org): - keyname = 'CONTRIB_ORG:{org}:{orgCateg}' + keyname = '{mainKey}:{org}:{orgCateg}' final_rank = 0 requirement_fulfilled = [] requirement_not_fulfilled = [] for i in range(1, self.org_rank_maxLevel+1): - key = keyname.format(org=org, orgCateg='CONTRIB_REQ_'+str(i)) + key = keyname.format(mainKey=self.keyContribReq, org=org, orgCateg='CONTRIB_REQ_'+str(i)) if self.serv_redis_db.get(key) is None: #non existing requirement_not_fulfilled.append(i) else: @@ -109,12 +162,12 @@ class Contributor_helper: return {'final_rank': final_rank, 'req_fulfilled': requirement_fulfilled, 'req_not_fulfilled': requirement_not_fulfilled} def giveContribRankToOrg(self, org, rankNum): - keyname = 'CONTRIB_ORG:{org}:{orgCateg}' - self.serv_redis_db.set(keyname.format(org=org, orgCateg='CONTRIB_REQ_'+str(rankNum)), 1) + keyname = '{mainKey}:{org}:{orgCateg}' + self.serv_redis_db.set(keyname.format(mainKey=self.keyContribReq, org=org, orgCateg='CONTRIB_REQ_'+str(rankNum)), 1) def removeContribRankFromOrg(self, org, rankNum): - keyname = 'CONTRIB_ORG:{org}:{orgCateg}' - self.serv_redis_db.delete(keyname.format(org=org, orgCateg='CONTRIB_REQ_'+str(rankNum))) + keyname = '{mainKey}:{org}:{orgCateg}' + self.serv_redis_db.delete(keyname.format(mainKey=self.keyContribReq, org=org, orgCateg='CONTRIB_REQ_'+str(rankNum))) # 1 for fulfilled, 0 for not fulfilled, -1 for not relevant def getCurrentContributionStatus(self, org): @@ -138,7 +191,7 @@ class Contributor_helper: oldContributionStatus = ContributionStatus['status'] oldHonorBadges = self.getOrgHonorBadges(orgName) oldTrophy = self.getOrgTrophies(orgName) - keyname = 'CONTRIB_ORG:{org}:{orgCateg}' + keyname = self.keyContribReq+':{org}:{orgCateg}' # update total points totOrgPnts = self.serv_redis_db.incrby(keyname.format(org=orgName, orgCateg='points'), pnts_to_add) @@ -246,29 +299,29 @@ class Contributor_helper: ''' HONOR BADGES ''' def getOrgHonorBadges(self, org): - keyname = 'CONTRIB_ORG:{org}:{orgCateg}' + keyname = '{mainKey}:{org}:{orgCateg}' honorBadge = [] for i in range(1, self.honorBadgeNum+1): - key = keyname.format(org=org, orgCateg='BADGE_'+str(i)) + key = keyname.format(mainKey=self.keyContribReq, org=org, orgCateg='BADGE_'+str(i)) if self.serv_redis_db.get(key) is not None: #existing honorBadge.append(i) return honorBadge def giveBadgeToOrg(self, org, badgeNum): - keyname = 'CONTRIB_ORG:{org}:{orgCateg}' - self.serv_redis_db.set(keyname.format(org=org, orgCateg='BADGE_'+str(badgeNum)), 1) + keyname = '{mainKey}:{org}:{orgCateg}' + self.serv_redis_db.set(keyname.format(mainKey=self.keyContribReq, org=org, orgCateg='BADGE_'+str(badgeNum)), 1) def removeBadgeFromOrg(self, org, badgeNum): - keyname = 'CONTRIB_ORG:{org}:{orgCateg}' - self.serv_redis_db.delete(keyname.format(org=org, orgCateg='BADGE_'+str(badgeNum))) + keyname = '{mainKey}:{org}:{orgCateg}' + self.serv_redis_db.delete(keyname.format(mainKey=self.keyContribReq, org=org, orgCateg='BADGE_'+str(badgeNum))) ''' TROPHIES ''' def getOrgTrophies(self, org): self.getAllOrgsTrophyRanking() - keyname = 'CONTRIB_TROPHY:{orgCateg}' + keyname = '{mainKey}:{orgCateg}' trophy = [] for categ in self.categories_in_trophy: - key = keyname.format(orgCateg=categ) + key = keyname.format(mainKey=self.keyTrophy, orgCateg=categ) totNum = self.serv_redis_db.zcard(key) if totNum == 0: continue @@ -281,8 +334,8 @@ class Contributor_helper: return trophy def getOrgsTrophyRanking(self, categ): - keyname = 'CONTRIB_TROPHY:{orgCateg}' - res = self.serv_redis_db.zrange(keyname.format(orgCateg=categ), 0, -1, withscores=True, desc=True) + keyname = '{mainKey}:{orgCateg}' + res = self.serv_redis_db.zrange(keyname.format(mainKey=self.keyTrophy, orgCateg=categ), 0, -1, withscores=True, desc=True) res = [[org.decode('utf8'), score] for org, score in res] return res @@ -310,23 +363,23 @@ class Contributor_helper: return self.trophyNum+1 - position def giveTrophyPointsToOrg(self, org, categ, points): - keyname = 'CONTRIB_TROPHY:{orgCateg}' - self.serv_redis_db.zincrby(keyname.format(orgCateg=categ), org, points) + keyname = '{mainKey}:{orgCateg}' + self.serv_redis_db.zincrby(keyname.format(mainKey=self.keyTrophy, orgCateg=categ), org, points) def removeTrophyPointsFromOrg(self, org, categ, points): - keyname = 'CONTRIB_TROPHY:{orgCateg}' - self.serv_redis_db.zincrby(keyname.format(orgCateg=categ), org, -points) + keyname = '{mainKey}:{orgCateg}' + self.serv_redis_db.zincrby(keyname.format(mainKey=self.keyTrophy, orgCateg=categ), org, -points) ''' AWARDS HELPER ''' def getLastAwardsFromRedis(self): date = datetime.datetime.now() - keyname = "CONTRIB_LAST_AWARDS" + keyname = self.keyLastAward prev_days = 7 topNum = self.MAX_NUMBER_OF_LAST_CONTRIBUTOR # default Num addedOrg = [] data = [] for curDate in util.getXPrevDaysSpan(date, prev_days): - last_awards = self.getZrange(keyname, curDate, topNum) + last_awards = getZrange(self.serv_redis_db, keyname, curDate, topNum) for dico_award, sec in last_awards: dico_award = json.loads(dico_award) org = dico_award['org'] @@ -341,11 +394,10 @@ class Contributor_helper: ''' MONTHLY CONTRIBUTION ''' def getOrgPntFromRedis(self, org, date): - keyCateg = 'CONTRIB_DAY' scoreSum = 0 for curDate in util.getMonthSpan(date): date_str = util.getDateStrFormat(curDate) - keyname = "{}:{}".format(keyCateg, date_str) + keyname = "{}:{}".format(self.keyDay, date_str) data = self.serv_redis_db.zscore(keyname, org) if data is None: data = 0 @@ -359,13 +411,12 @@ class Contributor_helper: def getLastContributorsFromRedis(self): date = datetime.datetime.now() - keyname = "CONTRIB_LAST" prev_days = 7 topNum = self.MAX_NUMBER_OF_LAST_CONTRIBUTOR # default Num addedOrg = [] data = [] for curDate in util.getXPrevDaysSpan(date, prev_days): - last_contrib_org = self.getZrange(keyname, curDate, topNum) + last_contrib_org = getZrange(self.serv_redis_db, self.keyLastContrib, curDate, topNum) for org, sec in last_contrib_org: if org in addedOrg: continue @@ -384,7 +435,7 @@ class Contributor_helper: def getContributorFromRedis(self, org): date = datetime.datetime.now() - epoch = self.serv_redis_db.zscore("CONTRIB_LAST", org) + epoch = self.serv_redis_db.zscore(self.keyLastContrib, org) dic = {} dic['rank'] = self.getOrgRankFromRedis(org, date) dic['orgRank'] = self.getOrgContributionRank(org)['final_rank'] @@ -398,9 +449,8 @@ class Contributor_helper: def getTopContributorFromRedis(self, date): orgDicoPnts = {} for curDate in util.getMonthSpan(date): - keyCateg = "CONTRIB_DAY" topNum = 0 # all - contrib_org = self.getZrange(keyCateg, curDate, topNum) + contrib_org = getZrange(self.serv_redis_db, self.keyDay, curDate, topNum) for org, pnts in contrib_org: if org not in orgDicoPnts: orgDicoPnts[org] = 0 @@ -437,7 +487,7 @@ class Contributor_helper: today = today.replace(hour=0, minute=0, second=0, microsecond=0) for curDate in util.getXPrevDaysSpan(today, 7): timestamp = util.getTimestamp(curDate) - keyname = 'CONTRIB_DAY:'+util.getDateStrFormat(curDate) + keyname = "{}:{}".format(self.keyDay, util.getDateStrFormat(curDate)) org_score = self.serv_redis_db.zscore(keyname, org) if org_score is None: org_score = 0 @@ -446,7 +496,6 @@ class Contributor_helper: return to_return def getCategPerContribFromRedis(self, date): - keyCateg = "CONTRIB_DAY" topNum = 0 # all contrib_org = self.getTopContributorFromRedis(date) for dic in contrib_org: @@ -454,7 +503,7 @@ class Contributor_helper: for categ in self.categories_in_datatable: categ_score = 0 for curDate in util.getMonthSpan(date): - keyname = 'CONTRIB_CATEG:'+util.getDateStrFormat(curDate)+':'+categ + keyname = "{}:{}:{}".format(self.keyCateg, util.getDateStrFormat(curDate), categ) temp = self.serv_redis_db.zscore(keyname, org) if temp is None: temp = 0 @@ -464,7 +513,7 @@ class Contributor_helper: def getAllOrgFromRedis(self): - data = self.serv_redis_db.smembers('CONTRIB_ALL_ORG') + data = self.serv_redis_db.smembers(self.keyAllOrg) data = [x.decode('utf8') for x in data] return data @@ -502,213 +551,3 @@ class Contributor_helper: prev = i return { 'remainingPts': 0, 'stepPts': self.rankMultiplier**self.levelMax } - ''' ''' - ''' TEST DATA ''' - ''' ''' - - def TEST_getCategPerContribFromRedis(self, date): - data2 = [] - for d in range(15): - dic = {} - dic['rank'] = random.randint(1,self.levelMax) - dic['orgRank'] = random.randint(1,self.levelMax), - dic['honorBadge'] = [random.randint(1,2)], - dic['logo_path'] = 'logo' - dic['org'] = 'Org'+str(d) - dic['pnts'] = random.randint(1,2**self.levelMax) - for f in self.categories_in_datatable: - dic[f] = random.randint(0,1600) - data2.append(dic) - return data2 - - def TEST_getTop5OvertimeFromRedis(self): - import time - now = time.time() - data2 = [ - {'label': 'CIRCL', 'data': [[now, random.randint(1,50)], [now-util.ONE_DAY, random.randint(1,50)], [now-util.ONE_DAY*2, random.randint(1,50)], [now-util.ONE_DAY*3, random.randint(1,50)], [now-util.ONE_DAY*4, random.randint(1,50)]]}, - {'label': 'CASES', 'data': [[now, random.randint(1,50)], [now-util.ONE_DAY, random.randint(1,50)], [now-util.ONE_DAY*2, random.randint(1,50)], [now-util.ONE_DAY*3, random.randint(1,50)], [now-util.ONE_DAY*4, random.randint(1,50)]]}, - {'label': 'Org1', 'data': [[now, random.randint(1,50)], [now-util.ONE_DAY, random.randint(1,50)], [now-util.ONE_DAY*2, random.randint(1,50)], [now-util.ONE_DAY*3, random.randint(1,50)], [now-util.ONE_DAY*4, random.randint(1,50)]]}, - {'label': 'Org2', 'data': [[now, random.randint(1,50)], [now-util.ONE_DAY, random.randint(1,50)], [now-util.ONE_DAY*2, random.randint(1,50)], [now-util.ONE_DAY*3, random.randint(1,50)], [now-util.ONE_DAY*4, random.randint(1,50)]]}, - {'label': 'SMILE', 'data': [[now, random.randint(1,50)], [now-util.ONE_DAY, random.randint(1,50)], [now-util.ONE_DAY*2, random.randint(1,50)], [now-util.ONE_DAY*3, random.randint(1,50)], [now-util.ONE_DAY*4, random.randint(1,50)]]}, - ] - return data2 - - def TEST_getOrgOvertime(self, org): - import time - now = time.time() - data = [ - {'label': org, 'data': [[now, random.randint(1,30)], [now-util.ONE_DAY, random.randint(1,30)], [now-util.ONE_DAY*2, random.randint(1,30)], [now-util.ONE_DAY*3, random.randint(1,30)], [now-util.ONE_DAY*4, random.randint(1,40)]]} - ] - return data - - def TEST_getTopContributorFromRedis(self, date): - data2 = [ - { - 'rank': random.randint(1,self.levelMax), - 'orgRank': random.randint(1,self.levelMax), - 'honorBadge': [1,2], - 'logo_path': self.getOrgLogoFromMISP('MISP'), - 'org': 'MISP', - 'pnts': random.randint(1,2**self.levelMax) - }, - { - 'rank': random.randint(1,self.levelMax), - 'orgRank': random.randint(1,self.levelMax), - 'honorBadge': [1], - 'logo_path': 'logo1', - 'org': 'CIRCL', - 'pnts': random.randint(1,2**self.levelMax) - }, - { - 'rank': random.randint(1,self.levelMax), - 'orgRank': random.randint(1,self.levelMax), - 'honorBadge': [2], - 'logo_path': 'logo2', - 'org': 'CASES', - 'pnts': random.randint(1,2**self.levelMax) - }, - { - 'rank': random.randint(1,self.levelMax), - 'orgRank': random.randint(1,self.levelMax), - 'honorBadge': [], - 'logo_path': 'logo3', - 'org': 'SMILE', - 'pnts': random.randint(1,2**self.levelMax) - }, - { - 'rank': random.randint(1,self.levelMax), - 'orgRank': random.randint(1,self.levelMax), - 'honorBadge': [], - 'logo_path': 'logo4', - 'org': 'ORG4', - 'pnts': random.randint(1,2**self.levelMax) - }, - { - 'rank': random.randint(1,self.levelMax), - 'orgRank': random.randint(1,self.levelMax), - 'honorBadge': [], - 'logo_path': 'logo5', - 'org': 'ORG5', - 'pnts': random.randint(1,2**self.levelMax) - }, - ] - return data2*2 - - def TEST_getLastContributorsFromRedis(self): - import time - data2 = [ - { - 'rank': random.randint(1,self.levelMax), - 'orgRank': random.randint(1,self.levelMax), - 'honorBadge': [1,2], - 'logo_path': self.getOrgLogoFromMISP('MISP'), - 'org': 'MISP', - 'pnts': random.randint(1,2**self.levelMax), - 'epoch': time.time() - random.randint(0, 10000) - }, - { - 'rank': random.randint(1,self.levelMax), - 'orgRank': random.randint(1,self.levelMax), - 'honorBadge': [1], - 'logo_path': 'logo1', - 'org': 'CIRCL', - 'pnts': random.randint(1,2**self.levelMax), - 'epoch': time.time() - random.randint(0, 10000) - }, - { - 'rank': random.randint(1,self.levelMax), - 'orgRank': random.randint(1,self.levelMax), - 'honorBadge': [2], - 'logo_path': 'logo2', - 'org': 'CASES', - 'pnts': random.randint(1,2**self.levelMax), - 'epoch': time.time() - random.randint(0, 10000) - }, - { - 'rank': random.randint(1,self.levelMax), - 'orgRank': random.randint(1,self.levelMax), - 'honorBadge': [], - 'logo_path': 'logo3', - 'org': 'SMILE', - 'pnts': random.randint(1,2**self.levelMax), - 'epoch': time.time() - random.randint(0, 10000) - }, - { - 'rank': random.randint(1,self.levelMax), - 'orgRank': random.randint(1,self.levelMax), - 'honorBadge': [], - 'logo_path': 'logo4', - 'org': 'ORG4', - 'pnts': random.randint(1,2**self.levelMax), - 'epoch': time.time() - random.randint(0, 10000) - }, - { - 'rank': random.randint(1,self.levelMax), - 'orgRank': random.randint(1,self.levelMax), - 'honorBadge': [], - 'logo_path': 'logo5', - 'org': 'ORG5', - 'pnts': random.randint(1,2**self.levelMax), - 'epoch': time.time() - random.randint(0, 10000) - }, - ] - return data2*2 - - def TEST_getAllOrgFromRedis(self): - data2 = ['CIRCL', 'CASES', 'SMILE' ,'ORG4' ,'ORG5', 'SUPER HYPER LONG ORGINZATION NAME', 'Org3', 'MISP'] - return data2 - - def TEST_getCurrentOrgRankFromRedis(self, org): - date = datetime.datetime.now() - points = random.randint(1,2**self.levelMax) - remainingPts = self.getRemainingPoints(points) - data = { - 'org': org, - 'points': points, - 'rank': self.getRankLevel(points), - 'remainingPts': remainingPts['remainingPts'], - 'stepPts': remainingPts['stepPts'], - } - return data - - def TEST_getCurrentContributionStatus(self, org): - num = random.randint(1, self.org_rank_maxLevel) - requirement_fulfilled = [x for x in range(1,num+1)] - requirement_not_fulfilled = [x for x in range(num,self.org_rank_maxLevel+1-num)] - - num2 = random.randint(1, self.org_rank_maxLevel) - if num2 < num-1: - to_swap = requirement_fulfilled[num2] - del requirement_fulfilled[num2] - requirement_not_fulfilled = [to_swap] + requirement_not_fulfilled - - final_rank = len(requirement_fulfilled) - to_ret = {} - for i in range(1, self.org_rank_maxLevel+1): - if i in requirement_fulfilled: - to_ret[i] = 1 - elif i in requirement_not_fulfilled and i<=final_rank: - to_ret[i] = 0 - else: - to_ret[i] = -1 - return {'rank': final_rank, 'status': to_ret, 'totPoints': random.randint(2**final_rank, 2**self.org_rank_maxLevel*4)} - - def TEST_getOrgHonorBadges(self, org): - keyname = 'CONTRIB_ORG:{org}:{orgCateg}' - honorBadge = [] - for i in range(1, self.honorBadgeNum+1): - key = keyname.format(org=org, orgCateg='BADGE_'+str(i)) - if random.randint(0,1) == 1: #existing - honorBadge.append(1) - else: - honorBadge.append(0) - return honorBadge - - def TEST_getOrgTrophies(self, org): - keyname = 'CONTRIB_ORG:{org}:{orgCateg}' - trophy = [] - for categ in self.categories_in_trophy: - key = keyname.format(org=org, orgCateg='TROPHY_'+categ) - trophy_Pnts = random.randint(0,10) - trophy.append({'categName': categ, 'rank': trophy_Pnts}) - return trophy diff --git a/trendings_helper.py b/trendings_helper.py index cd04010..19c5dd6 100644 --- a/trendings_helper.py +++ b/trendings_helper.py @@ -11,6 +11,14 @@ class Trendings_helper: self.serv_redis_db = serv_redis_db self.cfg = cfg + # REDIS keys + self.keyEvent = "TRENDINGS_EVENTS" + self.keyCateg = "TRENDINGS_CATEGS" + self.keyTag = "TRENDINGS_TAGS" + self.keyDisc = "TRENDINGS_DISC" + self.keySigh = "TRENDINGS_SIGHT_sightings" + self.keyFalse = "TRENDINGS_SIGHT_false_positive" + ''' SETTER ''' def addGenericTrending(self, trendingType, data, timestamp): @@ -24,13 +32,13 @@ class Trendings_helper: self.serv_redis_db.zincrby(keyname, to_save, 1) def addTrendingEvent(self, eventName, timestamp): - self.addGenericTrending('TRENDINGS_EVENTS', eventName, timestamp) + self.addGenericTrending(self.keyEvent, eventName, timestamp) def addTrendingCateg(self, categName, timestamp): - self.addGenericTrending('TRENDINGS_CATEGS', categName, timestamp) + self.addGenericTrending(self.keyCateg, categName, timestamp) def addTrendingDisc(self, eventName, timestamp): - self.addGenericTrending('TRENDINGS_DISC', eventName, timestamp) + self.addGenericTrending(self.keyDisc, eventName, timestamp) def addTrendingTags(self, tags, timestamp): for tag in tags: @@ -38,18 +46,18 @@ class Trendings_helper: ordDic['id'] = tag['id'] ordDic['name'] = tag['name'] ordDic['colour'] = tag['colour'] - self.addGenericTrending('TRENDINGS_TAGS', ordDic, timestamp) + self.addGenericTrending(self.keyTag, ordDic, timestamp) def addSightings(self, timestamp): timestampDate = datetime.datetime.fromtimestamp(float(timestamp)) timestampDate_str = util.getDateStrFormat(timestampDate) - keyname = "{}:{}".format("TRENDINGS_SIGHT_sightings", timestampDate_str) + keyname = "{}:{}".format(self.keySigh, timestampDate_str) self.serv_redis_db.incrby(keyname, 1) def addFalsePositive(self, timestamp): timestampDate = datetime.datetime.fromtimestamp(float(timestamp)) timestampDate_str = util.getDateStrFormat(timestampDate) - keyname = "{}:{}".format("TRENDINGS_SIGHT_false_positive", timestampDate_str) + keyname = "{}:{}".format(self.keyFalse, timestampDate_str) self.serv_redis_db.incrby(keyname, 1) ''' GETTER ''' @@ -77,20 +85,20 @@ class Trendings_helper: def getTrendingEvents(self, dateS, dateE, specificLabel=None): if specificLabel is None: - return self.getGenericTrending('TRENDINGS_EVENTS', dateS, dateE) + return self.getGenericTrending(self.keyEvent, dateS, dateE) else: specificLabel = specificLabel.replace('\\n', '\n'); # reset correctly label with their \n (CR) instead of their char value - return self.getSpecificTrending('TRENDINGS_EVENTS', dateS, dateE, specificLabel) + return self.getSpecificTrending(self.keyEvent, dateS, dateE, specificLabel) def getTrendingCategs(self, dateS, dateE): - return self.getGenericTrending('TRENDINGS_CATEGS', dateS, dateE) + return self.getGenericTrending(self.keyCateg, dateS, dateE) # FIXME: Construct this when getting data def getTrendingTags(self, dateS, dateE, topNum=12): to_ret = [] prev_days = (dateE - dateS).days for curDate in util.getXPrevDaysSpan(dateE, prev_days): - keyname = "{}:{}".format('TRENDINGS_TAGS', util.getDateStrFormat(curDate)) + keyname = "{}:{}".format(self.keyTag, util.getDateStrFormat(curDate)) data = self.serv_redis_db.zrange(keyname, 0, topNum-1, desc=True, withscores=True) data = [ [record[0].decode('utf8'), record[1]] for record in data ] data = data if data is not None else [] @@ -105,21 +113,21 @@ class Trendings_helper: to_ret = [] prev_days = (dateE - dateS).days for curDate in util.getXPrevDaysSpan(dateE, prev_days): - keyname = "{}:{}".format("TRENDINGS_SIGHT_sightings", util.getDateStrFormat(curDate)) + keyname = "{}:{}".format(self.keySigh, util.getDateStrFormat(curDate)) sight = self.serv_redis_db.get(keyname) sight = 0 if sight is None else int(sight.decode('utf8')) - keyname = "{}:{}".format("TRENDINGS_SIGHT_false_positive", util.getDateStrFormat(curDate)) + keyname = "{}:{}".format(self.keyFalse, util.getDateStrFormat(curDate)) fp = self.serv_redis_db.get(keyname) fp = 0 if fp is None else int(fp.decode('utf8')) to_ret.append([util.getTimestamp(curDate), { 'sightings': sight, 'false_positive': fp}]) return to_ret def getTrendingDisc(self, dateS, dateE): - return self.getGenericTrending('TRENDINGS_DISC', dateS, dateE) + return self.getGenericTrending(self.keyDisc, dateS, dateE) def getTypeaheadData(self, dateS, dateE): to_ret = {} - for trendingType in ['TRENDINGS_EVENTS', 'TRENDINGS_CATEGS']: + for trendingType in [self.keyEvent, self.keyCateg]: allSet = set() prev_days = (dateE - dateS).days for curDate in util.getXPrevDaysSpan(dateE, prev_days): @@ -135,5 +143,5 @@ class Trendings_helper: for tag in tagList: tag = tag[0] tagSet.add(tag['name']) - to_ret['TRENDINGS_TAGS'] = list(tagSet) + to_ret[self.keyTag] = list(tagSet) return to_ret diff --git a/users_helper.py b/users_helper.py index 48aed17..80ad79f 100644 --- a/users_helper.py +++ b/users_helper.py @@ -4,23 +4,30 @@ import json import datetime, time import util +import contributor_helper class Users_helper: def __init__(self, serv_redis_db, cfg): self.serv_redis_db = serv_redis_db self.cfg = cfg + # REDIS keys + self.keyTimestamp = "LOGIN_TIMESTAMP" + self.keyTimestampSet = "LOGIN_TIMESTAMPSET" + self.keyOrgLog = "LOGIN_ORG" + contrib_helper = contributor_helper.Contributor_helper(serv_redis_db, cfg) + self.keyContribDay = contrib_helper.keyDay # Key to get monthly contribution def addTemporary(self, org, timestamp): timestampDate = datetime.datetime.fromtimestamp(float(timestamp)) timestampDate_str = util.getDateHoursStrFormat(timestampDate) - keyname_timestamp = "{}:{}".format('LOGIN_TIMESTAMPSET', timestampDate_str) + keyname_timestamp = "{}:{}".format(self.keyTimestampSet, timestampDate_str) self.serv_redis_db.sadd(keyname_timestamp, org) self.serv_redis_db.expire(keyname_timestamp, 60*60) def hasAlreadyBeenAdded(self, org, timestamp): timestampDate = datetime.datetime.fromtimestamp(float(timestamp)) timestampDate_str = util.getDateHoursStrFormat(timestampDate) - keyname_timestamp = "{}:{}".format('LOGIN_TIMESTAMPSET', timestampDate_str) + keyname_timestamp = "{}:{}".format(self.keyTimestampSet, timestampDate_str) orgs = [ org.decode('utf8') for org in self.serv_redis_db.smembers(keyname_timestamp) ] if orgs is None: return False @@ -31,21 +38,21 @@ class Users_helper: timestampDate_str = util.getDateStrFormat(timestampDate) if not self.hasAlreadyBeenAdded(org, timestamp): - keyname_timestamp = "{}:{}".format('LOGIN_TIMESTAMP', timestampDate_str) + keyname_timestamp = "{}:{}".format(self.keyTimestamp, timestampDate_str) self.serv_redis_db.sadd(keyname_timestamp, timestamp) self.addTemporary(org, timestamp) - keyname_org = "{}:{}".format('LOGIN_ORG', timestampDate_str) + keyname_org = "{}:{}".format(self.keyOrgLog, timestampDate_str) self.serv_redis_db.zincrby(keyname_org, org, 1) def getUserLogins(self, date): - keyname = "LOGIN_TIMESTAMP:{}".format(util.getDateStrFormat(date)) + keyname = "{}:{}".format(self.keyTimestamp, util.getDateStrFormat(date)) timestamps = self.serv_redis_db.smembers(keyname) timestamps = [int(timestamp.decode('utf8')) for timestamp in timestamps] return timestamps def getOrgslogin(self, date, topNum=12): - keyname = "LOGIN_ORG:{}".format(util.getDateStrFormat(date)) + keyname = "{}:{}".format(self.keyOrgLog, util.getDateStrFormat(date)) data = self.serv_redis_db.zrange(keyname, 0, topNum-1, desc=True, withscores=True) data = [ [record[0].decode('utf8'), record[1]] for record in data ] return data @@ -53,20 +60,20 @@ class Users_helper: def getAllLoggedInOrgs(self, date, prev_days=31): orgs = set() for curDate in util.getXPrevDaysSpan(date, prev_days): - keyname = "LOGIN_ORG:{}".format(util.getDateStrFormat(curDate)) + keyname = "{}:{}".format(self.keyOrgLog, util.getDateStrFormat(curDate)) data = self.serv_redis_db.zrange(keyname, 0, -1, desc=True, withscores=True) for org in data: orgs.add(org[0].decode('utf8')) return list(orgs) def getOrgContribAndLogin(self, date, org, prev_days=31): - keyname_log = "LOGIN_ORG:{}" - keyname_contrib = "CONTRIB_DAY:{}" + keyname_log = "{}:{}" + keyname_contrib = "{}:{}" data = [] for curDate in util.getXPrevDaysSpan(date, prev_days): - log = self.serv_redis_db.zscore(keyname_log.format(util.getDateStrFormat(curDate)), org) + log = self.serv_redis_db.zscore(keyname_log.format(self.keyOrgLog, util.getDateStrFormat(curDate)), org) log = 0 if log is None else 1 - contrib = self.serv_redis_db.zscore(keyname_contrib.format(util.getDateStrFormat(curDate)), org) + contrib = self.serv_redis_db.zscore(keyname_contrib.format(keyContribDay, util.getDateStrFormat(curDate)), org) contrib = 0 if contrib is None else 1 data.append([log, contrib]) return data @@ -93,7 +100,7 @@ class Users_helper: def getLoginVSCOntribution(self, date): - keyname = "CONTRIB_DAY:{}".format(util.getDateStrFormat(date)) + keyname = "{}:{}".format(keyContribDay, util.getDateStrFormat(date)) orgs_contri = self.serv_redis_db.zrange(keyname, 0, -1, desc=True, withscores=False) orgs_contri = [ org.decode('utf8') for org in orgs_contri ] orgs_login = [ org[0] for org in self.getOrgslogin(date, topNum=0) ] @@ -145,7 +152,7 @@ class Users_helper: for curDate in util.getXPrevDaysSpan(date, prev_days): timestamps = self.getUserLogins(curDate) - keyname = "CONTRIB_DAY:{}".format(util.getDateStrFormat(curDate)) + keyname = "{}:{}".format(keyContribDay, util.getDateStrFormat(curDate)) orgs_contri = self.serv_redis_db.zrange(keyname, 0, -1, desc=True, withscores=False) orgs_contri_num = len(orgs_contri) diff --git a/util.py b/util.py index 5f937e6..50364ba 100644 --- a/util.py +++ b/util.py @@ -9,6 +9,12 @@ def getZrange(serv_redis_db, keyCateg, date, topNum, endSubkey=""): data = [ [record[0].decode('utf8'), record[1]] for record in data ] return data +def push_to_redis_zset(serv_redis_db, mainKey, toAdd, endSubkey="", count=1): + now = datetime.datetime.now() + today_str = util.getDateStrFormat(now) + keyname = "{}:{}{}".format(mainKey, today_str, endSubkey) + serv_redis_db.zincrby(keyname, toAdd, count) + def getMonthSpan(date): ds = datetime.datetime(date.year, date.month, 1) dyear = 1 if ds.month+1 > 12 else 0 diff --git a/zmq_subscriber.py b/zmq_subscriber.py index 6e1514c..598eec7 100755 --- a/zmq_subscriber.py +++ b/zmq_subscriber.py @@ -22,18 +22,8 @@ configfile = os.path.join(os.environ['DASH_CONFIG'], 'config.cfg') cfg = configparser.ConfigParser() cfg.read(configfile) -ONE_DAY = 60*60*24 ZMQ_URL = cfg.get('RedisGlobal', 'zmq_url') CHANNEL = cfg.get('RedisLog', 'channel') -CHANNEL_LASTCONTRIB = cfg.get('RedisLog', 'channelLastContributor') -CHANNEL_LASTAWARDS = cfg.get('RedisLog', 'channelLastAwards') - -DEFAULT_PNTS_REWARD = cfg.get('CONTRIB', 'default_pnts_per_contribution') -categories_in_datatable = json.loads(cfg.get('CONTRIB', 'categories_in_datatable')) -DICO_PNTS_REWARD = {} -temp = json.loads(cfg.get('CONTRIB', 'pnts_per_contribution')) -for categ, pnts in temp: - DICO_PNTS_REWARD[categ] = pnts serv_log = redis.StrictRedis( host=cfg.get('RedisGlobal', 'host'), @@ -54,12 +44,6 @@ def publish_log(zmq_name, name, content, channel=CHANNEL): to_send = { 'name': name, 'log': json.dumps(content), 'zmqName': zmq_name } serv_log.publish(channel, json.dumps(to_send)) -def push_to_redis_zset(keyCateg, toAdd, endSubkey="", count=1): - now = datetime.datetime.now() - today_str = util.getDateStrFormat(now) - keyname = "{}:{}{}".format(keyCateg, today_str, endSubkey) - serv_redis_db.zincrby(keyname, toAdd, count) - def getFields(obj, fields): jsonWalker = fields.split('.') itemToExplore = obj @@ -78,49 +62,6 @@ def getFields(obj, fields): def noSpaceLower(text): return text.lower().replace(' ', '_') -#pntMultiplier if one contribution rewards more than others. (e.g. shighting may gives more points than editing) -def handleContribution(zmq_name, org, contribType, categ, action, pntMultiplier=1, eventTime=datetime.datetime.now(), isLabeled=False): - if action in ['edit', None]: - pass - #return #not a contribution? - - now = datetime.datetime.now() - nowSec = int(time.time()) - pnts_to_add = DEFAULT_PNTS_REWARD - - # if there is a contribution, there is a login (even if ti comes from the API) - users_helper.add_user_login(nowSec, org) - - # is a valid contribution - if categ is not None: - try: - pnts_to_add = DICO_PNTS_REWARD[noSpaceLower(categ)] - except KeyError: - pnts_to_add = DEFAULT_PNTS_REWARD - pnts_to_add *= pntMultiplier - - push_to_redis_zset('CONTRIB_DAY', org, count=pnts_to_add) - #CONTRIB_CATEG retain the contribution per category, not the point earned in this categ - push_to_redis_zset('CONTRIB_CATEG', org, count=1, endSubkey=':'+noSpaceLower(categ)) - publish_log(zmq_name, 'CONTRIBUTION', {'org': org, 'categ': categ, 'action': action, 'epoch': nowSec }, channel=CHANNEL_LASTCONTRIB) - else: - categ = "" - - serv_redis_db.sadd('CONTRIB_ALL_ORG', org) - - serv_redis_db.zadd('CONTRIB_LAST:'+util.getDateStrFormat(now), nowSec, org) - serv_redis_db.expire('CONTRIB_LAST:'+util.getDateStrFormat(now), ONE_DAY*7) #expire after 7 day - - awards_given = contributor_helper.updateOrgContributionRank(org, pnts_to_add, action, contribType, eventTime=datetime.datetime.now(), isLabeled=isLabeled, categ=noSpaceLower(categ)) - - for award in awards_given: - # update awards given - serv_redis_db.zadd('CONTRIB_LAST_AWARDS:'+util.getDateStrFormat(now), nowSec, json.dumps({'org': org, 'award': award, 'epoch': nowSec })) - serv_redis_db.expire('CONTRIB_LAST_AWARDS:'+util.getDateStrFormat(now), ONE_DAY*7) #expire after 7 day - # publish - publish_log(zmq_name, 'CONTRIBUTION', {'org': org, 'award': award, 'epoch': nowSec }, channel=CHANNEL_LASTAWARDS) - - ############## ## HANDLERS ## ############## @@ -159,7 +100,7 @@ def handler_conversation(zmq_name, jsonevent): categ = None action = 'add' eventName = 'no name or id yet...' - handleContribution(zmq_name, org, + contributor_helper.handleContribution(zmq_name, org, 'Discussion', None, action, @@ -177,17 +118,11 @@ def handler_sighting(zmq_name, jsondata): jsonsight = jsondata['Sighting'] org = jsonsight['Event']['Orgc']['name'] categ = jsonsight['Attribute']['category'] - try: - action = jsondata['action'] - except KeyError: - action = None - handleContribution(zmq_name, org, 'Sighting', categ, action, pntMultiplier=2) + action = jsondata.get('action', None) + contributor_helper.handleContribution(zmq_name, org, 'Sighting', categ, action, pntMultiplier=2) handler_attribute(zmq_name, jsonsight, hasAlreadyBeenContributed=True) - try: - timestamp = jsonsight['date_sighting'] - except KeyError: - pass + timestamp = jsonsight.get('date_sighting', None) if jsonsight['type'] == "0": # sightings trendings_helper.addSightings(timestamp) @@ -202,13 +137,12 @@ def handler_event(zmq_name, jsonobj): eventName = jsonevent['info'] timestamp = jsonevent['timestamp'] trendings_helper.addTrendingEvent(eventName, timestamp) - try: - temp = jsonobj['EventTag'] - tags = [] - for tag in temp: + tags = [] + for tag in jsonobj.get('EventTag', []): + try: tags.append(tag['Tag']) - except KeyError: - tags = [] + except KeyError: + pass trendings_helper.addTrendingTags(tags, timestamp) #redirect to handler_attribute @@ -222,21 +156,12 @@ def handler_event(zmq_name, jsonobj): else: handler_attribute(zmq_name, attributes) - try: - action = jsonobj['action'] - except KeyError: - action = None - try: - eventLabeled = len(jsonobj['EventTag']) > 0 - except KeyError: - eventLabeled = False - try: - org = jsonobj['Orgc']['name'] - except KeyError: - org = None + action = jsonobj.get('action', None) + eventLabeled = len(jsonobj.get('EventTag', [])) > 0 + org = jsonobj.get('Orgc', {}).get('name', None) if org is not None: - handleContribution(zmq_name, org, + contributor_helper.handleContribution(zmq_name, org, 'Event', None, action, @@ -249,18 +174,14 @@ def handler_attribute(zmq_name, jsonobj, hasAlreadyBeenContributed=False): #Add trending categName = jsonattr['category'] - try: - timestamp = jsonattr['timestamp'] - except KeyError: - timestamp = int(time.time()) + timestamp = jsonattr.get('timestamp', int(time.time())) trendings_helper.addTrendingCateg(categName, timestamp) - try: - temp = jsonattr['Tag'] - tags = [] - for tag in temp: + tags = [] + for tag in jsonattr.get('Tag', []): + try: tags.append(tag['Tag']) - except KeyError: - tags = [] + except KeyError: + pass trendings_helper.addTrendingTags(tags, timestamp) to_push = [] @@ -283,15 +204,9 @@ def handler_attribute(zmq_name, jsonobj, hasAlreadyBeenContributed=False): geo_helper.getCoordFromPhoneAndPublish(jsonattr['value'], jsonattr['category']) if not hasAlreadyBeenContributed: - try: - eventLabeled = len(jsonattr['Tag']) > 0 - except KeyError: - eventLabeled = False - try: - action = jsonobj['action'] - except KeyError: - action = None - handleContribution(zmq_name, jsonobj['Event']['Orgc']['name'], + eventLabeled = len(jsonobj.get('EventTag', [])) > 0 + action = jsonobj.get('action', None) + contributor_helper.handleContribution(zmq_name, jsonobj['Event']['Orgc']['name'], 'Attribute', jsonattr['category'], action,