import configparser import datetime import json import logging import math import os import random import sys import time import redis import util from util import getZrange from . import users_helper KEYDAY = "CONTRIB_DAY" # To be used by other module KEYALLORG = "CONTRIB_ALL_ORG" # To be used by other module class Contributor_helper: def __init__(self, serv_redis_db, cfg): self.serv_redis_db = serv_redis_db self.serv_log = redis.StrictRedis( host=cfg.get('RedisGlobal', 'host'), port=cfg.getint('RedisGlobal', 'port'), db=cfg.getint('RedisLog', 'db')) self.cfg = cfg self.cfg_org_rank = configparser.ConfigParser() self.cfg_org_rank.read(os.path.join(os.path.dirname(os.path.realpath(__file__)), '../config/ranking.cfg')) self.CHANNEL_LASTAWARDS = cfg.get('RedisLog', 'channelLastAwards') self.CHANNEL_LASTCONTRIB = cfg.get('RedisLog', 'channelLastContributor') self.users_helper = users_helper.Users_helper(serv_redis_db, cfg) #logger logDir = cfg.get('Log', 'directory') logfilename = cfg.get('Log', 'helpers_filename') logPath = os.path.join(logDir, logfilename) if not os.path.exists(logDir): os.makedirs(logDir) try: handler = logging.FileHandler(logPath) except PermissionError as error: print(error) print("Please fix the above and try again.") sys.exit(126) formatter = logging.Formatter('%(asctime)s:%(levelname)s:%(name)s:%(message)s') handler.setFormatter(formatter) self.logger = logging.getLogger(__name__) self.logger.setLevel(logging.INFO) self.logger.addHandler(handler) #honorBadge self.honorBadgeNum = len(self.cfg_org_rank.options('HonorBadge')) self.heavilyCount = self.cfg_org_rank.getint('rankRequirementsMisc', 'heavilyCount') self.recentDays = self.cfg_org_rank.getint('rankRequirementsMisc', 'recentDays') self.regularlyDays = self.cfg_org_rank.getint('rankRequirementsMisc', 'regularlyDays') self.org_honor_badge_title = {} for badgeNum in range(1, self.honorBadgeNum+1): #get Num of honorBadge self.org_honor_badge_title[badgeNum] = self.cfg_org_rank.get('HonorBadge', str(badgeNum)) self.trophyMapping = json.loads(self.cfg_org_rank.get('TrophyDifficulty', 'trophyMapping')) self.trophyMappingIncremental = [sum(self.trophyMapping[:i]) for i in range(len(self.trophyMapping)+1)] self.trophyNum = len(self.cfg_org_rank.options('HonorTrophy'))-1 #0 is not a trophy self.categories_in_trophy = json.loads(self.cfg_org_rank.get('HonorTrophyCateg', 'categ')) self.trophy_title = {} for trophyNum in range(0, len(self.cfg_org_rank.options('HonorTrophy'))): #get Num of trophy self.trophy_title[trophyNum] = self.cfg_org_rank.get('HonorTrophy', str(trophyNum)) #GLOBAL RANKING self.org_rank_maxLevel = self.cfg_org_rank.getint('rankTitle', 'maxLevel') self.org_rank = {} for rank in range(1, self.org_rank_maxLevel+1): self.org_rank[rank] = self.cfg_org_rank.get('rankTitle', str(rank)) self.org_rank_requirement_pnts = {} for rank in range(1, self.org_rank_maxLevel+1): self.org_rank_requirement_pnts[rank] = self.cfg_org_rank.getint('rankRequirementsPnts', str(rank)) self.org_rank_requirement_text = {} for rank in range(1, self.org_rank_maxLevel+1): self.org_rank_requirement_text[rank] = self.cfg_org_rank.get('rankRequirementsText', str(rank)) self.org_rank_additional_info = json.loads(self.cfg_org_rank.get('additionalInfo', 'textsArray')) #WEB STUFF self.misp_web_url = cfg.get('RedisGlobal', 'misp_web_url') self.MAX_NUMBER_OF_LAST_CONTRIBUTOR = cfg.getint('CONTRIB', 'max_number_of_last_contributor') self.categories_in_datatable = json.loads(self.cfg_org_rank.get('monthlyRanking', 'categories_in_datatable')) #MONTHLY RANKING self.default_pnts_per_contribution = json.loads(self.cfg_org_rank.get('monthlyRanking', 'default_pnts_per_contribution')) temp = json.loads(self.cfg_org_rank.get('monthlyRanking', 'pnts_per_contribution')) self.DICO_PNTS_REWARD = {} for categ, pnts in temp: self.DICO_PNTS_REWARD[categ] = pnts # fill other categ with default points for categ in self.categories_in_datatable: if categ in self.DICO_PNTS_REWARD: continue else: self.DICO_PNTS_REWARD[categ] = self.default_pnts_per_contribution self.rankMultiplier = self.cfg_org_rank.getfloat('monthlyRanking' ,'rankMultiplier') self.levelMax = self.cfg_org_rank.getint('monthlyRanking', 'levelMax') # REDIS KEYS self.keyDay = KEYDAY self.keyCateg = "CONTRIB_CATEG" self.keyLastContrib = "CONTRIB_LAST" self.keyAllOrg = KEYALLORG self.keyContribReq = "CONTRIB_ORG" self.keyTrophy = "CONTRIB_TROPHY" self.keyLastAward = "CONTRIB_LAST_AWARDS" ''' HELPER ''' def getOrgLogoFromMISP(self, org): return "{}/img/orgs/{}.png".format(self.misp_web_url, org) def addContributionToCateg(self, date, categ, org, count=1): today_str = util.getDateStrFormat(date) keyname = "{}:{}:{}".format(self.keyCateg, today_str, categ) self.serv_redis_db.zincrby(keyname, count, org) self.logger.debug('Added to redis: keyname={}, org={}, count={}'.format(keyname, org, count)) def publish_log(self, zmq_name, name, content, channel=""): to_send = {'name': name, 'log': json.dumps(content), 'zmqName': zmq_name } self.serv_log.publish(channel, json.dumps(to_send)) self.logger.debug('Published: {}'.format(json.dumps(to_send))) ''' HANDLER ''' #pntMultiplier if one contribution rewards more than others. (e.g. shighting may gives more points than editing) def handleContribution(self, zmq_name, org, contribType, categ, action, pntMultiplier=1, eventTime=datetime.datetime.now(), isLabeled=False): if action in ['edit', None]: pass #return #not a contribution? now = datetime.datetime.now() nowSec = int(time.time()) pnts_to_add = self.default_pnts_per_contribution # Do not consider contribution as login anymore #self.users_helper.add_user_login(nowSec, org) # is a valid contribution if categ is not None: try: pnts_to_add = self.DICO_PNTS_REWARD[util.noSpaceLower(categ)] except KeyError: pnts_to_add = self.default_pnts_per_contribution pnts_to_add *= pntMultiplier util.push_to_redis_zset(self.serv_redis_db, self.keyDay, org, count=pnts_to_add) #CONTRIB_CATEG retain the contribution per category, not the point earned in this categ util.push_to_redis_zset(self.serv_redis_db, self.keyCateg, org, count=1, endSubkey=':'+util.noSpaceLower(categ)) self.publish_log(zmq_name, 'CONTRIBUTION', {'org': org, 'categ': categ, 'action': action, 'epoch': nowSec }, channel=self.CHANNEL_LASTCONTRIB) else: categ = "" self.serv_redis_db.sadd(self.keyAllOrg, org) keyname = "{}:{}".format(self.keyLastContrib, util.getDateStrFormat(now)) self.serv_redis_db.zadd(keyname, {org: nowSec}) self.logger.debug('Added to redis: keyname={}, nowSec={}, org={}'.format(keyname, nowSec, org)) self.serv_redis_db.expire(keyname, util.ONE_DAY*7) #expire after 7 day awards_given = self.updateOrgContributionRank(org, pnts_to_add, action, contribType, eventTime=datetime.datetime.now(), isLabeled=isLabeled, categ=util.noSpaceLower(categ)) for award in awards_given: # update awards given keyname = "{}:{}".format(self.keyLastAward, util.getDateStrFormat(now)) self.serv_redis_db.zadd(keyname, {json.dumps({'org': org, 'award': award, 'epoch': nowSec }): nowSec}) self.logger.debug('Added to redis: keyname={}, nowSec={}, content={}'.format(keyname, nowSec, json.dumps({'org': org, 'award': award, 'epoch': nowSec }))) self.serv_redis_db.expire(keyname, util.ONE_DAY*7) #expire after 7 day # publish self.publish_log(zmq_name, 'CONTRIBUTION', {'org': org, 'award': award, 'epoch': nowSec }, channel=self.CHANNEL_LASTAWARDS) ''' CONTRIBUTION RANK ''' def getOrgContributionTotalPoints(self, org): keyname = '{mainKey}:{org}:{orgCateg}' pnts = self.serv_redis_db.get(keyname.format(mainKey=self.keyContribReq, org=org, orgCateg='points')) if pnts is None: pnts = 0 else: pnts = int(pnts) return pnts # return: [final_rank, requirement_fulfilled, requirement_not_fulfilled] def getOrgContributionRank(self, org): keyname = '{mainKey}:{org}:{orgCateg}' final_rank = 0 requirement_fulfilled = [] requirement_not_fulfilled = [] for i in range(1, self.org_rank_maxLevel+1): key = keyname.format(mainKey=self.keyContribReq, org=org, orgCateg='CONTRIB_REQ_'+str(i)) if self.serv_redis_db.get(key) is None: #non existing requirement_not_fulfilled.append(i) else: requirement_fulfilled.append(i) final_rank += 1 #num_of_previous_req_not_fulfilled = len([x for x in requirement_not_fulfilled if x recentDays #update contribution Requirement contrib = [] #[[contrib_level, contrib_ttl], [], ...] if totOrgPnts >= self.org_rank_requirement_pnts[1] and contribType == 'Sighting': #[contrib_level, contrib_ttl] contrib.append([1, util.ONE_DAY*365]) if totOrgPnts >= self.org_rank_requirement_pnts[2] and contribType == 'Attribute' or contribType == 'Object': contrib.append([2, util.ONE_DAY*365]) if totOrgPnts >= self.org_rank_requirement_pnts[3] and contribType == 'Proposal' or contribType == 'Discussion': contrib.append([3, util.ONE_DAY*365]) if totOrgPnts >= self.org_rank_requirement_pnts[4] and contribType == 'Sighting' and isRecent: contrib.append([4, util.ONE_DAY*recentDays]) if totOrgPnts >= self.org_rank_requirement_pnts[5] and contribType == 'Proposal' and isRecent: contrib.append([5, util.ONE_DAY*recentDays]) if totOrgPnts >= self.org_rank_requirement_pnts[6] and contribType == 'Event': contrib.append([6, util.ONE_DAY*365]) if totOrgPnts >= self.org_rank_requirement_pnts[7] and contribType == 'Event' and eventMonthCount>=1: contrib.append([7, util.ONE_DAY*recentDays]) if totOrgPnts >= self.org_rank_requirement_pnts[8] and contribType == 'Event' and eventWeekCount>=1: contrib.append([8, util.ONE_DAY*regularlyDays]) if totOrgPnts >= self.org_rank_requirement_pnts[9] and contribType == 'Event' and isLabeled: contrib.append([9, util.ONE_DAY*regularlyDays]) if totOrgPnts >= self.org_rank_requirement_pnts[10] and contribType == 'Sighting' and sightingWeekCount>heavilyCount: contrib.append([10, util.ONE_DAY*regularlyDays]) if totOrgPnts >= self.org_rank_requirement_pnts[11] and (contribType == 'Attribute' or contribType == 'Object') and attributeWeekCount>heavilyCount: contrib.append([11, util.ONE_DAY*regularlyDays]) if totOrgPnts >= self.org_rank_requirement_pnts[12] and contribType == 'Proposal' and proposalWeekCount>heavilyCount: contrib.append([12, util.ONE_DAY*regularlyDays]) if totOrgPnts >= self.org_rank_requirement_pnts[13] and contribType == 'Event' and eventWeekCount>heavilyCount: contrib.append([13, util.ONE_DAY*regularlyDays]) if totOrgPnts >= self.org_rank_requirement_pnts[14] and contribType == 'Event' and eventWeekCount>heavilyCount and isLabeled: contrib.append([14, util.ONE_DAY*regularlyDays]) for rankReq, ttl in contrib: self.serv_redis_db.set(keyname.format(org=orgName, orgCateg='CONTRIB_REQ_'+str(rankReq)), 1) self.logger.debug('Set: keyname={}'.format(keyname.format(org=orgName, orgCateg='CONTRIB_REQ_'+str(rankReq)))) self.serv_redis_db.expire(keyname.format(org=orgName, orgCateg='CONTRIB_REQ_'+str(rankReq)), ttl) ContributionStatus = self.getCurrentContributionStatus(orgName) newContributionStatus = ContributionStatus['status'] newHonorBadges = self.getOrgHonorBadges(orgName) newTrophy = self.getOrgTrophies(orgName) # awards to publish awards_given = [] for i in newContributionStatus.keys(): if oldContributionStatus[i] < newContributionStatus[i] and i != ContributionStatus['rank']: awards_given.append(['contribution_status', i]) for badgeNum in newHonorBadges: if badgeNum not in oldHonorBadges: awards_given.append(['badge', badgeNum]) temp = {} for item in oldTrophy: categ = item['categ'] rank = item['trophy_true_rank'] temp[categ] = rank for item in newTrophy: categ = item['categ'] rank = item['trophy_true_rank'] try: oldCategRank = temp[categ] except KeyError: oldCategRank = 0 if rank > oldCategRank: awards_given.append(['trophy', [categ, rank]]) return awards_given ''' HONOR BADGES ''' def getOrgHonorBadges(self, org): keyname = '{mainKey}:{org}:{orgCateg}' honorBadge = [] for i in range(1, self.honorBadgeNum+1): key = keyname.format(mainKey=self.keyContribReq, org=org, orgCateg='BADGE_'+str(i)) if self.serv_redis_db.get(key) is not None: #existing honorBadge.append(i) return honorBadge def giveBadgeToOrg(self, org, badgeNum): keyname = '{mainKey}:{org}:{orgCateg}' self.serv_redis_db.set(keyname.format(mainKey=self.keyContribReq, org=org, orgCateg='BADGE_'+str(badgeNum)), 1) self.logger.debug('Giving badge {} to org {}'.format(org, badgeNum)) def removeBadgeFromOrg(self, org, badgeNum): keyname = '{mainKey}:{org}:{orgCateg}' self.serv_redis_db.delete(keyname.format(mainKey=self.keyContribReq, org=org, orgCateg='BADGE_'+str(badgeNum))) self.logger.debug('Removing badge {} from org {}'.format(org, badgeNum)) ''' TROPHIES ''' def getOrgTrophies(self, org): keyname = '{mainKey}:{orgCateg}' trophy = [] for categ in self.categories_in_trophy: key = keyname.format(mainKey=self.keyTrophy, orgCateg=categ) totNum = self.serv_redis_db.zcard(key) if totNum == 0: continue pos = self.serv_redis_db.zrevrank(key, org) if pos is None: continue trophy_rank = self.posToRankMapping(pos, totNum) trophy_Pnts = self.serv_redis_db.zscore(key, org) trophy.append({ 'categ': categ, 'trophy_points': trophy_Pnts, 'trophy_rank': trophy_rank, 'trophy_true_rank': self.trophyNum-trophy_rank, 'trophy_title': self.trophy_title[trophy_rank]}) return trophy def getOrgsTrophyRanking(self, categ): keyname = '{mainKey}:{orgCateg}' res = self.serv_redis_db.zrange(keyname.format(mainKey=self.keyTrophy, orgCateg=categ), 0, -1, withscores=True, desc=True) res = [[org, score] for org, score in res] return res def getAllOrgsTrophyRanking(self, category=None): concerned_categ = self.categories_in_trophy if category is None else category dico_categ = {} for categ in [concerned_categ]: res = self.getOrgsTrophyRanking(categ) # add ranking info tot = len(res) for pos in range(tot): res[pos].append(self.trophyNum-self.posToRankMapping(pos, tot)) dico_categ[categ] = res toret = dico_categ if category is None else dico_categ.get(category, []) return toret def posToRankMapping(self, pos, totNum): ratio = pos/totNum*100 rank = 0 if pos == totNum: return 0 else: for i in range(len(self.trophyMappingIncremental)): if self.trophyMappingIncremental[i] < ratio <= self.trophyMappingIncremental[i+1]: rank = i+1 return rank def giveTrophyPointsToOrg(self, org, categ, points): keyname = '{mainKey}:{orgCateg}' self.serv_redis_db.zincrby(keyname.format(mainKey=self.keyTrophy, orgCateg=categ), points, org) self.logger.debug('Giving {} trophy points to {} in {}'.format(points, org, categ)) def removeTrophyPointsFromOrg(self, org, categ, points): keyname = '{mainKey}:{orgCateg}' self.serv_redis_db.zincrby(keyname.format(mainKey=self.keyTrophy, orgCateg=categ), -points, org) self.logger.debug('Removing {} trophy points from {} in {}'.format(points, org, categ)) ''' AWARDS HELPER ''' def getLastAwardsFromRedis(self): date = datetime.datetime.now() keyname = self.keyLastAward prev_days = 7 topNum = self.MAX_NUMBER_OF_LAST_CONTRIBUTOR # default Num addedOrg = [] data = [] for curDate in util.getXPrevDaysSpan(date, prev_days): last_awards = getZrange(self.serv_redis_db, keyname, curDate, topNum) for dico_award, sec in last_awards: dico_award = json.loads(dico_award) org = dico_award['org'] dic = {} dic['orgRank'] = self.getOrgContributionRank(org)['final_rank'] dic['logo_path'] = self.getOrgLogoFromMISP(org) dic['org'] = org dic['epoch'] = sec dic['award'] = dico_award['award'] data.append(dic) return data ''' MONTHLY CONTRIBUTION ''' def getOrgPntFromRedis(self, org, date): scoreSum = 0 for curDate in util.getMonthSpan(date): date_str = util.getDateStrFormat(curDate) keyname = "{}:{}".format(self.keyDay, date_str) data = self.serv_redis_db.zscore(keyname, org) if data is None: data = 0 scoreSum += data return scoreSum def getOrgRankFromRedis(self, org, date): ptns = self.getOrgPntFromRedis(org, date) return self.getTrueRank(ptns) def getLastContributorsFromRedis(self): date = datetime.datetime.now() prev_days = 7 topNum = self.MAX_NUMBER_OF_LAST_CONTRIBUTOR # default Num addedOrg = [] data = [] for curDate in util.getXPrevDaysSpan(date, prev_days): last_contrib_org = getZrange(self.serv_redis_db, self.keyLastContrib, curDate, topNum) for org, sec in last_contrib_org: if org in addedOrg: continue dic = {} dic['rank'] = self.getOrgRankFromRedis(org, date) dic['orgRank'] = self.getOrgContributionRank(org)['final_rank'] dic['honorBadge'] = self.getOrgHonorBadges(org) dic['logo_path'] = self.getOrgLogoFromMISP(org) dic['org'] = org dic['pnts'] = self.getOrgPntFromRedis(org, date) dic['epoch'] = sec data.append(dic) addedOrg.append(org) return data def getContributorFromRedis(self, org): date = datetime.datetime.now() epoch = self.serv_redis_db.zscore(self.keyLastContrib, org) dic = {} dic['rank'] = self.getOrgRankFromRedis(org, date) dic['orgRank'] = self.getOrgContributionRank(org)['final_rank'] dic['honorBadge'] = self.getOrgHonorBadges(org) dic['logo_path'] = self.getOrgLogoFromMISP(org) dic['org'] = org dic['pnts'] = self.getOrgPntFromRedis(org, date) dic['epoch'] = epoch return dic def getTopContributorFromRedis(self, date, maxNum=100): orgDicoPnts = {} for curDate in util.getMonthSpan(date): topNum = 0 # all contrib_org = getZrange(self.serv_redis_db, self.keyDay, curDate, topNum) for org, pnts in contrib_org: if org not in orgDicoPnts: orgDicoPnts[org] = 0 orgDicoPnts[org] += pnts data = [] for org, pnts in orgDicoPnts.items(): dic = {} dic['rank'] = self.getTrueRank(pnts) dic['orgRank'] = self.getOrgContributionRank(org)['final_rank'] dic['honorBadge'] = self.getOrgHonorBadges(org) dic['logo_path'] = self.getOrgLogoFromMISP(org) dic['org'] = org dic['pnts'] = pnts data.append(dic) data.sort(key=lambda x: x['pnts'], reverse=True) return data[:maxNum] def getTop5OvertimeFromRedis(self): data = [] today = datetime.datetime.now() topSortedOrg = self.getTopContributorFromRedis(today) #Get current top # show current top 5 org points overtime (last 5 days) for dic in topSortedOrg[0:5]: org = dic['org'] to_append = self.getOrgOvertime(org) data.append(to_append) return data def getOrgOvertime(self, org): overtime = [] today = datetime.datetime.today() today = today.replace(hour=0, minute=0, second=0, microsecond=0) for curDate in util.getXPrevDaysSpan(today, 7): timestamp = util.getTimestamp(curDate) keyname = "{}:{}".format(self.keyDay, util.getDateStrFormat(curDate)) org_score = self.serv_redis_db.zscore(keyname, org) if org_score is None: org_score = 0 overtime.append([timestamp, org_score]) to_return = {'label': org, 'data': overtime} return to_return def getCategPerContribFromRedis(self, date): topNum = 0 # all contrib_org = self.getTopContributorFromRedis(date) for dic in contrib_org: org = dic['org'] for categ in self.categories_in_datatable: categ_score = 0 for curDate in util.getMonthSpan(date): keyname = "{}:{}:{}".format(self.keyCateg, util.getDateStrFormat(curDate), categ) temp = self.serv_redis_db.zscore(keyname, org) if temp is None: temp = 0 categ_score += temp dic[categ] = categ_score return contrib_org def getAllOrgFromRedis(self): data = self.serv_redis_db.smembers(self.keyAllOrg) data = [x for x in data] return data def getCurrentOrgRankFromRedis(self, org): date = datetime.datetime.now() points = self.getOrgPntFromRedis(org, date) remainingPts = self.getRemainingPoints(points) data = { 'org': org, 'points': points, 'rank': self.getRankLevel(points), 'orgRank': self.getOrgContributionRank(org)['final_rank'], 'honorBadge': self.getOrgHonorBadges(org), 'remainingPts': remainingPts['remainingPts'], 'stepPts': remainingPts['stepPts'], } return data def getRankLevel(self, points): if points == 0: return 0 elif points == 1: return 1 else: return float("{:.2f}".format(math.log(points, self.rankMultiplier))) def getTrueRank(self, ptns): return int(self.getRankLevel(ptns)) def getRemainingPoints(self, points): prev = 0 for i in [math.floor(self.rankMultiplier**x) for x in range(1,self.levelMax+1)]: if prev <= points < i: return { 'remainingPts': i-points, 'stepPts': prev } prev = i return { 'remainingPts': 0, 'stepPts': self.rankMultiplier**self.levelMax }