diff --git a/contributor_helper.py b/contributor_helper.py new file mode 100644 index 0000000..26c890a --- /dev/null +++ b/contributor_helper.py @@ -0,0 +1,241 @@ +import util +import math, random +import configparser +import json +import datetime + +class Contributor_helper: + def __init__(self, serv_redis_db, cfg): + self.serv_redis_db = serv_redis_db + self.cfg = cfg + + self.MAX_NUMBER_OF_LAST_CONTRIBUTOR = cfg.getint('CONTRIB', 'max_number_of_last_contributor') + self.categories_in_datatable = json.loads(cfg.get('CONTRIB', 'categories_in_datatable')) + self.rankMultiplier = cfg.getfloat('CONTRIB' ,'rankMultiplier') + self.levelMax = 16 + + def getZrange(self, keyCateg, date, topNum, endSubkey=""): + date_str = util.getDateStrFormat(date) + keyname = "{}:{}{}".format(keyCateg, date_str, endSubkey) + data = self.serv_redis_db.zrange(keyname, 0, topNum-1, desc=True, withscores=True) + data = [ [record[0].decode('utf8'), record[1]] for record in data ] + return data + + def getOrgPntFromRedis(self, org, date): + keyCateg = 'CONTRIB_DAY' + scoreSum = 0 + for curDate in util.getMonthSpan(date): + date_str = util.getDateStrFormat(curDate) + keyname = "{}:{}".format(keyCateg, date_str) + data = self.serv_redis_db.zscore(keyname, org) + if data is None: + data = 0 + scoreSum += data + return scoreSum + + def getOrgRankFromRedis(self, org, date): + ptns = self.getOrgPntFromRedis(org, date) + return self.getTrueRank(ptns) + + def getOrgLogoFromRedis(self, org): + return 'logo_'+org + + def getLastContributorFromRedis(self): + date = datetime.datetime.now() + keyCateg = "CONTRIB_LAST" + topNum = self.MAX_NUMBER_OF_LAST_CONTRIBUTOR # default Num + last_contrib_org = self.getZrange(keyCateg, date, topNum) + data = [] + for org, sec in last_contrib_org: + dic = {} + dic['rank'] = self.getOrgRankFromRedis(org, date) + dic['logo_path'] = self.getOrgLogoFromRedis(org) + dic['org'] = org + dic['pnts'] = self.getOrgPntFromRedis(org, date) + data.append(dic) + return data + + def getTopContributorFromRedis(self, date): + orgDicoPnts = {} + for curDate in util.getMonthSpan(date): + keyCateg = "CONTRIB_DAY" + topNum = 0 # all + contrib_org = self.getZrange(keyCateg, curDate, topNum) + for org, pnts in contrib_org: + if org not in orgDicoPnts: + orgDicoPnts[org] = 0 + orgDicoPnts[org] += pnts + + data = [] + for org, pnts in orgDicoPnts.items(): + dic = {} + dic['rank'] = self.getTrueRank(pnts) + dic['logo_path'] = self.getOrgLogoFromRedis(org) + dic['org'] = org + dic['pnts'] = pnts + data.append(dic) + data.sort(key=lambda x: x['pnts'], reverse=True) + + return data + + def getTop5OvertimeFromRedis(self): + data = [] + today = datetime.datetime.now() + topSortedOrg = self.getTopContributorFromRedis(today) #Get current top + # show current top 5 org points overtime (last 5 days) + for dic in topSortedOrg[0:5]: + org = dic['org'] + overtime = [] + for deltaD in range(1,6,1): + date = (datetime.datetime(today.year, today.month, today.day) - datetime.timedelta(days=deltaD)) + keyname = 'CONTRIB_DAY:'+util.getDateStrFormat(date) + org_score = self.serv_redis_db.zscore(keyname, org) + if org_score is None: + org_score = 0 + overtime.append([deltaD, org_score]) + to_append = {'label': org, 'data': overtime} + data.append(to_append) + return data + + def getCategPerContribFromRedis(self, date): + keyCateg = "CONTRIB_DAY" + topNum = 0 # all + contrib_org = self.getZrange(keyCateg, date, topNum) + data = [] + for org, pnts in contrib_org: + dic = {} + dic['rank'] = self.getTrueRank(pnts) + dic['logo_path'] = self.getOrgLogoFromRedis(org) + dic['org'] = org + dic['pnts'] = pnts + for categ in self.categories_in_datatable: + keyname = 'CONTRIB_CATEG:'+util.getDateStrFormat(date)+':'+categ + categ_score = self.serv_redis_db.zscore(keyname, org) + if categ_score is None: + categ_score = 0 + dic[categ] = categ_score + data.append(dic) + return data + + def getAllOrgFromRedis(self): + data = self.serv_redis_db.smembers('CONTRIB_ALL_ORG') + data = [x.decode('utf8') for x in data] + return data + + def getCurrentOrgRankFromRedis(self, org): + date = datetime.datetime.now() + points = self.getOrgPntFromRedis(org, date) + remainingPts = self.getRemainingPoints(points) + data = { + 'org': org, + 'points': points, + 'rank': self.getRankLevel(points), + 'remainingPts': remainingPts['remainingPts'], + 'stepPts': remainingPts['stepPts'], + } + return data + + def getRankLevel(self, points): + if points == 0: + return 0 + elif points == 1: + return 1 + else: + return float("{:.2f}".format(math.log(points, self.rankMultiplier))) + + def getTrueRank(self, ptns): + return int(self.getRankLevel(ptns)) + + def getRemainingPoints(self, points): + prev = 0 + for i in [math.floor(self.rankMultiplier**x) for x in range(1,self.levelMax+1)]: + if prev <= points < i: + return { 'remainingPts': i-points, 'stepPts': prev } + prev = i + return { 'remainingPts': 0, 'stepPts': self.rankMultiplier**self.levelMax } + + + ''' TEST DATA ''' + + def TEST_getCategPerContribFromRedis(self, date): + data2 = [] + for d in range(15): + dic = {} + dic['rank'] = random.randint(1,self.levelMax) + dic['logo_path'] = 'logo' + dic['org'] = 'Org'+str(d) + for f in categories_in_datatable: + dic[f] = random.randint(0,1600) + data2.append(dic) + return data2 + + def TEST_getTop5OvertimeFromRedis(self): + data2 = [ + {'label': 'CIRCL', 'data': [[0, 4], [1, 7], [2,14]]}, + {'label': 'CASES', 'data': [[0, 1], [1, 5], [2,2]]} + ] + return data2 + + def TEST_getTopContributorFromRedis(self, date): + data2 = [ + { + 'rank': random.randint(1,self.levelMax), + 'logo_path': 'logo1', + 'org': 'CIRCL', + }, + { + 'rank': random.randint(1,self.levelMax), + 'logo_path': 'logo2', + 'org': 'CASES', + }, + { + 'rank': random.randint(1,self.levelMax), + 'logo_path': 'logo3', + 'org': 'SMILE', + }, + { + 'rank': random.randint(1,self.levelMax), + 'logo_path': 'logo4', + 'org': 'ORG4', + }, + { + 'rank': random.randint(1,self.levelMax), + 'logo_path': 'logo5', + 'org': 'ORG5', + }, + ] + return data2 + + def TEST_getLastContributorFromRedis(self): + data2 = [ + { + 'rank': random.randint(1,self.levelMax), + 'logo_path': 'logo1', + 'org': 'CIRCL', + }, + { + 'rank': random.randint(1,self.levelMax), + 'logo_path': 'logo2', + 'org': 'CASES', + }, + { + 'rank': random.randint(1,self.levelMax), + 'logo_path': 'logo3', + 'org': 'SMILE', + }, + { + 'rank': random.randint(1,self.levelMax), + 'logo_path': 'logo4', + 'org': 'ORG4', + }, + { + 'rank': random.randint(1,self.levelMax), + 'logo_path': 'logo5', + 'org': 'ORG5', + }, + ] + return data2 + + def TEST_getAllOrgFromRedis(self): + data2 = ['CIRCL', 'CASES', 'SMILE' ,'ORG4' ,'ORG5', 'SUPER HYPER LONG ORGINZATION NAME', 'Org3'] + return data2 diff --git a/server.py b/server.py index eb075f9..fd464fe 100755 --- a/server.py +++ b/server.py @@ -9,6 +9,9 @@ from time import sleep, strftime import datetime import os +import util +import contributor_helper + configfile = os.path.join(os.environ['DASH_CONFIG'], 'config.cfg') cfg = configparser.ConfigParser() cfg.read(configfile) @@ -28,8 +31,7 @@ serv_redis_db = redis.StrictRedis( port=cfg.getint('RedisGlobal', 'port'), db=cfg.getint('RedisDB', 'db')) -categories_in_datatable = json.loads(cfg.get('CONTRIB', 'categories_in_datatable')) -MAX_NUMBER_OF_LAST_CONTRIBUTOR = cfg.getint('CONTRIB', 'max_number_of_last_contributor') +contributor_helper = contributor_helper.Contributor_helper(serv_redis_db, cfg) subscriber_log = redis_server_log.pubsub(ignore_subscribe_messages=True) subscriber_log.psubscribe(cfg.get('RedisLog', 'channel')) @@ -99,123 +101,14 @@ class EventMessage(): to_ret = { 'log': self.feed, 'feedName': self.feedName, 'zmqName': self.zmqName } return 'data: {}\n\n'.format(json.dumps(to_ret)) -''' CONTRIB ''' - -# max lvl is 16 -def getRankLevel(points): - if points == 0: - return 0 - elif points == 1: - return 1 - else: - return float("{:.2f}".format(math.log(points, cfg.getfloat('CONTRIB' ,'rankMultiplier')))) - -def getTrueRank(ptns): - return int(getRankLevel(ptns)) - -def getRemainingPoints(points): - prev = 0 - for i in [math.floor(cfg.getfloat('CONTRIB' ,'rankMultiplier')**x) for x in range(1,17)]: - if prev <= points < i: - return { 'remainingPts': i-points, 'stepPts': prev } - prev = i - return { 'remainingPts': 0, 'stepPts': cfg.getfloat('CONTRIB' ,'rankMultiplier')**16 } - - ''' GENERAL ''' -def getMonthSpan(date): - ds = datetime.datetime(date.year, date.month, 1) - dyear = 1 if ds.month+1 > 12 else 0 - dmonth = -12 if ds.month+1 > 12 else 0 - de = datetime.datetime(ds.year + dyear, ds.month+1 + dmonth, 1) - - delta = de - ds - to_return = [] - for i in range(delta.days): - to_return.append(ds + datetime.timedelta(days=i)) - return to_return - -def getDateStrFormat(date): - return str(date.year)+str(date.month).zfill(2)+str(date.day).zfill(2) - def getZrange(keyCateg, date, topNum, endSubkey=""): - date_str = getDateStrFormat(date) + date_str = util.getDateStrFormat(date) keyname = "{}:{}{}".format(keyCateg, date_str, endSubkey) data = serv_redis_db.zrange(keyname, 0, topNum-1, desc=True, withscores=True) data = [ [record[0].decode('utf8'), record[1]] for record in data ] return data -def getOrgPntFromRedis(org, date): - keyCateg = 'CONTRIB_DAY' - scoreSum = 0 - for curDate in getMonthSpan(date): - date_str = getDateStrFormat(curDate) - keyname = "{}:{}".format(keyCateg, date_str) - data = serv_redis_db.zscore(keyname, org) - if data is None: - data = 0 - scoreSum += data - return scoreSum - -def getOrgRankFromRedis(org, date): - ptns = getOrgPntFromRedis(org, date) - return getTrueRank(ptns) - -def getOrgLogoFromRedis(org): - return 'logo_'+org - -def getTopContributor_fromRedis(date): - data2 = [ - { - 'rank': random.randint(1,16), - 'logo_path': 'logo1', - 'org': 'CIRCL', - }, - { - 'rank': random.randint(1,16), - 'logo_path': 'logo2', - 'org': 'CASES', - }, - { - 'rank': random.randint(1,16), - 'logo_path': 'logo3', - 'org': 'SMILE', - }, - { - 'rank': random.randint(1,16), - 'logo_path': 'logo4', - 'org': 'ORG4', - }, - { - 'rank': random.randint(1,16), - 'logo_path': 'logo5', - 'org': 'ORG5', - }, - ] - - orgDicoPnts = {} - for curDate in getMonthSpan(date): - keyCateg = "CONTRIB_DAY" - topNum = 0 # all - contrib_org = getZrange(keyCateg, curDate, topNum) - for org, pnts in contrib_org: - if org not in orgDicoPnts: - orgDicoPnts[org] = 0 - orgDicoPnts[org] += pnts - - data = [] - for org, pnts in orgDicoPnts.items(): - dic = {} - dic['rank'] = getTrueRank(pnts) - dic['logo_path'] = getOrgLogoFromRedis(org) - dic['org'] = org - dic['pnts'] = pnts - data.append(dic) - data.sort(key=lambda x: x['pnts'], reverse=True) - - return data - #return data2 - ########### ## ROUTE ## ########### @@ -253,14 +146,14 @@ def geo(): @app.route("/contrib") def contrib(): - categ_list = categories_in_datatable - categ_list_str = [ s[0].upper() + s[1:].replace('_', ' ') for s in categories_in_datatable] + categ_list = contributor_helper.categories_in_datatable + categ_list_str = [ s[0].upper() + s[1:].replace('_', ' ') for s in contributor_helper.categories_in_datatable] currOrg = request.args.get('org') if currOrg is None: currOrg = "" return render_template('contrib.html', currOrg=currOrg, - rankMultiplier=cfg.getfloat('CONTRIB' ,'rankMultiplier'), + rankMultiplier=contributor_helper.rankMultiplier, categ_list=json.dumps(categ_list), categ_list_str=categ_list_str ) @@ -337,7 +230,7 @@ def getCoordsByRadius(): delta = dateEnd - dateStart for i in range(delta.days+1): correctDatetime = dateStart + datetime.timedelta(days=i) - date_str = getDateStrFormat(correctDatetime) + date_str = util.getDateStrFormat(correctDatetime) keyCateg = 'GEO_RAD' keyname = "{}:{}".format(keyCateg, date_str) res = serv_redis_db.georadius(keyname, centerLon, centerLat, radius, unit='km', withcoord=True) @@ -369,50 +262,7 @@ def getCoordsByRadius(): @app.route("/_getLastContributor") def getLastContributor(): - data2 = [ - { - 'rank': random.randint(1,16), - 'logo_path': 'logo1', - 'org': 'CIRCL', - }, - { - 'rank': random.randint(1,16), - 'logo_path': 'logo2', - 'org': 'CASES', - }, - { - 'rank': random.randint(1,16), - 'logo_path': 'logo3', - 'org': 'SMILE', - }, - { - 'rank': random.randint(1,16), - 'logo_path': 'logo4', - 'org': 'ORG4', - }, - { - 'rank': random.randint(1,16), - 'logo_path': 'logo5', - 'org': 'ORG5', - }, - ] - try: - date = datetime.datetime.fromtimestamp(float(request.args.get('date'))) - except: - date = datetime.datetime.now() - keyCateg = "CONTRIB_LAST" - topNum = MAX_NUMBER_OF_LAST_CONTRIBUTOR # default Num - last_contrib_org = getZrange(keyCateg, date, topNum) - data = [] - for org, sec in last_contrib_org: - dic = {} - dic['rank'] = getOrgRankFromRedis(org, datetime.datetime.now()) - dic['logo_path'] = getOrgLogoFromRedis(org) - dic['org'] = org - dic['pnts'] = getOrgPntFromRedis(org, date) - data.append(dic) - return jsonify(data) - #return jsonify(data2*2) + return jsonify(contributor_helper.getLastContributorFromRedis()) @app.route("/_getTopContributor") def getTopContributor(suppliedDate=None): @@ -424,7 +274,7 @@ def getTopContributor(suppliedDate=None): else: date = suppliedDate - data = getTopContributor_fromRedis(date) + data = contributor_helper.getTopContributorFromRedis(date) return jsonify(data) @app.route("/_getFameContributor") @@ -435,79 +285,25 @@ def getFameContributor(): today = datetime.datetime.now() # get previous month date = (datetime.datetime(today.year, today.month, 1) - datetime.timedelta(days=1)) - return getTopContributor(date) + return getTopContributor(suppliedDate=date) @app.route("/_getTop5Overtime") def getTop5Overtime(): - data2 = [ - {'label': 'CIRCL', 'data': [[0, 4], [1, 7], [2,14]]}, - {'label': 'CASES', 'data': [[0, 1], [1, 5], [2,2]]} - ] - data = [] - today = datetime.datetime.now() - topSortedOrg = getTopContributor_fromRedis(today) #Get current top - # show current top 5 org points overtime (last 5 days) - for dic in topSortedOrg[0:5]: - org = dic['org'] - overtime = [] - for deltaD in range(1,6,1): - date = (datetime.datetime(today.year, today.month, today.day) - datetime.timedelta(days=deltaD)) - keyname = 'CONTRIB_DAY:'+getDateStrFormat(date) - org_score = serv_redis_db.zscore(keyname, org) - if org_score is None: - org_score = 0 - overtime.append([deltaD, org_score]) - to_append = {'label': org, 'data': overtime} - data.append(to_append) - return jsonify(data) - #return jsonify(data2) + return jsonify(contributor_helper.getTop5OvertimeFromRedis()) @app.route("/_getCategPerContrib") def getCategPerContrib(): - - data2 = [] - for d in range(15): - dic = {} - dic['rank'] = random.randint(1,16) - dic['logo_path'] = 'logo' - dic['org'] = 'Org'+str(d) - for f in categories_in_datatable: - dic[f] = random.randint(0,1600) - data2.append(dic) - try: date = datetime.datetime.fromtimestamp(float(request.args.get('date'))) except: date = datetime.datetime.now() - keyCateg = "CONTRIB_DAY" - topNum = 0 # all - contrib_org = getZrange(keyCateg, date, topNum) - data = [] - for org, pnts in contrib_org: - dic = {} - dic['rank'] = getTrueRank(pnts) - dic['logo_path'] = getOrgLogoFromRedis(org) - dic['org'] = org - dic['pnts'] = pnts - for categ in categories_in_datatable: - keyname = 'CONTRIB_CATEG:'+getDateStrFormat(date)+':'+categ - categ_score = serv_redis_db.zscore(keyname, org) - if categ_score is None: - categ_score = 0 - dic[categ] = categ_score - data.append(dic) - return jsonify(data) - return jsonify(data2) + return jsonify(contributor_helper.getCategPerContribFromRedis(date)) @app.route("/_getAllOrg") def getAllOrg(): - data = serv_redis_db.smembers('CONTRIB_ALL_ORG') - data = [x.decode('utf8') for x in data] - data2 = ['CIRCL', 'CASES', 'SMILE' ,'ORG4' ,'ORG5', 'SUPER HYPER LONG ORGINZATION NAME', 'Org3'] - return jsonify(data) - #return jsonify(data2) + return jsonify(contributor_helper.getAllOrgFromRedis()) @app.route("/_getOrgRank") def getOrgRank(): @@ -515,18 +311,7 @@ def getOrgRank(): org = request.args.get('org') except: org = '' - date = datetime.datetime.now() - points = random.randint(1,math.floor(cfg.getfloat('CONTRIB' ,'rankMultiplier')**16)) - points = getOrgPntFromRedis(org, date) - #FIXME put 0 if org has no points - remainingPts = getRemainingPoints(points) - data = {'org': org, - 'points': points, - 'rank': getRankLevel(points), - 'remainingPts': remainingPts['remainingPts'], - 'stepPts': remainingPts['stepPts'], - } - return jsonify(data) + return jsonify(contributor_helper.getCurrentOrgRankFromRedis(org)) if __name__ == '__main__': app.run(host='localhost', port=8001, threaded=True) diff --git a/util.py b/util.py new file mode 100644 index 0000000..e419597 --- /dev/null +++ b/util.py @@ -0,0 +1,16 @@ +import datetime + +def getMonthSpan(date): + ds = datetime.datetime(date.year, date.month, 1) + dyear = 1 if ds.month+1 > 12 else 0 + dmonth = -12 if ds.month+1 > 12 else 0 + de = datetime.datetime(ds.year + dyear, ds.month+1 + dmonth, 1) + + delta = de - ds + to_return = [] + for i in range(delta.days): + to_return.append(ds + datetime.timedelta(days=i)) + return to_return + +def getDateStrFormat(date): + return str(date.year)+str(date.month).zfill(2)+str(date.day).zfill(2)