Separeted contributor functions from server script

pull/3/head
Sami Mokaddem 2017-11-06 10:42:51 +01:00
parent 85d7d10df5
commit 29e08d8592
3 changed files with 273 additions and 231 deletions

241
contributor_helper.py Normal file
View File

@ -0,0 +1,241 @@
import util
import math, random
import configparser
import json
import datetime
class Contributor_helper:
def __init__(self, serv_redis_db, cfg):
self.serv_redis_db = serv_redis_db
self.cfg = cfg
self.MAX_NUMBER_OF_LAST_CONTRIBUTOR = cfg.getint('CONTRIB', 'max_number_of_last_contributor')
self.categories_in_datatable = json.loads(cfg.get('CONTRIB', 'categories_in_datatable'))
self.rankMultiplier = cfg.getfloat('CONTRIB' ,'rankMultiplier')
self.levelMax = 16
def getZrange(self, keyCateg, date, topNum, endSubkey=""):
date_str = util.getDateStrFormat(date)
keyname = "{}:{}{}".format(keyCateg, date_str, endSubkey)
data = self.serv_redis_db.zrange(keyname, 0, topNum-1, desc=True, withscores=True)
data = [ [record[0].decode('utf8'), record[1]] for record in data ]
return data
def getOrgPntFromRedis(self, org, date):
keyCateg = 'CONTRIB_DAY'
scoreSum = 0
for curDate in util.getMonthSpan(date):
date_str = util.getDateStrFormat(curDate)
keyname = "{}:{}".format(keyCateg, date_str)
data = self.serv_redis_db.zscore(keyname, org)
if data is None:
data = 0
scoreSum += data
return scoreSum
def getOrgRankFromRedis(self, org, date):
ptns = self.getOrgPntFromRedis(org, date)
return self.getTrueRank(ptns)
def getOrgLogoFromRedis(self, org):
return 'logo_'+org
def getLastContributorFromRedis(self):
date = datetime.datetime.now()
keyCateg = "CONTRIB_LAST"
topNum = self.MAX_NUMBER_OF_LAST_CONTRIBUTOR # default Num
last_contrib_org = self.getZrange(keyCateg, date, topNum)
data = []
for org, sec in last_contrib_org:
dic = {}
dic['rank'] = self.getOrgRankFromRedis(org, date)
dic['logo_path'] = self.getOrgLogoFromRedis(org)
dic['org'] = org
dic['pnts'] = self.getOrgPntFromRedis(org, date)
data.append(dic)
return data
def getTopContributorFromRedis(self, date):
orgDicoPnts = {}
for curDate in util.getMonthSpan(date):
keyCateg = "CONTRIB_DAY"
topNum = 0 # all
contrib_org = self.getZrange(keyCateg, curDate, topNum)
for org, pnts in contrib_org:
if org not in orgDicoPnts:
orgDicoPnts[org] = 0
orgDicoPnts[org] += pnts
data = []
for org, pnts in orgDicoPnts.items():
dic = {}
dic['rank'] = self.getTrueRank(pnts)
dic['logo_path'] = self.getOrgLogoFromRedis(org)
dic['org'] = org
dic['pnts'] = pnts
data.append(dic)
data.sort(key=lambda x: x['pnts'], reverse=True)
return data
def getTop5OvertimeFromRedis(self):
data = []
today = datetime.datetime.now()
topSortedOrg = self.getTopContributorFromRedis(today) #Get current top
# show current top 5 org points overtime (last 5 days)
for dic in topSortedOrg[0:5]:
org = dic['org']
overtime = []
for deltaD in range(1,6,1):
date = (datetime.datetime(today.year, today.month, today.day) - datetime.timedelta(days=deltaD))
keyname = 'CONTRIB_DAY:'+util.getDateStrFormat(date)
org_score = self.serv_redis_db.zscore(keyname, org)
if org_score is None:
org_score = 0
overtime.append([deltaD, org_score])
to_append = {'label': org, 'data': overtime}
data.append(to_append)
return data
def getCategPerContribFromRedis(self, date):
keyCateg = "CONTRIB_DAY"
topNum = 0 # all
contrib_org = self.getZrange(keyCateg, date, topNum)
data = []
for org, pnts in contrib_org:
dic = {}
dic['rank'] = self.getTrueRank(pnts)
dic['logo_path'] = self.getOrgLogoFromRedis(org)
dic['org'] = org
dic['pnts'] = pnts
for categ in self.categories_in_datatable:
keyname = 'CONTRIB_CATEG:'+util.getDateStrFormat(date)+':'+categ
categ_score = self.serv_redis_db.zscore(keyname, org)
if categ_score is None:
categ_score = 0
dic[categ] = categ_score
data.append(dic)
return data
def getAllOrgFromRedis(self):
data = self.serv_redis_db.smembers('CONTRIB_ALL_ORG')
data = [x.decode('utf8') for x in data]
return data
def getCurrentOrgRankFromRedis(self, org):
date = datetime.datetime.now()
points = self.getOrgPntFromRedis(org, date)
remainingPts = self.getRemainingPoints(points)
data = {
'org': org,
'points': points,
'rank': self.getRankLevel(points),
'remainingPts': remainingPts['remainingPts'],
'stepPts': remainingPts['stepPts'],
}
return data
def getRankLevel(self, points):
if points == 0:
return 0
elif points == 1:
return 1
else:
return float("{:.2f}".format(math.log(points, self.rankMultiplier)))
def getTrueRank(self, ptns):
return int(self.getRankLevel(ptns))
def getRemainingPoints(self, points):
prev = 0
for i in [math.floor(self.rankMultiplier**x) for x in range(1,self.levelMax+1)]:
if prev <= points < i:
return { 'remainingPts': i-points, 'stepPts': prev }
prev = i
return { 'remainingPts': 0, 'stepPts': self.rankMultiplier**self.levelMax }
''' TEST DATA '''
def TEST_getCategPerContribFromRedis(self, date):
data2 = []
for d in range(15):
dic = {}
dic['rank'] = random.randint(1,self.levelMax)
dic['logo_path'] = 'logo'
dic['org'] = 'Org'+str(d)
for f in categories_in_datatable:
dic[f] = random.randint(0,1600)
data2.append(dic)
return data2
def TEST_getTop5OvertimeFromRedis(self):
data2 = [
{'label': 'CIRCL', 'data': [[0, 4], [1, 7], [2,14]]},
{'label': 'CASES', 'data': [[0, 1], [1, 5], [2,2]]}
]
return data2
def TEST_getTopContributorFromRedis(self, date):
data2 = [
{
'rank': random.randint(1,self.levelMax),
'logo_path': 'logo1',
'org': 'CIRCL',
},
{
'rank': random.randint(1,self.levelMax),
'logo_path': 'logo2',
'org': 'CASES',
},
{
'rank': random.randint(1,self.levelMax),
'logo_path': 'logo3',
'org': 'SMILE',
},
{
'rank': random.randint(1,self.levelMax),
'logo_path': 'logo4',
'org': 'ORG4',
},
{
'rank': random.randint(1,self.levelMax),
'logo_path': 'logo5',
'org': 'ORG5',
},
]
return data2
def TEST_getLastContributorFromRedis(self):
data2 = [
{
'rank': random.randint(1,self.levelMax),
'logo_path': 'logo1',
'org': 'CIRCL',
},
{
'rank': random.randint(1,self.levelMax),
'logo_path': 'logo2',
'org': 'CASES',
},
{
'rank': random.randint(1,self.levelMax),
'logo_path': 'logo3',
'org': 'SMILE',
},
{
'rank': random.randint(1,self.levelMax),
'logo_path': 'logo4',
'org': 'ORG4',
},
{
'rank': random.randint(1,self.levelMax),
'logo_path': 'logo5',
'org': 'ORG5',
},
]
return data2
def TEST_getAllOrgFromRedis(self):
data2 = ['CIRCL', 'CASES', 'SMILE' ,'ORG4' ,'ORG5', 'SUPER HYPER LONG ORGINZATION NAME', 'Org3']
return data2

247
server.py
View File

@ -9,6 +9,9 @@ from time import sleep, strftime
import datetime
import os
import util
import contributor_helper
configfile = os.path.join(os.environ['DASH_CONFIG'], 'config.cfg')
cfg = configparser.ConfigParser()
cfg.read(configfile)
@ -28,8 +31,7 @@ serv_redis_db = redis.StrictRedis(
port=cfg.getint('RedisGlobal', 'port'),
db=cfg.getint('RedisDB', 'db'))
categories_in_datatable = json.loads(cfg.get('CONTRIB', 'categories_in_datatable'))
MAX_NUMBER_OF_LAST_CONTRIBUTOR = cfg.getint('CONTRIB', 'max_number_of_last_contributor')
contributor_helper = contributor_helper.Contributor_helper(serv_redis_db, cfg)
subscriber_log = redis_server_log.pubsub(ignore_subscribe_messages=True)
subscriber_log.psubscribe(cfg.get('RedisLog', 'channel'))
@ -99,123 +101,14 @@ class EventMessage():
to_ret = { 'log': self.feed, 'feedName': self.feedName, 'zmqName': self.zmqName }
return 'data: {}\n\n'.format(json.dumps(to_ret))
''' CONTRIB '''
# max lvl is 16
def getRankLevel(points):
if points == 0:
return 0
elif points == 1:
return 1
else:
return float("{:.2f}".format(math.log(points, cfg.getfloat('CONTRIB' ,'rankMultiplier'))))
def getTrueRank(ptns):
return int(getRankLevel(ptns))
def getRemainingPoints(points):
prev = 0
for i in [math.floor(cfg.getfloat('CONTRIB' ,'rankMultiplier')**x) for x in range(1,17)]:
if prev <= points < i:
return { 'remainingPts': i-points, 'stepPts': prev }
prev = i
return { 'remainingPts': 0, 'stepPts': cfg.getfloat('CONTRIB' ,'rankMultiplier')**16 }
''' GENERAL '''
def getMonthSpan(date):
ds = datetime.datetime(date.year, date.month, 1)
dyear = 1 if ds.month+1 > 12 else 0
dmonth = -12 if ds.month+1 > 12 else 0
de = datetime.datetime(ds.year + dyear, ds.month+1 + dmonth, 1)
delta = de - ds
to_return = []
for i in range(delta.days):
to_return.append(ds + datetime.timedelta(days=i))
return to_return
def getDateStrFormat(date):
return str(date.year)+str(date.month).zfill(2)+str(date.day).zfill(2)
def getZrange(keyCateg, date, topNum, endSubkey=""):
date_str = getDateStrFormat(date)
date_str = util.getDateStrFormat(date)
keyname = "{}:{}{}".format(keyCateg, date_str, endSubkey)
data = serv_redis_db.zrange(keyname, 0, topNum-1, desc=True, withscores=True)
data = [ [record[0].decode('utf8'), record[1]] for record in data ]
return data
def getOrgPntFromRedis(org, date):
keyCateg = 'CONTRIB_DAY'
scoreSum = 0
for curDate in getMonthSpan(date):
date_str = getDateStrFormat(curDate)
keyname = "{}:{}".format(keyCateg, date_str)
data = serv_redis_db.zscore(keyname, org)
if data is None:
data = 0
scoreSum += data
return scoreSum
def getOrgRankFromRedis(org, date):
ptns = getOrgPntFromRedis(org, date)
return getTrueRank(ptns)
def getOrgLogoFromRedis(org):
return 'logo_'+org
def getTopContributor_fromRedis(date):
data2 = [
{
'rank': random.randint(1,16),
'logo_path': 'logo1',
'org': 'CIRCL',
},
{
'rank': random.randint(1,16),
'logo_path': 'logo2',
'org': 'CASES',
},
{
'rank': random.randint(1,16),
'logo_path': 'logo3',
'org': 'SMILE',
},
{
'rank': random.randint(1,16),
'logo_path': 'logo4',
'org': 'ORG4',
},
{
'rank': random.randint(1,16),
'logo_path': 'logo5',
'org': 'ORG5',
},
]
orgDicoPnts = {}
for curDate in getMonthSpan(date):
keyCateg = "CONTRIB_DAY"
topNum = 0 # all
contrib_org = getZrange(keyCateg, curDate, topNum)
for org, pnts in contrib_org:
if org not in orgDicoPnts:
orgDicoPnts[org] = 0
orgDicoPnts[org] += pnts
data = []
for org, pnts in orgDicoPnts.items():
dic = {}
dic['rank'] = getTrueRank(pnts)
dic['logo_path'] = getOrgLogoFromRedis(org)
dic['org'] = org
dic['pnts'] = pnts
data.append(dic)
data.sort(key=lambda x: x['pnts'], reverse=True)
return data
#return data2
###########
## ROUTE ##
###########
@ -253,14 +146,14 @@ def geo():
@app.route("/contrib")
def contrib():
categ_list = categories_in_datatable
categ_list_str = [ s[0].upper() + s[1:].replace('_', ' ') for s in categories_in_datatable]
categ_list = contributor_helper.categories_in_datatable
categ_list_str = [ s[0].upper() + s[1:].replace('_', ' ') for s in contributor_helper.categories_in_datatable]
currOrg = request.args.get('org')
if currOrg is None:
currOrg = ""
return render_template('contrib.html',
currOrg=currOrg,
rankMultiplier=cfg.getfloat('CONTRIB' ,'rankMultiplier'),
rankMultiplier=contributor_helper.rankMultiplier,
categ_list=json.dumps(categ_list),
categ_list_str=categ_list_str
)
@ -337,7 +230,7 @@ def getCoordsByRadius():
delta = dateEnd - dateStart
for i in range(delta.days+1):
correctDatetime = dateStart + datetime.timedelta(days=i)
date_str = getDateStrFormat(correctDatetime)
date_str = util.getDateStrFormat(correctDatetime)
keyCateg = 'GEO_RAD'
keyname = "{}:{}".format(keyCateg, date_str)
res = serv_redis_db.georadius(keyname, centerLon, centerLat, radius, unit='km', withcoord=True)
@ -369,50 +262,7 @@ def getCoordsByRadius():
@app.route("/_getLastContributor")
def getLastContributor():
data2 = [
{
'rank': random.randint(1,16),
'logo_path': 'logo1',
'org': 'CIRCL',
},
{
'rank': random.randint(1,16),
'logo_path': 'logo2',
'org': 'CASES',
},
{
'rank': random.randint(1,16),
'logo_path': 'logo3',
'org': 'SMILE',
},
{
'rank': random.randint(1,16),
'logo_path': 'logo4',
'org': 'ORG4',
},
{
'rank': random.randint(1,16),
'logo_path': 'logo5',
'org': 'ORG5',
},
]
try:
date = datetime.datetime.fromtimestamp(float(request.args.get('date')))
except:
date = datetime.datetime.now()
keyCateg = "CONTRIB_LAST"
topNum = MAX_NUMBER_OF_LAST_CONTRIBUTOR # default Num
last_contrib_org = getZrange(keyCateg, date, topNum)
data = []
for org, sec in last_contrib_org:
dic = {}
dic['rank'] = getOrgRankFromRedis(org, datetime.datetime.now())
dic['logo_path'] = getOrgLogoFromRedis(org)
dic['org'] = org
dic['pnts'] = getOrgPntFromRedis(org, date)
data.append(dic)
return jsonify(data)
#return jsonify(data2*2)
return jsonify(contributor_helper.getLastContributorFromRedis())
@app.route("/_getTopContributor")
def getTopContributor(suppliedDate=None):
@ -424,7 +274,7 @@ def getTopContributor(suppliedDate=None):
else:
date = suppliedDate
data = getTopContributor_fromRedis(date)
data = contributor_helper.getTopContributorFromRedis(date)
return jsonify(data)
@app.route("/_getFameContributor")
@ -435,79 +285,25 @@ def getFameContributor():
today = datetime.datetime.now()
# get previous month
date = (datetime.datetime(today.year, today.month, 1) - datetime.timedelta(days=1))
return getTopContributor(date)
return getTopContributor(suppliedDate=date)
@app.route("/_getTop5Overtime")
def getTop5Overtime():
data2 = [
{'label': 'CIRCL', 'data': [[0, 4], [1, 7], [2,14]]},
{'label': 'CASES', 'data': [[0, 1], [1, 5], [2,2]]}
]
data = []
today = datetime.datetime.now()
topSortedOrg = getTopContributor_fromRedis(today) #Get current top
# show current top 5 org points overtime (last 5 days)
for dic in topSortedOrg[0:5]:
org = dic['org']
overtime = []
for deltaD in range(1,6,1):
date = (datetime.datetime(today.year, today.month, today.day) - datetime.timedelta(days=deltaD))
keyname = 'CONTRIB_DAY:'+getDateStrFormat(date)
org_score = serv_redis_db.zscore(keyname, org)
if org_score is None:
org_score = 0
overtime.append([deltaD, org_score])
to_append = {'label': org, 'data': overtime}
data.append(to_append)
return jsonify(data)
#return jsonify(data2)
return jsonify(contributor_helper.getTop5OvertimeFromRedis())
@app.route("/_getCategPerContrib")
def getCategPerContrib():
data2 = []
for d in range(15):
dic = {}
dic['rank'] = random.randint(1,16)
dic['logo_path'] = 'logo'
dic['org'] = 'Org'+str(d)
for f in categories_in_datatable:
dic[f] = random.randint(0,1600)
data2.append(dic)
try:
date = datetime.datetime.fromtimestamp(float(request.args.get('date')))
except:
date = datetime.datetime.now()
keyCateg = "CONTRIB_DAY"
topNum = 0 # all
contrib_org = getZrange(keyCateg, date, topNum)
data = []
for org, pnts in contrib_org:
dic = {}
dic['rank'] = getTrueRank(pnts)
dic['logo_path'] = getOrgLogoFromRedis(org)
dic['org'] = org
dic['pnts'] = pnts
for categ in categories_in_datatable:
keyname = 'CONTRIB_CATEG:'+getDateStrFormat(date)+':'+categ
categ_score = serv_redis_db.zscore(keyname, org)
if categ_score is None:
categ_score = 0
dic[categ] = categ_score
data.append(dic)
return jsonify(data)
return jsonify(data2)
return jsonify(contributor_helper.getCategPerContribFromRedis(date))
@app.route("/_getAllOrg")
def getAllOrg():
data = serv_redis_db.smembers('CONTRIB_ALL_ORG')
data = [x.decode('utf8') for x in data]
data2 = ['CIRCL', 'CASES', 'SMILE' ,'ORG4' ,'ORG5', 'SUPER HYPER LONG ORGINZATION NAME', 'Org3']
return jsonify(data)
#return jsonify(data2)
return jsonify(contributor_helper.getAllOrgFromRedis())
@app.route("/_getOrgRank")
def getOrgRank():
@ -515,18 +311,7 @@ def getOrgRank():
org = request.args.get('org')
except:
org = ''
date = datetime.datetime.now()
points = random.randint(1,math.floor(cfg.getfloat('CONTRIB' ,'rankMultiplier')**16))
points = getOrgPntFromRedis(org, date)
#FIXME put 0 if org has no points
remainingPts = getRemainingPoints(points)
data = {'org': org,
'points': points,
'rank': getRankLevel(points),
'remainingPts': remainingPts['remainingPts'],
'stepPts': remainingPts['stepPts'],
}
return jsonify(data)
return jsonify(contributor_helper.getCurrentOrgRankFromRedis(org))
if __name__ == '__main__':
app.run(host='localhost', port=8001, threaded=True)

16
util.py Normal file
View File

@ -0,0 +1,16 @@
import datetime
def getMonthSpan(date):
ds = datetime.datetime(date.year, date.month, 1)
dyear = 1 if ds.month+1 > 12 else 0
dmonth = -12 if ds.month+1 > 12 else 0
de = datetime.datetime(ds.year + dyear, ds.month+1 + dmonth, 1)
delta = de - ds
to_return = []
for i in range(delta.days):
to_return.append(ds + datetime.timedelta(days=i))
return to_return
def getDateStrFormat(date):
return str(date.year)+str(date.month).zfill(2)+str(date.day).zfill(2)