import math, random import os import json import datetime, time import logging import util from . import contributor_helper class Users_helper: def __init__(self, serv_redis_db, cfg): self.serv_redis_db = serv_redis_db self.cfg = cfg # REDIS keys self.keyTimestamp = "LOGIN_TIMESTAMP" self.keyTimestampSet = "LOGIN_TIMESTAMPSET" self.keyOrgLog = "LOGIN_ORG" self.keyContribDay = contributor_helper.KEYDAY # Key to get monthly contribution #logger logDir = cfg.get('Log', 'directory') logfilename = cfg.get('Log', 'filename') logPath = os.path.join(logDir, logfilename) if not os.path.exists(logDir): os.makedirs(logDir) logging.basicConfig(filename=logPath, filemode='w', level=logging.INFO) self.logger = logging.getLogger(__name__) def addTemporary(self, org, timestamp): timestampDate = datetime.datetime.fromtimestamp(float(timestamp)) timestampDate_str = util.getDateHoursStrFormat(timestampDate) keyname_timestamp = "{}:{}".format(self.keyTimestampSet, timestampDate_str) self.serv_redis_db.sadd(keyname_timestamp, org) self.logger.debug('Added to redis: keyname={}, org={}'.format(keyname_timestamp, org)) self.serv_redis_db.expire(keyname_timestamp, 60*60) def hasAlreadyBeenAdded(self, org, timestamp): timestampDate = datetime.datetime.fromtimestamp(float(timestamp)) timestampDate_str = util.getDateHoursStrFormat(timestampDate) keyname_timestamp = "{}:{}".format(self.keyTimestampSet, timestampDate_str) orgs = [ org.decode('utf8') for org in self.serv_redis_db.smembers(keyname_timestamp) ] if orgs is None: return False return (org in orgs) def add_user_login(self, timestamp, org): timestampDate = datetime.datetime.fromtimestamp(float(timestamp)) timestampDate_str = util.getDateStrFormat(timestampDate) if not self.hasAlreadyBeenAdded(org, timestamp): keyname_timestamp = "{}:{}".format(self.keyTimestamp, timestampDate_str) self.serv_redis_db.sadd(keyname_timestamp, timestamp) self.logger.debug('Added to redis: keyname={}, org={}'.format(keyname_timestamp, timestamp)) self.addTemporary(org, timestamp) keyname_org = "{}:{}".format(self.keyOrgLog, timestampDate_str) self.serv_redis_db.zincrby(keyname_org, org, 1) self.logger.debug('Added to redis: keyname={}, org={}'.format(keyname_org, org)) def getUserLogins(self, date): keyname = "{}:{}".format(self.keyTimestamp, util.getDateStrFormat(date)) timestamps = self.serv_redis_db.smembers(keyname) timestamps = [int(timestamp.decode('utf8')) for timestamp in timestamps] return timestamps def getOrgslogin(self, date, topNum=12): keyname = "{}:{}".format(self.keyOrgLog, util.getDateStrFormat(date)) data = self.serv_redis_db.zrange(keyname, 0, topNum-1, desc=True, withscores=True) data = [ [record[0].decode('utf8'), record[1]] for record in data ] return data def getAllLoggedInOrgs(self, date, prev_days=31): orgs = set() for curDate in util.getXPrevDaysSpan(date, prev_days): keyname = "{}:{}".format(self.keyOrgLog, util.getDateStrFormat(curDate)) data = self.serv_redis_db.zrange(keyname, 0, -1, desc=True, withscores=True) for org in data: orgs.add(org[0].decode('utf8')) return list(orgs) def getOrgContribAndLogin(self, date, org, prev_days=31): keyname_log = "{}:{}" keyname_contrib = "{}:{}" data = [] for curDate in util.getXPrevDaysSpan(date, prev_days): log = self.serv_redis_db.zscore(keyname_log.format(self.keyOrgLog, util.getDateStrFormat(curDate)), org) log = 0 if log is None else 1 contrib = self.serv_redis_db.zscore(keyname_contrib.format(self.keyContribDay, util.getDateStrFormat(curDate)), org) contrib = 0 if contrib is None else 1 data.append([log, contrib]) return data def getContribOverLoginScore(self, array): totLog = 0 totContrib = 0 for log, contrib in array: totLog += log totContrib += contrib if totLog == 0: # avoid div by 0 totLog = 1 return totContrib/totLog def getTopOrglogin(self, date, maxNum=12, prev_days=7): all_logged_in_orgs = self.getAllLoggedInOrgs(date, prev_days) data = [] for org in all_logged_in_orgs: orgStatus = self.getOrgContribAndLogin(date, org, prev_days) orgScore = self.getContribOverLoginScore(orgStatus) data.append([org, orgScore]) data.sort(key=lambda x: x[1], reverse=True) return data[:maxNum] def getLoginVSCOntribution(self, date): keyname = "{}:{}".format(self.keyContribDay, util.getDateStrFormat(date)) orgs_contri = self.serv_redis_db.zrange(keyname, 0, -1, desc=True, withscores=False) orgs_contri = [ org.decode('utf8') for org in orgs_contri ] orgs_login = [ org[0] for org in self.getOrgslogin(date, topNum=0) ] contributed_num = 0 non_contributed_num = 0 for org in orgs_login: if org in orgs_contri: contributed_num += 1 else: non_contributed_num +=1 return [contributed_num, non_contributed_num] def getUserLoginsForPunchCard(self, date, prev_days=6): week = {} for curDate in util.getXPrevDaysSpan(date, prev_days): timestamps = self.getUserLogins(curDate) day = {} for timestamp in timestamps: date = datetime.datetime.fromtimestamp(float(timestamp)) if date.hour not in day: day[date.hour] = 0 day[date.hour] += 1 week[curDate.weekday()] = day # Format data data = [] for d in range(7): try: to_append = [] for h in range(24): try: to_append.append(week[d][h]) except KeyError: to_append.append(0) data.append(to_append) except KeyError: # no data data.append([0 for x in range(24)]) # swap: punchcard day starts on monday data = [data[6]]+data[:6] return data def getUserLoginsAndContribOvertime(self, date, prev_days=6): dico_hours_contrib = {} dico_hours = {} for curDate in util.getXPrevHoursSpan(date, prev_days*24): dico_hours[util.getTimestamp(curDate)] = 0 # populate with empty data dico_hours_contrib[util.getTimestamp(curDate)] = 0 # populate with empty data for curDate in util.getXPrevDaysSpan(date, prev_days): timestamps = self.getUserLogins(curDate) keyname = "{}:{}".format(self.keyContribDay, util.getDateStrFormat(curDate)) orgs_contri = self.serv_redis_db.zrange(keyname, 0, -1, desc=True, withscores=False) orgs_contri_num = len(orgs_contri) for curDate in util.getHoursSpanOfDate(curDate, adaptToFitCurrentTime=True): #fill hole day dico_hours_contrib[util.getTimestamp(curDate)] = orgs_contri_num for timestamp in timestamps: # sum occurence during the current hour dateTimestamp = datetime.datetime.fromtimestamp(float(timestamp)) dateTimestamp = dateTimestamp.replace(minute=0, second=0, microsecond=0) try: dico_hours[util.getTimestamp(dateTimestamp)] += 1 except KeyError: # timestamp out of bound (greater than 1 week) pass # Format data # login to_ret = {} data = [] for curDate, occ in dico_hours.items(): data.append([curDate, occ]) data.sort(key=lambda x: x[0]) to_ret['login'] = data # contrib data = [] for curDate, occ in dico_hours_contrib.items(): data.append([curDate, occ]) data.sort(key=lambda x: x[0]) to_ret['contrib'] = data return to_ret