import datetime import time from collections import defaultdict ONE_DAY = 60*60*24 def getZrange(serv_redis_db, keyCateg, date, topNum, endSubkey=""): date_str = getDateStrFormat(date) keyname = "{}:{}{}".format(keyCateg, date_str, endSubkey) data = serv_redis_db.zrange(keyname, 0, topNum-1, desc=True, withscores=True) data = [ [record[0], record[1]] for record in data ] return data def noSpaceLower(text): return text.lower().replace(' ', '_') def push_to_redis_zset(serv_redis_db, mainKey, toAdd, endSubkey="", count=1): now = datetime.datetime.now() today_str = getDateStrFormat(now) keyname = "{}:{}{}".format(mainKey, today_str, endSubkey) serv_redis_db.zincrby(keyname, count, toAdd) def getMonthSpan(date): ds = datetime.datetime(date.year, date.month, 1) dyear = 1 if ds.month+1 > 12 else 0 dmonth = -12 if ds.month+1 > 12 else 0 de = datetime.datetime(ds.year + dyear, ds.month+1 + dmonth, 1) delta = de - ds to_return = [] for i in range(delta.days): to_return.append(ds + datetime.timedelta(days=i)) return to_return def getXPrevDaysSpan(date, days): de = date ds = de - datetime.timedelta(days=days) delta = de - ds to_return = [] for i in range(delta.days+1): to_return.append(de - datetime.timedelta(days=i)) return to_return def getXPrevHoursSpan(date, hours): de = date de = de.replace(minute=0, second=0, microsecond=0) ds = de - datetime.timedelta(hours=hours) delta = de - ds to_return = [] for i in range(0, int(delta.total_seconds()/3600)+1): to_return.append(de - datetime.timedelta(hours=i)) return to_return def getHoursSpanOfDate(date, adaptToFitCurrentTime=True, daySpanned=6): ds = date ds = ds.replace(hour=0, minute=0, second=0, microsecond=0) to_return = [] now = datetime.datetime.now() for i in range(0, 24): the_date = ds + datetime.timedelta(hours=i) if the_date > now or the_date < now - datetime.timedelta(days=daySpanned): # avoid going outside continue to_return.append(the_date) return to_return def getDateStrFormat(date): return str(date.year)+str(date.month).zfill(2)+str(date.day).zfill(2) def getDateHoursStrFormat(date): return getDateStrFormat(date)+str(date.hour) def getTimestamp(date): return int(time.mktime(date.timetuple())) def sortByTrendingScore(toSort, topNum=5): scoredLabels = defaultdict(float) numDay = len(toSort) baseDecay = 1.0 decayRate = lambda x: baseDecay*((numDay-x**2)/numDay) for i, arr in enumerate(toSort): timestamp = arr[0] dailyData = arr[1] for item in dailyData: label = item[0] occ = item[1] scoredLabels[label] += occ*decayRate(i) topList = [[l, s] for l, s in scoredLabels.items()] topList.sort(key=lambda x: x[1], reverse=True) topSet = [ l for l, v in topList[:topNum]] # now that we have the top, filter out poor scored elements topArray = [] for arr in toSort: timestamp = arr[0] dailyData = arr[1] topDailyArray = list(filter(lambda item: (item[0] in topSet), dailyData)) dailyCombi = [timestamp, topDailyArray] topArray.append(dailyCombi) return topArray def getFields(obj, fields): jsonWalker = fields.split('.') itemToExplore = obj lastName = "" try: for i in jsonWalker: itemToExplore = itemToExplore[i] lastName = i if type(itemToExplore) is list: return {'name': lastName, 'data': itemToExplore} else: if i == 'timestamp': itemToExplore = datetime.datetime.utcfromtimestamp( int(itemToExplore)).strftime('%Y-%m-%d %H:%M:%S') return itemToExplore except KeyError as e: return None