From c1a2d17bbf38fb53ef5a8ce92c9a18bb0f5bd10d Mon Sep 17 00:00:00 2001 From: mokaddem Date: Wed, 19 Jun 2019 11:32:06 +0200 Subject: [PATCH] chg: More sane response decoding, done by the ORM --- helpers/contributor_helper.py | 6 +++--- helpers/live_helper.py | 2 +- helpers/trendings_helper.py | 12 ++++++------ helpers/users_helper.py | 6 +++--- server.py | 16 +++++++++------- util.py | 2 +- zmq_dispatcher.py | 11 +++++++---- zmq_subscriber.py | 3 ++- 8 files changed, 32 insertions(+), 26 deletions(-) diff --git a/helpers/contributor_helper.py b/helpers/contributor_helper.py index 6eb9582..3a8c861 100644 --- a/helpers/contributor_helper.py +++ b/helpers/contributor_helper.py @@ -177,7 +177,7 @@ class Contributor_helper: if pnts is None: pnts = 0 else: - pnts = int(pnts.decode('utf8')) + pnts = int(pnts) return pnts # return: [final_rank, requirement_fulfilled, requirement_not_fulfilled] @@ -381,7 +381,7 @@ class Contributor_helper: def getOrgsTrophyRanking(self, categ): keyname = '{mainKey}:{orgCateg}' res = self.serv_redis_db.zrange(keyname.format(mainKey=self.keyTrophy, orgCateg=categ), 0, -1, withscores=True, desc=True) - res = [[org.decode('utf8'), score] for org, score in res] + res = [[org, score] for org, score in res] return res def getAllOrgsTrophyRanking(self, category=None): @@ -562,7 +562,7 @@ class Contributor_helper: def getAllOrgFromRedis(self): data = self.serv_redis_db.smembers(self.keyAllOrg) - data = [x.decode('utf8') for x in data] + data = [x for x in data] return data def getCurrentOrgRankFromRedis(self, org): diff --git a/helpers/live_helper.py b/helpers/live_helper.py index 8a5c2dc..a8fa21a 100644 --- a/helpers/live_helper.py +++ b/helpers/live_helper.py @@ -48,7 +48,7 @@ class Live_helper: entries = self.serv_live.lrange(rKey, 0, -1) to_ret = [] for entry in entries: - jentry = json.loads(entry.decode('utf8')) + jentry = json.loads(entry) to_ret.append(jentry) return to_ret diff --git a/helpers/trendings_helper.py b/helpers/trendings_helper.py index 6c26a37..359d614 100644 --- a/helpers/trendings_helper.py +++ b/helpers/trendings_helper.py @@ -91,7 +91,7 @@ class Trendings_helper: for curDate in util.getXPrevDaysSpan(dateE, prev_days): keyname = "{}:{}".format(trendingType, util.getDateStrFormat(curDate)) data = self.serv_redis_db.zrange(keyname, 0, -1, desc=True, withscores=True) - data = [ [record[0].decode('utf8'), record[1]] for record in data ] + data = [ [record[0], record[1]] for record in data ] data = data if data is not None else [] to_ret.append([util.getTimestamp(curDate), data]) to_ret = util.sortByTrendingScore(to_ret, topNum=topNum) @@ -124,7 +124,7 @@ class Trendings_helper: for curDate in util.getXPrevDaysSpan(dateE, prev_days): keyname = "{}:{}".format(self.keyTag, util.getDateStrFormat(curDate)) data = self.serv_redis_db.zrange(keyname, 0, topNum-1, desc=True, withscores=True) - data = [ [record[0].decode('utf8'), record[1]] for record in data ] + data = [ [record[0], record[1]] for record in data ] data = data if data is not None else [] temp = [] for jText, score in data: @@ -139,10 +139,10 @@ class Trendings_helper: for curDate in util.getXPrevDaysSpan(dateE, prev_days): keyname = "{}:{}".format(self.keySigh, util.getDateStrFormat(curDate)) sight = self.serv_redis_db.get(keyname) - sight = 0 if sight is None else int(sight.decode('utf8')) + sight = 0 if sight is None else int(sight) keyname = "{}:{}".format(self.keyFalse, util.getDateStrFormat(curDate)) fp = self.serv_redis_db.get(keyname) - fp = 0 if fp is None else int(fp.decode('utf8')) + fp = 0 if fp is None else int(fp) to_ret.append([util.getTimestamp(curDate), { 'sightings': sight, 'false_positive': fp}]) return to_ret @@ -158,7 +158,7 @@ class Trendings_helper: keyname = "{}:{}".format(trendingType, util.getDateStrFormat(curDate)) data = self.serv_redis_db.zrange(keyname, 0, -1, desc=True) for elem in data: - allSet.add(elem.decode('utf8')) + allSet.add(elem) to_ret[trendingType] = list(allSet) tags = self.getTrendingTags(dateS, dateE) tagSet = set() @@ -187,7 +187,7 @@ class Trendings_helper: for curDate in util.getXPrevDaysSpan(dateE, prev_days): keyname = "{}:{}".format(trendingType, util.getDateStrFormat(curDate)) data = self.serv_redis_db.zrange(keyname, 0, topNum-1, desc=True, withscores=True) - data = [ [record[0].decode('utf8'), record[1]] for record in data ] + data = [ [record[0], record[1]] for record in data ] data = data if data is not None else [] to_format.append([util.getTimestamp(curDate), data]) diff --git a/helpers/users_helper.py b/helpers/users_helper.py index a2726e9..a730161 100644 --- a/helpers/users_helper.py +++ b/helpers/users_helper.py @@ -53,7 +53,7 @@ class Users_helper: def getAllOrg(self): temp = self.serv_redis_db.smembers(self.keyAllOrgLog) - return [ org.decode('utf8') for org in temp ] + return [ org for org in temp ] # return: All timestamps for one org for the spanned time or not def getDates(self, org, date=None): @@ -90,7 +90,7 @@ class Users_helper: keyname = "{}:{}".format(self.keyOrgLog, util.getDateStrFormat(curDate)) data = self.serv_redis_db.zrange(keyname, 0, -1, desc=True) for org in data: - orgs.add(org.decode('utf8')) + orgs.add(org) return list(orgs) # return: list composed of the number of [log, contrib] for one org for the time spanned @@ -134,7 +134,7 @@ class Users_helper: def getLoginVSCOntribution(self, date): keyname = "{}:{}".format(self.keyContribDay, util.getDateStrFormat(date)) orgs_contri = self.serv_redis_db.zrange(keyname, 0, -1, desc=True, withscores=False) - orgs_contri = [ org.decode('utf8') for org in orgs_contri ] + orgs_contri = [ org for org in orgs_contri ] orgs_login = [ org for org in self.getAllLoggedInOrgs(date, prev_days=0) ] contributed_num = 0 non_contributed_num = 0 diff --git a/server.py b/server.py index 41ff751..e240de6 100755 --- a/server.py +++ b/server.py @@ -34,15 +34,18 @@ app = Flask(__name__) redis_server_log = redis.StrictRedis( host=cfg.get('RedisGlobal', 'host'), port=cfg.getint('RedisGlobal', 'port'), - db=cfg.getint('RedisLog', 'db')) + db=cfg.getint('RedisLog', 'db'), + decode_responses=True) redis_server_map = redis.StrictRedis( host=cfg.get('RedisGlobal', 'host'), port=cfg.getint('RedisGlobal', 'port'), - db=cfg.getint('RedisMap', 'db')) + db=cfg.getint('RedisMap', 'db'), + decode_responses=True) serv_redis_db = redis.StrictRedis( host=cfg.get('RedisGlobal', 'host'), port=cfg.getint('RedisGlobal', 'port'), - db=cfg.getint('RedisDB', 'db')) + db=cfg.getint('RedisDB', 'db'), + decode_responses=True) streamLogCacheKey = cfg.get('RedisLog', 'streamLogCacheKey') streamMapCacheKey = cfg.get('RedisLog', 'streamMapCacheKey') @@ -110,7 +113,6 @@ class EventMessage(): # Suppose the event message is a json with the format {name: 'feedName', log:'logData'} def __init__(self, msg, filters): if not isinstance(msg, dict): - msg = msg.decode('utf8') try: jsonMsg = json.loads(msg) jsonMsg['log'] = json.loads(jsonMsg['log']) @@ -314,7 +316,7 @@ def event_stream_maps(): subscriber_map.psubscribe(cfg.get('RedisMap', 'channelDisp')) try: for msg in subscriber_map.listen(): - content = msg['data'].decode('utf8') + content = msg['data'] to_ret = 'data: {}\n\n'.format(content) yield to_ret except GeneratorExit: @@ -373,7 +375,7 @@ def eventStreamLastContributor(): subscriber_lastContrib.psubscribe(cfg.get('RedisLog', 'channelLastContributor')) try: for msg in subscriber_lastContrib.listen(): - content = msg['data'].decode('utf8') + content = msg['data'] contentJson = json.loads(content) lastContribJson = json.loads(contentJson['log']) org = lastContribJson['org'] @@ -389,7 +391,7 @@ def eventStreamAwards(): subscriber_lastAwards.psubscribe(cfg.get('RedisLog', 'channelLastAwards')) try: for msg in subscriber_lastAwards.listen(): - content = msg['data'].decode('utf8') + content = msg['data'] contentJson = json.loads(content) lastAwardJson = json.loads(contentJson['log']) org = lastAwardJson['org'] diff --git a/util.py b/util.py index 7e6e011..63eaa88 100644 --- a/util.py +++ b/util.py @@ -8,7 +8,7 @@ def getZrange(serv_redis_db, keyCateg, date, topNum, endSubkey=""): date_str = getDateStrFormat(date) keyname = "{}:{}{}".format(keyCateg, date_str, endSubkey) data = serv_redis_db.zrange(keyname, 0, topNum-1, desc=True, withscores=True) - data = [ [record[0].decode('utf8'), record[1]] for record in data ] + data = [ [record[0], record[1]] for record in data ] return data def noSpaceLower(text): diff --git a/zmq_dispatcher.py b/zmq_dispatcher.py index 13492ad..d4d68ab 100755 --- a/zmq_dispatcher.py +++ b/zmq_dispatcher.py @@ -41,15 +41,18 @@ LISTNAME = cfg.get('RedisLIST', 'listName') serv_log = redis.StrictRedis( host=cfg.get('RedisGlobal', 'host'), port=cfg.getint('RedisGlobal', 'port'), - db=cfg.getint('RedisLog', 'db')) + db=cfg.getint('RedisLog', 'db'), + decode_responses=True) serv_redis_db = redis.StrictRedis( host=cfg.get('RedisGlobal', 'host'), port=cfg.getint('RedisGlobal', 'port'), - db=cfg.getint('RedisDB', 'db')) + db=cfg.getint('RedisDB', 'db'), + decode_responses=True) serv_list = redis.StrictRedis( host=cfg.get('RedisGlobal', 'host'), port=cfg.getint('RedisGlobal', 'port'), - db=cfg.getint('RedisLIST', 'db')) + db=cfg.getint('RedisLIST', 'db'), + decode_responses=True) live_helper = live_helper.Live_helper(serv_redis_db, cfg) geo_helper = geo_helper.Geo_helper(serv_redis_db, cfg) @@ -271,7 +274,7 @@ def main(sleeptime): numMsg = 0 time.sleep(sleeptime) continue - content = content.decode('utf8') + content = content the_json = json.loads(content) zmqName = the_json['zmq_name'] content = the_json['content'] diff --git a/zmq_subscriber.py b/zmq_subscriber.py index 2948163..828df10 100755 --- a/zmq_subscriber.py +++ b/zmq_subscriber.py @@ -35,7 +35,8 @@ LISTNAME = cfg.get('RedisLIST', 'listName') serv_list = redis.StrictRedis( host=cfg.get('RedisGlobal', 'host'), port=cfg.getint('RedisGlobal', 'port'), - db=cfg.getint('RedisLIST', 'db')) + db=cfg.getint('RedisLIST', 'db'), + decode_responses=True) ###############