mirror of https://github.com/MISP/misp-dashboard
chg: More sane response decoding, done by the ORM
parent
4bfc7e2f48
commit
c1a2d17bbf
|
@ -177,7 +177,7 @@ class Contributor_helper:
|
||||||
if pnts is None:
|
if pnts is None:
|
||||||
pnts = 0
|
pnts = 0
|
||||||
else:
|
else:
|
||||||
pnts = int(pnts.decode('utf8'))
|
pnts = int(pnts)
|
||||||
return pnts
|
return pnts
|
||||||
|
|
||||||
# return: [final_rank, requirement_fulfilled, requirement_not_fulfilled]
|
# return: [final_rank, requirement_fulfilled, requirement_not_fulfilled]
|
||||||
|
@ -381,7 +381,7 @@ class Contributor_helper:
|
||||||
def getOrgsTrophyRanking(self, categ):
|
def getOrgsTrophyRanking(self, categ):
|
||||||
keyname = '{mainKey}:{orgCateg}'
|
keyname = '{mainKey}:{orgCateg}'
|
||||||
res = self.serv_redis_db.zrange(keyname.format(mainKey=self.keyTrophy, orgCateg=categ), 0, -1, withscores=True, desc=True)
|
res = self.serv_redis_db.zrange(keyname.format(mainKey=self.keyTrophy, orgCateg=categ), 0, -1, withscores=True, desc=True)
|
||||||
res = [[org.decode('utf8'), score] for org, score in res]
|
res = [[org, score] for org, score in res]
|
||||||
return res
|
return res
|
||||||
|
|
||||||
def getAllOrgsTrophyRanking(self, category=None):
|
def getAllOrgsTrophyRanking(self, category=None):
|
||||||
|
@ -562,7 +562,7 @@ class Contributor_helper:
|
||||||
|
|
||||||
def getAllOrgFromRedis(self):
|
def getAllOrgFromRedis(self):
|
||||||
data = self.serv_redis_db.smembers(self.keyAllOrg)
|
data = self.serv_redis_db.smembers(self.keyAllOrg)
|
||||||
data = [x.decode('utf8') for x in data]
|
data = [x for x in data]
|
||||||
return data
|
return data
|
||||||
|
|
||||||
def getCurrentOrgRankFromRedis(self, org):
|
def getCurrentOrgRankFromRedis(self, org):
|
||||||
|
|
|
@ -48,7 +48,7 @@ class Live_helper:
|
||||||
entries = self.serv_live.lrange(rKey, 0, -1)
|
entries = self.serv_live.lrange(rKey, 0, -1)
|
||||||
to_ret = []
|
to_ret = []
|
||||||
for entry in entries:
|
for entry in entries:
|
||||||
jentry = json.loads(entry.decode('utf8'))
|
jentry = json.loads(entry)
|
||||||
to_ret.append(jentry)
|
to_ret.append(jentry)
|
||||||
return to_ret
|
return to_ret
|
||||||
|
|
||||||
|
|
|
@ -91,7 +91,7 @@ class Trendings_helper:
|
||||||
for curDate in util.getXPrevDaysSpan(dateE, prev_days):
|
for curDate in util.getXPrevDaysSpan(dateE, prev_days):
|
||||||
keyname = "{}:{}".format(trendingType, util.getDateStrFormat(curDate))
|
keyname = "{}:{}".format(trendingType, util.getDateStrFormat(curDate))
|
||||||
data = self.serv_redis_db.zrange(keyname, 0, -1, desc=True, withscores=True)
|
data = self.serv_redis_db.zrange(keyname, 0, -1, desc=True, withscores=True)
|
||||||
data = [ [record[0].decode('utf8'), record[1]] for record in data ]
|
data = [ [record[0], record[1]] for record in data ]
|
||||||
data = data if data is not None else []
|
data = data if data is not None else []
|
||||||
to_ret.append([util.getTimestamp(curDate), data])
|
to_ret.append([util.getTimestamp(curDate), data])
|
||||||
to_ret = util.sortByTrendingScore(to_ret, topNum=topNum)
|
to_ret = util.sortByTrendingScore(to_ret, topNum=topNum)
|
||||||
|
@ -124,7 +124,7 @@ class Trendings_helper:
|
||||||
for curDate in util.getXPrevDaysSpan(dateE, prev_days):
|
for curDate in util.getXPrevDaysSpan(dateE, prev_days):
|
||||||
keyname = "{}:{}".format(self.keyTag, util.getDateStrFormat(curDate))
|
keyname = "{}:{}".format(self.keyTag, util.getDateStrFormat(curDate))
|
||||||
data = self.serv_redis_db.zrange(keyname, 0, topNum-1, desc=True, withscores=True)
|
data = self.serv_redis_db.zrange(keyname, 0, topNum-1, desc=True, withscores=True)
|
||||||
data = [ [record[0].decode('utf8'), record[1]] for record in data ]
|
data = [ [record[0], record[1]] for record in data ]
|
||||||
data = data if data is not None else []
|
data = data if data is not None else []
|
||||||
temp = []
|
temp = []
|
||||||
for jText, score in data:
|
for jText, score in data:
|
||||||
|
@ -139,10 +139,10 @@ class Trendings_helper:
|
||||||
for curDate in util.getXPrevDaysSpan(dateE, prev_days):
|
for curDate in util.getXPrevDaysSpan(dateE, prev_days):
|
||||||
keyname = "{}:{}".format(self.keySigh, util.getDateStrFormat(curDate))
|
keyname = "{}:{}".format(self.keySigh, util.getDateStrFormat(curDate))
|
||||||
sight = self.serv_redis_db.get(keyname)
|
sight = self.serv_redis_db.get(keyname)
|
||||||
sight = 0 if sight is None else int(sight.decode('utf8'))
|
sight = 0 if sight is None else int(sight)
|
||||||
keyname = "{}:{}".format(self.keyFalse, util.getDateStrFormat(curDate))
|
keyname = "{}:{}".format(self.keyFalse, util.getDateStrFormat(curDate))
|
||||||
fp = self.serv_redis_db.get(keyname)
|
fp = self.serv_redis_db.get(keyname)
|
||||||
fp = 0 if fp is None else int(fp.decode('utf8'))
|
fp = 0 if fp is None else int(fp)
|
||||||
to_ret.append([util.getTimestamp(curDate), { 'sightings': sight, 'false_positive': fp}])
|
to_ret.append([util.getTimestamp(curDate), { 'sightings': sight, 'false_positive': fp}])
|
||||||
return to_ret
|
return to_ret
|
||||||
|
|
||||||
|
@ -158,7 +158,7 @@ class Trendings_helper:
|
||||||
keyname = "{}:{}".format(trendingType, util.getDateStrFormat(curDate))
|
keyname = "{}:{}".format(trendingType, util.getDateStrFormat(curDate))
|
||||||
data = self.serv_redis_db.zrange(keyname, 0, -1, desc=True)
|
data = self.serv_redis_db.zrange(keyname, 0, -1, desc=True)
|
||||||
for elem in data:
|
for elem in data:
|
||||||
allSet.add(elem.decode('utf8'))
|
allSet.add(elem)
|
||||||
to_ret[trendingType] = list(allSet)
|
to_ret[trendingType] = list(allSet)
|
||||||
tags = self.getTrendingTags(dateS, dateE)
|
tags = self.getTrendingTags(dateS, dateE)
|
||||||
tagSet = set()
|
tagSet = set()
|
||||||
|
@ -187,7 +187,7 @@ class Trendings_helper:
|
||||||
for curDate in util.getXPrevDaysSpan(dateE, prev_days):
|
for curDate in util.getXPrevDaysSpan(dateE, prev_days):
|
||||||
keyname = "{}:{}".format(trendingType, util.getDateStrFormat(curDate))
|
keyname = "{}:{}".format(trendingType, util.getDateStrFormat(curDate))
|
||||||
data = self.serv_redis_db.zrange(keyname, 0, topNum-1, desc=True, withscores=True)
|
data = self.serv_redis_db.zrange(keyname, 0, topNum-1, desc=True, withscores=True)
|
||||||
data = [ [record[0].decode('utf8'), record[1]] for record in data ]
|
data = [ [record[0], record[1]] for record in data ]
|
||||||
data = data if data is not None else []
|
data = data if data is not None else []
|
||||||
to_format.append([util.getTimestamp(curDate), data])
|
to_format.append([util.getTimestamp(curDate), data])
|
||||||
|
|
||||||
|
|
|
@ -53,7 +53,7 @@ class Users_helper:
|
||||||
|
|
||||||
def getAllOrg(self):
|
def getAllOrg(self):
|
||||||
temp = self.serv_redis_db.smembers(self.keyAllOrgLog)
|
temp = self.serv_redis_db.smembers(self.keyAllOrgLog)
|
||||||
return [ org.decode('utf8') for org in temp ]
|
return [ org for org in temp ]
|
||||||
|
|
||||||
# return: All timestamps for one org for the spanned time or not
|
# return: All timestamps for one org for the spanned time or not
|
||||||
def getDates(self, org, date=None):
|
def getDates(self, org, date=None):
|
||||||
|
@ -90,7 +90,7 @@ class Users_helper:
|
||||||
keyname = "{}:{}".format(self.keyOrgLog, util.getDateStrFormat(curDate))
|
keyname = "{}:{}".format(self.keyOrgLog, util.getDateStrFormat(curDate))
|
||||||
data = self.serv_redis_db.zrange(keyname, 0, -1, desc=True)
|
data = self.serv_redis_db.zrange(keyname, 0, -1, desc=True)
|
||||||
for org in data:
|
for org in data:
|
||||||
orgs.add(org.decode('utf8'))
|
orgs.add(org)
|
||||||
return list(orgs)
|
return list(orgs)
|
||||||
|
|
||||||
# return: list composed of the number of [log, contrib] for one org for the time spanned
|
# return: list composed of the number of [log, contrib] for one org for the time spanned
|
||||||
|
@ -134,7 +134,7 @@ class Users_helper:
|
||||||
def getLoginVSCOntribution(self, date):
|
def getLoginVSCOntribution(self, date):
|
||||||
keyname = "{}:{}".format(self.keyContribDay, util.getDateStrFormat(date))
|
keyname = "{}:{}".format(self.keyContribDay, util.getDateStrFormat(date))
|
||||||
orgs_contri = self.serv_redis_db.zrange(keyname, 0, -1, desc=True, withscores=False)
|
orgs_contri = self.serv_redis_db.zrange(keyname, 0, -1, desc=True, withscores=False)
|
||||||
orgs_contri = [ org.decode('utf8') for org in orgs_contri ]
|
orgs_contri = [ org for org in orgs_contri ]
|
||||||
orgs_login = [ org for org in self.getAllLoggedInOrgs(date, prev_days=0) ]
|
orgs_login = [ org for org in self.getAllLoggedInOrgs(date, prev_days=0) ]
|
||||||
contributed_num = 0
|
contributed_num = 0
|
||||||
non_contributed_num = 0
|
non_contributed_num = 0
|
||||||
|
|
16
server.py
16
server.py
|
@ -34,15 +34,18 @@ app = Flask(__name__)
|
||||||
redis_server_log = redis.StrictRedis(
|
redis_server_log = redis.StrictRedis(
|
||||||
host=cfg.get('RedisGlobal', 'host'),
|
host=cfg.get('RedisGlobal', 'host'),
|
||||||
port=cfg.getint('RedisGlobal', 'port'),
|
port=cfg.getint('RedisGlobal', 'port'),
|
||||||
db=cfg.getint('RedisLog', 'db'))
|
db=cfg.getint('RedisLog', 'db'),
|
||||||
|
decode_responses=True)
|
||||||
redis_server_map = redis.StrictRedis(
|
redis_server_map = redis.StrictRedis(
|
||||||
host=cfg.get('RedisGlobal', 'host'),
|
host=cfg.get('RedisGlobal', 'host'),
|
||||||
port=cfg.getint('RedisGlobal', 'port'),
|
port=cfg.getint('RedisGlobal', 'port'),
|
||||||
db=cfg.getint('RedisMap', 'db'))
|
db=cfg.getint('RedisMap', 'db'),
|
||||||
|
decode_responses=True)
|
||||||
serv_redis_db = redis.StrictRedis(
|
serv_redis_db = redis.StrictRedis(
|
||||||
host=cfg.get('RedisGlobal', 'host'),
|
host=cfg.get('RedisGlobal', 'host'),
|
||||||
port=cfg.getint('RedisGlobal', 'port'),
|
port=cfg.getint('RedisGlobal', 'port'),
|
||||||
db=cfg.getint('RedisDB', 'db'))
|
db=cfg.getint('RedisDB', 'db'),
|
||||||
|
decode_responses=True)
|
||||||
|
|
||||||
streamLogCacheKey = cfg.get('RedisLog', 'streamLogCacheKey')
|
streamLogCacheKey = cfg.get('RedisLog', 'streamLogCacheKey')
|
||||||
streamMapCacheKey = cfg.get('RedisLog', 'streamMapCacheKey')
|
streamMapCacheKey = cfg.get('RedisLog', 'streamMapCacheKey')
|
||||||
|
@ -110,7 +113,6 @@ class EventMessage():
|
||||||
# Suppose the event message is a json with the format {name: 'feedName', log:'logData'}
|
# Suppose the event message is a json with the format {name: 'feedName', log:'logData'}
|
||||||
def __init__(self, msg, filters):
|
def __init__(self, msg, filters):
|
||||||
if not isinstance(msg, dict):
|
if not isinstance(msg, dict):
|
||||||
msg = msg.decode('utf8')
|
|
||||||
try:
|
try:
|
||||||
jsonMsg = json.loads(msg)
|
jsonMsg = json.loads(msg)
|
||||||
jsonMsg['log'] = json.loads(jsonMsg['log'])
|
jsonMsg['log'] = json.loads(jsonMsg['log'])
|
||||||
|
@ -314,7 +316,7 @@ def event_stream_maps():
|
||||||
subscriber_map.psubscribe(cfg.get('RedisMap', 'channelDisp'))
|
subscriber_map.psubscribe(cfg.get('RedisMap', 'channelDisp'))
|
||||||
try:
|
try:
|
||||||
for msg in subscriber_map.listen():
|
for msg in subscriber_map.listen():
|
||||||
content = msg['data'].decode('utf8')
|
content = msg['data']
|
||||||
to_ret = 'data: {}\n\n'.format(content)
|
to_ret = 'data: {}\n\n'.format(content)
|
||||||
yield to_ret
|
yield to_ret
|
||||||
except GeneratorExit:
|
except GeneratorExit:
|
||||||
|
@ -373,7 +375,7 @@ def eventStreamLastContributor():
|
||||||
subscriber_lastContrib.psubscribe(cfg.get('RedisLog', 'channelLastContributor'))
|
subscriber_lastContrib.psubscribe(cfg.get('RedisLog', 'channelLastContributor'))
|
||||||
try:
|
try:
|
||||||
for msg in subscriber_lastContrib.listen():
|
for msg in subscriber_lastContrib.listen():
|
||||||
content = msg['data'].decode('utf8')
|
content = msg['data']
|
||||||
contentJson = json.loads(content)
|
contentJson = json.loads(content)
|
||||||
lastContribJson = json.loads(contentJson['log'])
|
lastContribJson = json.loads(contentJson['log'])
|
||||||
org = lastContribJson['org']
|
org = lastContribJson['org']
|
||||||
|
@ -389,7 +391,7 @@ def eventStreamAwards():
|
||||||
subscriber_lastAwards.psubscribe(cfg.get('RedisLog', 'channelLastAwards'))
|
subscriber_lastAwards.psubscribe(cfg.get('RedisLog', 'channelLastAwards'))
|
||||||
try:
|
try:
|
||||||
for msg in subscriber_lastAwards.listen():
|
for msg in subscriber_lastAwards.listen():
|
||||||
content = msg['data'].decode('utf8')
|
content = msg['data']
|
||||||
contentJson = json.loads(content)
|
contentJson = json.loads(content)
|
||||||
lastAwardJson = json.loads(contentJson['log'])
|
lastAwardJson = json.loads(contentJson['log'])
|
||||||
org = lastAwardJson['org']
|
org = lastAwardJson['org']
|
||||||
|
|
2
util.py
2
util.py
|
@ -8,7 +8,7 @@ def getZrange(serv_redis_db, keyCateg, date, topNum, endSubkey=""):
|
||||||
date_str = getDateStrFormat(date)
|
date_str = getDateStrFormat(date)
|
||||||
keyname = "{}:{}{}".format(keyCateg, date_str, endSubkey)
|
keyname = "{}:{}{}".format(keyCateg, date_str, endSubkey)
|
||||||
data = serv_redis_db.zrange(keyname, 0, topNum-1, desc=True, withscores=True)
|
data = serv_redis_db.zrange(keyname, 0, topNum-1, desc=True, withscores=True)
|
||||||
data = [ [record[0].decode('utf8'), record[1]] for record in data ]
|
data = [ [record[0], record[1]] for record in data ]
|
||||||
return data
|
return data
|
||||||
|
|
||||||
def noSpaceLower(text):
|
def noSpaceLower(text):
|
||||||
|
|
|
@ -41,15 +41,18 @@ LISTNAME = cfg.get('RedisLIST', 'listName')
|
||||||
serv_log = redis.StrictRedis(
|
serv_log = redis.StrictRedis(
|
||||||
host=cfg.get('RedisGlobal', 'host'),
|
host=cfg.get('RedisGlobal', 'host'),
|
||||||
port=cfg.getint('RedisGlobal', 'port'),
|
port=cfg.getint('RedisGlobal', 'port'),
|
||||||
db=cfg.getint('RedisLog', 'db'))
|
db=cfg.getint('RedisLog', 'db'),
|
||||||
|
decode_responses=True)
|
||||||
serv_redis_db = redis.StrictRedis(
|
serv_redis_db = redis.StrictRedis(
|
||||||
host=cfg.get('RedisGlobal', 'host'),
|
host=cfg.get('RedisGlobal', 'host'),
|
||||||
port=cfg.getint('RedisGlobal', 'port'),
|
port=cfg.getint('RedisGlobal', 'port'),
|
||||||
db=cfg.getint('RedisDB', 'db'))
|
db=cfg.getint('RedisDB', 'db'),
|
||||||
|
decode_responses=True)
|
||||||
serv_list = redis.StrictRedis(
|
serv_list = redis.StrictRedis(
|
||||||
host=cfg.get('RedisGlobal', 'host'),
|
host=cfg.get('RedisGlobal', 'host'),
|
||||||
port=cfg.getint('RedisGlobal', 'port'),
|
port=cfg.getint('RedisGlobal', 'port'),
|
||||||
db=cfg.getint('RedisLIST', 'db'))
|
db=cfg.getint('RedisLIST', 'db'),
|
||||||
|
decode_responses=True)
|
||||||
|
|
||||||
live_helper = live_helper.Live_helper(serv_redis_db, cfg)
|
live_helper = live_helper.Live_helper(serv_redis_db, cfg)
|
||||||
geo_helper = geo_helper.Geo_helper(serv_redis_db, cfg)
|
geo_helper = geo_helper.Geo_helper(serv_redis_db, cfg)
|
||||||
|
@ -271,7 +274,7 @@ def main(sleeptime):
|
||||||
numMsg = 0
|
numMsg = 0
|
||||||
time.sleep(sleeptime)
|
time.sleep(sleeptime)
|
||||||
continue
|
continue
|
||||||
content = content.decode('utf8')
|
content = content
|
||||||
the_json = json.loads(content)
|
the_json = json.loads(content)
|
||||||
zmqName = the_json['zmq_name']
|
zmqName = the_json['zmq_name']
|
||||||
content = the_json['content']
|
content = the_json['content']
|
||||||
|
|
|
@ -35,7 +35,8 @@ LISTNAME = cfg.get('RedisLIST', 'listName')
|
||||||
serv_list = redis.StrictRedis(
|
serv_list = redis.StrictRedis(
|
||||||
host=cfg.get('RedisGlobal', 'host'),
|
host=cfg.get('RedisGlobal', 'host'),
|
||||||
port=cfg.getint('RedisGlobal', 'port'),
|
port=cfg.getint('RedisGlobal', 'port'),
|
||||||
db=cfg.getint('RedisLIST', 'db'))
|
db=cfg.getint('RedisLIST', 'db'),
|
||||||
|
decode_responses=True)
|
||||||
|
|
||||||
|
|
||||||
###############
|
###############
|
||||||
|
|
Loading…
Reference in New Issue