MISP
/
misp-dashboard
mirror of https://github.com/MISP/misp-dashboard
2
0
Fork 0
Code Issues Releases Wiki Activity

chg: More sane response decoding, done by the ORM

diagnosticTool
mokaddem 2019-06-19 11:32:06 +02:00
parent 4bfc7e2f48
commit c1a2d17bbf
8 changed files with 32 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
helpers/contributor_helper.py
View File

@ -177,7 +177,7 @@ class Contributor_helper:
if pnts is None:
pnts = 0
else:
pnts = int(pnts.decode('utf8'))
pnts = int(pnts)
return pnts
# return: [final_rank, requirement_fulfilled, requirement_not_fulfilled]
@ -381,7 +381,7 @@ class Contributor_helper:
def getOrgsTrophyRanking(self, categ):
keyname = '{mainKey}:{orgCateg}'
res = self.serv_redis_db.zrange(keyname.format(mainKey=self.keyTrophy, orgCateg=categ), 0, -1, withscores=True, desc=True)
res = [[org.decode('utf8'), score] for org, score in res]
res = [[org, score] for org, score in res]
return res
def getAllOrgsTrophyRanking(self, category=None):
@ -562,7 +562,7 @@ class Contributor_helper:
def getAllOrgFromRedis(self):
data = self.serv_redis_db.smembers(self.keyAllOrg)
data = [x.decode('utf8') for x in data]
data = [x for x in data]
return data
def getCurrentOrgRankFromRedis(self, org):

2
helpers/live_helper.py
View File

@ -48,7 +48,7 @@ class Live_helper:
entries = self.serv_live.lrange(rKey, 0, -1)
to_ret = []
for entry in entries:
jentry = json.loads(entry.decode('utf8'))
jentry = json.loads(entry)
to_ret.append(jentry)
return to_ret

12
helpers/trendings_helper.py
View File

@ -91,7 +91,7 @@ class Trendings_helper:
for curDate in util.getXPrevDaysSpan(dateE, prev_days):
keyname = "{}:{}".format(trendingType, util.getDateStrFormat(curDate))
data = self.serv_redis_db.zrange(keyname, 0, -1, desc=True, withscores=True)
data = [ [record[0].decode('utf8'), record[1]] for record in data ]
data = [ [record[0], record[1]] for record in data ]
data = data if data is not None else []
to_ret.append([util.getTimestamp(curDate), data])
to_ret = util.sortByTrendingScore(to_ret, topNum=topNum)
@ -124,7 +124,7 @@ class Trendings_helper:
for curDate in util.getXPrevDaysSpan(dateE, prev_days):
keyname = "{}:{}".format(self.keyTag, util.getDateStrFormat(curDate))
data = self.serv_redis_db.zrange(keyname, 0, topNum-1, desc=True, withscores=True)
data = [ [record[0].decode('utf8'), record[1]] for record in data ]
data = [ [record[0], record[1]] for record in data ]
data = data if data is not None else []
temp = []
for jText, score in data:
@ -139,10 +139,10 @@ class Trendings_helper:
for curDate in util.getXPrevDaysSpan(dateE, prev_days):
keyname = "{}:{}".format(self.keySigh, util.getDateStrFormat(curDate))
sight = self.serv_redis_db.get(keyname)
sight = 0 if sight is None else int(sight.decode('utf8'))
sight = 0 if sight is None else int(sight)
keyname = "{}:{}".format(self.keyFalse, util.getDateStrFormat(curDate))
fp = self.serv_redis_db.get(keyname)
fp = 0 if fp is None else int(fp.decode('utf8'))
fp = 0 if fp is None else int(fp)
to_ret.append([util.getTimestamp(curDate), { 'sightings': sight, 'false_positive': fp}])
return to_ret
@ -158,7 +158,7 @@ class Trendings_helper:
keyname = "{}:{}".format(trendingType, util.getDateStrFormat(curDate))
data = self.serv_redis_db.zrange(keyname, 0, -1, desc=True)
for elem in data:
allSet.add(elem.decode('utf8'))
allSet.add(elem)
to_ret[trendingType] = list(allSet)
tags = self.getTrendingTags(dateS, dateE)
tagSet = set()
@ -187,7 +187,7 @@ class Trendings_helper:
for curDate in util.getXPrevDaysSpan(dateE, prev_days):
keyname = "{}:{}".format(trendingType, util.getDateStrFormat(curDate))
data = self.serv_redis_db.zrange(keyname, 0, topNum-1, desc=True, withscores=True)
data = [ [record[0].decode('utf8'), record[1]] for record in data ]
data = [ [record[0], record[1]] for record in data ]
data = data if data is not None else []
to_format.append([util.getTimestamp(curDate), data])

6
helpers/users_helper.py
View File

@ -53,7 +53,7 @@ class Users_helper:
def getAllOrg(self):
temp = self.serv_redis_db.smembers(self.keyAllOrgLog)
return [ org.decode('utf8') for org in temp ]
return [ org for org in temp ]
# return: All timestamps for one org for the spanned time or not
def getDates(self, org, date=None):
@ -90,7 +90,7 @@ class Users_helper:
keyname = "{}:{}".format(self.keyOrgLog, util.getDateStrFormat(curDate))
data = self.serv_redis_db.zrange(keyname, 0, -1, desc=True)
for org in data:
orgs.add(org.decode('utf8'))
orgs.add(org)
return list(orgs)
# return: list composed of the number of [log, contrib] for one org for the time spanned
@ -134,7 +134,7 @@ class Users_helper:
def getLoginVSCOntribution(self, date):
keyname = "{}:{}".format(self.keyContribDay, util.getDateStrFormat(date))
orgs_contri = self.serv_redis_db.zrange(keyname, 0, -1, desc=True, withscores=False)
orgs_contri = [ org.decode('utf8') for org in orgs_contri ]
orgs_contri = [ org for org in orgs_contri ]
orgs_login = [ org for org in self.getAllLoggedInOrgs(date, prev_days=0) ]
contributed_num = 0
non_contributed_num = 0

16
server.py
View File

@ -34,15 +34,18 @@ app = Flask(__name__)
redis_server_log = redis.StrictRedis(
host=cfg.get('RedisGlobal', 'host'),
port=cfg.getint('RedisGlobal', 'port'),
db=cfg.getint('RedisLog', 'db'))
db=cfg.getint('RedisLog', 'db'),
decode_responses=True)
redis_server_map = redis.StrictRedis(
host=cfg.get('RedisGlobal', 'host'),
port=cfg.getint('RedisGlobal', 'port'),
db=cfg.getint('RedisMap', 'db'))
db=cfg.getint('RedisMap', 'db'),
decode_responses=True)
serv_redis_db = redis.StrictRedis(
host=cfg.get('RedisGlobal', 'host'),
port=cfg.getint('RedisGlobal', 'port'),
db=cfg.getint('RedisDB', 'db'))
db=cfg.getint('RedisDB', 'db'),
decode_responses=True)
streamLogCacheKey = cfg.get('RedisLog', 'streamLogCacheKey')
streamMapCacheKey = cfg.get('RedisLog', 'streamMapCacheKey')
@ -110,7 +113,6 @@ class EventMessage():
# Suppose the event message is a json with the format {name: 'feedName', log:'logData'}
def __init__(self, msg, filters):
if not isinstance(msg, dict):
msg = msg.decode('utf8')
try:
jsonMsg = json.loads(msg)
jsonMsg['log'] = json.loads(jsonMsg['log'])
@ -314,7 +316,7 @@ def event_stream_maps():
subscriber_map.psubscribe(cfg.get('RedisMap', 'channelDisp'))
try:
for msg in subscriber_map.listen():
content = msg['data'].decode('utf8')
content = msg['data']
to_ret = 'data: {}\n\n'.format(content)
yield to_ret
except GeneratorExit:
@ -373,7 +375,7 @@ def eventStreamLastContributor():
subscriber_lastContrib.psubscribe(cfg.get('RedisLog', 'channelLastContributor'))
try:
for msg in subscriber_lastContrib.listen():
content = msg['data'].decode('utf8')
content = msg['data']
contentJson = json.loads(content)
lastContribJson = json.loads(contentJson['log'])
org = lastContribJson['org']
@ -389,7 +391,7 @@ def eventStreamAwards():
subscriber_lastAwards.psubscribe(cfg.get('RedisLog', 'channelLastAwards'))
try:
for msg in subscriber_lastAwards.listen():
content = msg['data'].decode('utf8')
content = msg['data']
contentJson = json.loads(content)
lastAwardJson = json.loads(contentJson['log'])
org = lastAwardJson['org']

2
util.py
View File

@ -8,7 +8,7 @@ def getZrange(serv_redis_db, keyCateg, date, topNum, endSubkey=""):
date_str = getDateStrFormat(date)
keyname = "{}:{}{}".format(keyCateg, date_str, endSubkey)
data = serv_redis_db.zrange(keyname, 0, topNum-1, desc=True, withscores=True)
data = [ [record[0].decode('utf8'), record[1]] for record in data ]
data = [ [record[0], record[1]] for record in data ]
return data
def noSpaceLower(text):

11
zmq_dispatcher.py
View File

@ -41,15 +41,18 @@ LISTNAME = cfg.get('RedisLIST', 'listName')
serv_log = redis.StrictRedis(
host=cfg.get('RedisGlobal', 'host'),
port=cfg.getint('RedisGlobal', 'port'),
db=cfg.getint('RedisLog', 'db'))
db=cfg.getint('RedisLog', 'db'),
decode_responses=True)
serv_redis_db = redis.StrictRedis(
host=cfg.get('RedisGlobal', 'host'),
port=cfg.getint('RedisGlobal', 'port'),
db=cfg.getint('RedisDB', 'db'))
db=cfg.getint('RedisDB', 'db'),
decode_responses=True)
serv_list = redis.StrictRedis(
host=cfg.get('RedisGlobal', 'host'),
port=cfg.getint('RedisGlobal', 'port'),
db=cfg.getint('RedisLIST', 'db'))
db=cfg.getint('RedisLIST', 'db'),
decode_responses=True)
live_helper = live_helper.Live_helper(serv_redis_db, cfg)
geo_helper = geo_helper.Geo_helper(serv_redis_db, cfg)
@ -271,7 +274,7 @@ def main(sleeptime):
numMsg = 0
time.sleep(sleeptime)
continue
content = content.decode('utf8')
content = content
the_json = json.loads(content)
zmqName = the_json['zmq_name']
content = the_json['content']

3
zmq_subscriber.py
View File

@ -35,7 +35,8 @@ LISTNAME = cfg.get('RedisLIST', 'listName')
serv_list = redis.StrictRedis(
host=cfg.get('RedisGlobal', 'host'),
port=cfg.getint('RedisGlobal', 'port'),
db=cfg.getint('RedisLIST', 'db'))
db=cfg.getint('RedisLIST', 'db'),
decode_responses=True)
###############