fix: [all] Fixed issue with py-redis>2.x and fix failed merge conflict

diagnosticTool
mokaddem 2019-06-18 11:25:17 +02:00
parent 618da4eea5
commit 68f158ebed
10 changed files with 19 additions and 23 deletions

View File

@ -17,7 +17,7 @@ size_dashboard_left_width = 5
size_openStreet_pannel_perc = 55 size_openStreet_pannel_perc = 55
size_world_pannel_perc = 35 size_world_pannel_perc = 35
item_to_plot = Attribute.category item_to_plot = Attribute.category
fieldname_order=["Event.id", "Attribute.Tag", "Attribute.category", "Attribute.type", ["Attribute.value", "Attribute.comment"]] fieldname_order=["Attribute.timestamp", "Event.id", "Attribute.Tag", "Attribute.category", "Attribute.type", ["Attribute.value", "Attribute.comment"]]
char_separator=|| char_separator=||
[GEO] [GEO]

View File

@ -210,7 +210,7 @@ def main():
for award in awards_given: for award in awards_given:
# update awards given # update awards given
serv_redis_db.zadd('CONTRIB_LAST_AWARDS:'+util.getDateStrFormat(now), nowSec, json.dumps({'org': org, 'award': award, 'epoch': nowSec })) serv_redis_db.zadd('CONTRIB_LAST_AWARDS:'+util.getDateStrFormat(now), {json.dumps({'org': org, 'award': award, 'epoch': nowSec }): nowSec})
serv_redis_db.expire('CONTRIB_LAST_AWARDS:'+util.getDateStrFormat(now), ONE_DAY*7) #expire after 7 day serv_redis_db.expire('CONTRIB_LAST_AWARDS:'+util.getDateStrFormat(now), ONE_DAY*7) #expire after 7 day
# publish # publish
publish_log('GIVE_HONOR_ZMQ', 'CONTRIBUTION', {'org': org, 'award': award, 'epoch': nowSec }, CHANNEL_LASTAWARDS) publish_log('GIVE_HONOR_ZMQ', 'CONTRIBUTION', {'org': org, 'award': award, 'epoch': nowSec }, CHANNEL_LASTAWARDS)

View File

@ -115,7 +115,7 @@ class Contributor_helper:
def addContributionToCateg(self, date, categ, org, count=1): def addContributionToCateg(self, date, categ, org, count=1):
today_str = util.getDateStrFormat(date) today_str = util.getDateStrFormat(date)
keyname = "{}:{}:{}".format(self.keyCateg, today_str, categ) keyname = "{}:{}:{}".format(self.keyCateg, today_str, categ)
self.serv_redis_db.zincrby(keyname, org, count) self.serv_redis_db.zincrby(keyname, count, org)
self.logger.debug('Added to redis: keyname={}, org={}, count={}'.format(keyname, org, count)) self.logger.debug('Added to redis: keyname={}, org={}, count={}'.format(keyname, org, count))
def publish_log(self, zmq_name, name, content, channel=""): def publish_log(self, zmq_name, name, content, channel=""):
@ -155,7 +155,7 @@ class Contributor_helper:
self.serv_redis_db.sadd(self.keyAllOrg, org) self.serv_redis_db.sadd(self.keyAllOrg, org)
keyname = "{}:{}".format(self.keyLastContrib, util.getDateStrFormat(now)) keyname = "{}:{}".format(self.keyLastContrib, util.getDateStrFormat(now))
self.serv_redis_db.zadd(keyname, nowSec, org) self.serv_redis_db.zadd(keyname, {org: nowSec})
self.logger.debug('Added to redis: keyname={}, nowSec={}, org={}'.format(keyname, nowSec, org)) self.logger.debug('Added to redis: keyname={}, nowSec={}, org={}'.format(keyname, nowSec, org))
self.serv_redis_db.expire(keyname, util.ONE_DAY*7) #expire after 7 day self.serv_redis_db.expire(keyname, util.ONE_DAY*7) #expire after 7 day
@ -164,7 +164,7 @@ class Contributor_helper:
for award in awards_given: for award in awards_given:
# update awards given # update awards given
keyname = "{}:{}".format(self.keyLastAward, util.getDateStrFormat(now)) keyname = "{}:{}".format(self.keyLastAward, util.getDateStrFormat(now))
self.serv_redis_db.zadd(keyname, nowSec, json.dumps({'org': org, 'award': award, 'epoch': nowSec })) self.serv_redis_db.zadd(keyname, {json.dumps({'org': org, 'award': award, 'epoch': nowSec }): nowSec})
self.logger.debug('Added to redis: keyname={}, nowSec={}, content={}'.format(keyname, nowSec, json.dumps({'org': org, 'award': award, 'epoch': nowSec }))) self.logger.debug('Added to redis: keyname={}, nowSec={}, content={}'.format(keyname, nowSec, json.dumps({'org': org, 'award': award, 'epoch': nowSec })))
self.serv_redis_db.expire(keyname, util.ONE_DAY*7) #expire after 7 day self.serv_redis_db.expire(keyname, util.ONE_DAY*7) #expire after 7 day
# publish # publish
@ -410,12 +410,12 @@ class Contributor_helper:
def giveTrophyPointsToOrg(self, org, categ, points): def giveTrophyPointsToOrg(self, org, categ, points):
keyname = '{mainKey}:{orgCateg}' keyname = '{mainKey}:{orgCateg}'
self.serv_redis_db.zincrby(keyname.format(mainKey=self.keyTrophy, orgCateg=categ), org, points) self.serv_redis_db.zincrby(keyname.format(mainKey=self.keyTrophy, orgCateg=categ), points, org)
self.logger.debug('Giving {} trophy points to {} in {}'.format(points, org, categ)) self.logger.debug('Giving {} trophy points to {} in {}'.format(points, org, categ))
def removeTrophyPointsFromOrg(self, org, categ, points): def removeTrophyPointsFromOrg(self, org, categ, points):
keyname = '{mainKey}:{orgCateg}' keyname = '{mainKey}:{orgCateg}'
self.serv_redis_db.zincrby(keyname.format(mainKey=self.keyTrophy, orgCateg=categ), org, -points) self.serv_redis_db.zincrby(keyname.format(mainKey=self.keyTrophy, orgCateg=categ), -points, org)
self.logger.debug('Removing {} trophy points from {} in {}'.format(points, org, categ)) self.logger.debug('Removing {} trophy points from {} in {}'.format(points, org, categ))
''' AWARDS HELPER ''' ''' AWARDS HELPER '''

View File

@ -206,7 +206,7 @@ class Geo_helper:
now = datetime.datetime.now() now = datetime.datetime.now()
today_str = util.getDateStrFormat(now) today_str = util.getDateStrFormat(now)
keyname = "{}:{}{}".format(keyCateg, today_str, endSubkey) keyname = "{}:{}{}".format(keyCateg, today_str, endSubkey)
self.serv_redis_db.zincrby(keyname, toAdd, count) self.serv_redis_db.zincrby(keyname, count, toAdd)
self.logger.debug('Added to redis: keyname={}, toAdd={}, count={}'.format(keyname, toAdd, count)) self.logger.debug('Added to redis: keyname={}, toAdd={}, count={}'.format(keyname, toAdd, count))
def ip_to_coord(self, ip): def ip_to_coord(self, ip):

View File

@ -49,7 +49,7 @@ class Trendings_helper:
to_save = json.dumps(data) to_save = json.dumps(data)
else: else:
to_save = data to_save = data
self.serv_redis_db.zincrby(keyname, to_save, 1) self.serv_redis_db.zincrby(keyname, 1, to_save)
self.logger.debug('Added to redis: keyname={}, content={}'.format(keyname, to_save)) self.logger.debug('Added to redis: keyname={}, content={}'.format(keyname, to_save))
def addTrendingEvent(self, eventName, timestamp): def addTrendingEvent(self, eventName, timestamp):

View File

@ -41,11 +41,11 @@ class Users_helper:
timestampDate_str = util.getDateStrFormat(timestampDate) timestampDate_str = util.getDateStrFormat(timestampDate)
keyname_timestamp = "{}:{}".format(self.keyTimestamp, org) keyname_timestamp = "{}:{}".format(self.keyTimestamp, org)
self.serv_redis_db.zadd(keyname_timestamp, timestamp, timestamp) self.serv_redis_db.zadd(keyname_timestamp, {timestamp: timestamp})
self.logger.debug('Added to redis: keyname={}, org={}'.format(keyname_timestamp, timestamp)) self.logger.debug('Added to redis: keyname={}, org={}'.format(keyname_timestamp, timestamp))
keyname_org = "{}:{}".format(self.keyOrgLog, timestampDate_str) keyname_org = "{}:{}".format(self.keyOrgLog, timestampDate_str)
self.serv_redis_db.zincrby(keyname_org, org, 1) self.serv_redis_db.zincrby(keyname_org, 1, org)
self.logger.debug('Added to redis: keyname={}, org={}'.format(keyname_org, org)) self.logger.debug('Added to redis: keyname={}, org={}'.format(keyname_org, org))
self.serv_redis_db.sadd(self.keyAllOrgLog, org) self.serv_redis_db.sadd(self.keyAllOrgLog, org)

View File

@ -1,8 +1,7 @@
argparse argparse
flask flask
geoip2 geoip2
# Redis needs to be 2.10.6 due to a change in redis 3.10 client, see here: https://github.com/MISP/misp-dashboard/issues/76#issuecomment-439389621 redis
redis==2.10.6
phonenumbers phonenumbers
pip pip
pycountry pycountry

View File

@ -95,13 +95,6 @@ class LogItem():
else: else:
to_add = util.getFields(self.feed, field) to_add = util.getFields(self.feed, field)
to_ret[i] = to_add if to_add is not None else '' to_ret[i] = to_add if to_add is not None else ''
# Number to keep them sorted (jsonify sort keys)
for item in range(len(LogItem.FIELDNAME_ORDER)):
try:
to_ret[item] = self.fields[item]
except IndexError: # not enough field in rcv item
to_ret[item] = ''
return to_ret return to_ret

View File

@ -18,7 +18,7 @@ def push_to_redis_zset(serv_redis_db, mainKey, toAdd, endSubkey="", count=1):
now = datetime.datetime.now() now = datetime.datetime.now()
today_str = getDateStrFormat(now) today_str = getDateStrFormat(now)
keyname = "{}:{}{}".format(mainKey, today_str, endSubkey) keyname = "{}:{}{}".format(mainKey, today_str, endSubkey)
serv_redis_db.zincrby(keyname, toAdd, count) serv_redis_db.zincrby(keyname, count, toAdd)
def getMonthSpan(date): def getMonthSpan(date):
ds = datetime.datetime(date.year, date.month, 1) ds = datetime.datetime(date.year, date.month, 1)

View File

@ -240,7 +240,10 @@ def handler_attribute(zmq_name, jsonobj, hasAlreadyBeenContributed=False, parent
live_helper.publish_log(zmq_name, attributeType, jsonobj) live_helper.publish_log(zmq_name, attributeType, jsonobj)
def handler_diagnostic_tool(zmq_name, jsonobj): def handler_diagnostic_tool(zmq_name, jsonobj):
res = time.time() - float(jsonobj['content']) try:
res = time.time() - float(jsonobj['content'])
except Exception as e:
logger.error(e)
serv_list.set('diagnostic_tool_response', str(res)) serv_list.set('diagnostic_tool_response', str(res))
############### ###############
@ -263,7 +266,8 @@ def main(sleeptime):
while True: while True:
content = serv_list.rpop(LISTNAME) content = serv_list.rpop(LISTNAME)
if content is None: if content is None:
logger.info('Processed {} message(s) since last sleep.'.format(numMsg)) log_text = 'Processed {} message(s) since last sleep.'.format(numMsg)
logger.info(log_text)
numMsg = 0 numMsg = 0
time.sleep(sleeptime) time.sleep(sleeptime)
continue continue