fix: [all] Fixed issue with py-redis>2.x and fix failed merge conflict

config/config.cfg.default

@@ -17,7 +17,7 @@ size_dashboard_left_width = 5
 size_openStreet_pannel_perc = 55
 size_world_pannel_perc = 35
 item_to_plot = Attribute.category
-fieldname_order=["Event.id", "Attribute.Tag", "Attribute.category", "Attribute.type", ["Attribute.value", "Attribute.comment"]]
+fieldname_order=["Attribute.timestamp", "Event.id", "Attribute.Tag", "Attribute.category", "Attribute.type",  ["Attribute.value", "Attribute.comment"]]
 char_separator=||
 
 [GEO]

give_honors_to_org.py

@@ -210,7 +210,7 @@ def main():
 
     for award in awards_given:
         # update awards given
-        serv_redis_db.zadd('CONTRIB_LAST_AWARDS:'+util.getDateStrFormat(now), nowSec, json.dumps({'org': org, 'award': award, 'epoch': nowSec }))
+        serv_redis_db.zadd('CONTRIB_LAST_AWARDS:'+util.getDateStrFormat(now), {json.dumps({'org': org, 'award': award, 'epoch': nowSec }): nowSec})
         serv_redis_db.expire('CONTRIB_LAST_AWARDS:'+util.getDateStrFormat(now), ONE_DAY*7) #expire after 7 day
         # publish
         publish_log('GIVE_HONOR_ZMQ', 'CONTRIBUTION', {'org': org, 'award': award, 'epoch': nowSec }, CHANNEL_LASTAWARDS)

helpers/contributor_helper.py

@@ -115,7 +115,7 @@ class Contributor_helper:
     def addContributionToCateg(self, date, categ, org, count=1):
         today_str = util.getDateStrFormat(date)
         keyname = "{}:{}:{}".format(self.keyCateg, today_str, categ)
-        self.serv_redis_db.zincrby(keyname, org, count)
+        self.serv_redis_db.zincrby(keyname, count, org)
         self.logger.debug('Added to redis: keyname={}, org={}, count={}'.format(keyname, org, count))
 
     def publish_log(self, zmq_name, name, content, channel=""):
@@ -155,7 +155,7 @@ class Contributor_helper:
         self.serv_redis_db.sadd(self.keyAllOrg, org)
 
         keyname = "{}:{}".format(self.keyLastContrib, util.getDateStrFormat(now))
-        self.serv_redis_db.zadd(keyname, nowSec, org)
+        self.serv_redis_db.zadd(keyname, {org: nowSec})
         self.logger.debug('Added to redis: keyname={}, nowSec={}, org={}'.format(keyname, nowSec, org))
         self.serv_redis_db.expire(keyname, util.ONE_DAY*7) #expire after 7 day
 
@@ -164,7 +164,7 @@ class Contributor_helper:
         for award in awards_given:
             # update awards given
             keyname = "{}:{}".format(self.keyLastAward, util.getDateStrFormat(now))
-            self.serv_redis_db.zadd(keyname, nowSec, json.dumps({'org': org, 'award': award, 'epoch': nowSec }))
+            self.serv_redis_db.zadd(keyname, {json.dumps({'org': org, 'award': award, 'epoch': nowSec }): nowSec})
             self.logger.debug('Added to redis: keyname={}, nowSec={}, content={}'.format(keyname, nowSec, json.dumps({'org': org, 'award': award, 'epoch': nowSec })))
             self.serv_redis_db.expire(keyname, util.ONE_DAY*7) #expire after 7 day
             # publish
@@ -410,12 +410,12 @@ class Contributor_helper:
 
     def giveTrophyPointsToOrg(self, org, categ, points):
         keyname = '{mainKey}:{orgCateg}'
-        self.serv_redis_db.zincrby(keyname.format(mainKey=self.keyTrophy, orgCateg=categ), org, points)
+        self.serv_redis_db.zincrby(keyname.format(mainKey=self.keyTrophy, orgCateg=categ), points, org)
         self.logger.debug('Giving {} trophy points to {} in {}'.format(points, org, categ))
 
     def removeTrophyPointsFromOrg(self, org, categ, points):
         keyname = '{mainKey}:{orgCateg}'
-        self.serv_redis_db.zincrby(keyname.format(mainKey=self.keyTrophy, orgCateg=categ), org, -points)
+        self.serv_redis_db.zincrby(keyname.format(mainKey=self.keyTrophy, orgCateg=categ), -points, org)
         self.logger.debug('Removing {} trophy points from {} in {}'.format(points, org, categ))
 
     ''' AWARDS HELPER '''

helpers/geo_helper.py

@@ -206,7 +206,7 @@ class Geo_helper:
         now = datetime.datetime.now()
         today_str = util.getDateStrFormat(now)
         keyname = "{}:{}{}".format(keyCateg, today_str, endSubkey)
-        self.serv_redis_db.zincrby(keyname, toAdd, count)
+        self.serv_redis_db.zincrby(keyname, count, toAdd)
         self.logger.debug('Added to redis: keyname={}, toAdd={}, count={}'.format(keyname, toAdd, count))
 
     def ip_to_coord(self, ip):

helpers/trendings_helper.py

@@ -49,7 +49,7 @@ class Trendings_helper:
             to_save = json.dumps(data)
         else:
             to_save = data
-        self.serv_redis_db.zincrby(keyname, to_save, 1)
+        self.serv_redis_db.zincrby(keyname, 1, to_save)
         self.logger.debug('Added to redis: keyname={}, content={}'.format(keyname, to_save))
 
     def addTrendingEvent(self, eventName, timestamp):

helpers/users_helper.py

@@ -41,11 +41,11 @@ class Users_helper:
         timestampDate_str = util.getDateStrFormat(timestampDate)
 
         keyname_timestamp = "{}:{}".format(self.keyTimestamp, org)
-        self.serv_redis_db.zadd(keyname_timestamp, timestamp, timestamp)
+        self.serv_redis_db.zadd(keyname_timestamp, {timestamp: timestamp})
         self.logger.debug('Added to redis: keyname={}, org={}'.format(keyname_timestamp, timestamp))
 
         keyname_org = "{}:{}".format(self.keyOrgLog, timestampDate_str)
-        self.serv_redis_db.zincrby(keyname_org, org, 1)
+        self.serv_redis_db.zincrby(keyname_org, 1, org)
         self.logger.debug('Added to redis: keyname={}, org={}'.format(keyname_org, org))
 
         self.serv_redis_db.sadd(self.keyAllOrgLog, org)

requirements.txt

@@ -1,8 +1,7 @@
 argparse
 flask
 geoip2
-# Redis needs to be 2.10.6 due to a change in redis 3.10 client, see here: https://github.com/MISP/misp-dashboard/issues/76#issuecomment-439389621
+redis
-redis==2.10.6
 phonenumbers
 pip
 pycountry

server.py

@@ -95,13 +95,6 @@ class LogItem():
             else:
                 to_add = util.getFields(self.feed, field)
             to_ret[i] = to_add if to_add is not None else ''
-
-        # Number to keep them sorted (jsonify sort keys)
-        for item in range(len(LogItem.FIELDNAME_ORDER)):
-            try:
-                to_ret[item] = self.fields[item]
-            except IndexError: # not enough field in rcv item
-                to_ret[item] = ''
         return to_ret
 
 

util.py

@@ -18,7 +18,7 @@ def push_to_redis_zset(serv_redis_db, mainKey, toAdd, endSubkey="", count=1):
     now = datetime.datetime.now()
     today_str = getDateStrFormat(now)
     keyname = "{}:{}{}".format(mainKey, today_str, endSubkey)
-    serv_redis_db.zincrby(keyname, toAdd, count)
+    serv_redis_db.zincrby(keyname, count, toAdd)
 
 def getMonthSpan(date):
     ds = datetime.datetime(date.year, date.month, 1)

zmq_dispatcher.py

@@ -240,7 +240,10 @@ def handler_attribute(zmq_name, jsonobj, hasAlreadyBeenContributed=False, parent
     live_helper.publish_log(zmq_name, attributeType, jsonobj)
 
 def handler_diagnostic_tool(zmq_name, jsonobj):
-    res = time.time() - float(jsonobj['content'])
+    try:
+        res = time.time() - float(jsonobj['content'])
+    except Exception as e:
+        logger.error(e)
     serv_list.set('diagnostic_tool_response', str(res))
 
 ###############
@@ -263,7 +266,8 @@ def main(sleeptime):
     while True:
         content = serv_list.rpop(LISTNAME)
         if content is None:
-            logger.info('Processed {} message(s) since last sleep.'.format(numMsg))
+            log_text = 'Processed {} message(s) since last sleep.'.format(numMsg)
+            logger.info(log_text)
             numMsg = 0
             time.sleep(sleeptime)
             continue