diff --git a/config/config.cfg.default b/config/config.cfg.default index aaa4ada..8ef8740 100644 --- a/config/config.cfg.default +++ b/config/config.cfg.default @@ -17,7 +17,7 @@ size_dashboard_left_width = 5 size_openStreet_pannel_perc = 55 size_world_pannel_perc = 35 item_to_plot = Attribute.category -fieldname_order=["Event.id", "Attribute.Tag", "Attribute.category", "Attribute.type", ["Attribute.value", "Attribute.comment"]] +fieldname_order=["Attribute.timestamp", "Event.id", "Attribute.Tag", "Attribute.category", "Attribute.type", ["Attribute.value", "Attribute.comment"]] char_separator=|| [GEO] diff --git a/give_honors_to_org.py b/give_honors_to_org.py index 9279e64..b482049 100755 --- a/give_honors_to_org.py +++ b/give_honors_to_org.py @@ -210,7 +210,7 @@ def main(): for award in awards_given: # update awards given - serv_redis_db.zadd('CONTRIB_LAST_AWARDS:'+util.getDateStrFormat(now), nowSec, json.dumps({'org': org, 'award': award, 'epoch': nowSec })) + serv_redis_db.zadd('CONTRIB_LAST_AWARDS:'+util.getDateStrFormat(now), {json.dumps({'org': org, 'award': award, 'epoch': nowSec }): nowSec}) serv_redis_db.expire('CONTRIB_LAST_AWARDS:'+util.getDateStrFormat(now), ONE_DAY*7) #expire after 7 day # publish publish_log('GIVE_HONOR_ZMQ', 'CONTRIBUTION', {'org': org, 'award': award, 'epoch': nowSec }, CHANNEL_LASTAWARDS) diff --git a/helpers/contributor_helper.py b/helpers/contributor_helper.py index bc4813b..6eb9582 100644 --- a/helpers/contributor_helper.py +++ b/helpers/contributor_helper.py @@ -115,7 +115,7 @@ class Contributor_helper: def addContributionToCateg(self, date, categ, org, count=1): today_str = util.getDateStrFormat(date) keyname = "{}:{}:{}".format(self.keyCateg, today_str, categ) - self.serv_redis_db.zincrby(keyname, org, count) + self.serv_redis_db.zincrby(keyname, count, org) self.logger.debug('Added to redis: keyname={}, org={}, count={}'.format(keyname, org, count)) def publish_log(self, zmq_name, name, content, channel=""): @@ -155,7 +155,7 @@ class Contributor_helper: self.serv_redis_db.sadd(self.keyAllOrg, org) keyname = "{}:{}".format(self.keyLastContrib, util.getDateStrFormat(now)) - self.serv_redis_db.zadd(keyname, nowSec, org) + self.serv_redis_db.zadd(keyname, {org: nowSec}) self.logger.debug('Added to redis: keyname={}, nowSec={}, org={}'.format(keyname, nowSec, org)) self.serv_redis_db.expire(keyname, util.ONE_DAY*7) #expire after 7 day @@ -164,7 +164,7 @@ class Contributor_helper: for award in awards_given: # update awards given keyname = "{}:{}".format(self.keyLastAward, util.getDateStrFormat(now)) - self.serv_redis_db.zadd(keyname, nowSec, json.dumps({'org': org, 'award': award, 'epoch': nowSec })) + self.serv_redis_db.zadd(keyname, {json.dumps({'org': org, 'award': award, 'epoch': nowSec }): nowSec}) self.logger.debug('Added to redis: keyname={}, nowSec={}, content={}'.format(keyname, nowSec, json.dumps({'org': org, 'award': award, 'epoch': nowSec }))) self.serv_redis_db.expire(keyname, util.ONE_DAY*7) #expire after 7 day # publish @@ -410,12 +410,12 @@ class Contributor_helper: def giveTrophyPointsToOrg(self, org, categ, points): keyname = '{mainKey}:{orgCateg}' - self.serv_redis_db.zincrby(keyname.format(mainKey=self.keyTrophy, orgCateg=categ), org, points) + self.serv_redis_db.zincrby(keyname.format(mainKey=self.keyTrophy, orgCateg=categ), points, org) self.logger.debug('Giving {} trophy points to {} in {}'.format(points, org, categ)) def removeTrophyPointsFromOrg(self, org, categ, points): keyname = '{mainKey}:{orgCateg}' - self.serv_redis_db.zincrby(keyname.format(mainKey=self.keyTrophy, orgCateg=categ), org, -points) + self.serv_redis_db.zincrby(keyname.format(mainKey=self.keyTrophy, orgCateg=categ), -points, org) self.logger.debug('Removing {} trophy points from {} in {}'.format(points, org, categ)) ''' AWARDS HELPER ''' diff --git a/helpers/geo_helper.py b/helpers/geo_helper.py index ededda2..a61d4c4 100644 --- a/helpers/geo_helper.py +++ b/helpers/geo_helper.py @@ -206,7 +206,7 @@ class Geo_helper: now = datetime.datetime.now() today_str = util.getDateStrFormat(now) keyname = "{}:{}{}".format(keyCateg, today_str, endSubkey) - self.serv_redis_db.zincrby(keyname, toAdd, count) + self.serv_redis_db.zincrby(keyname, count, toAdd) self.logger.debug('Added to redis: keyname={}, toAdd={}, count={}'.format(keyname, toAdd, count)) def ip_to_coord(self, ip): diff --git a/helpers/trendings_helper.py b/helpers/trendings_helper.py index 4b55423..6c26a37 100644 --- a/helpers/trendings_helper.py +++ b/helpers/trendings_helper.py @@ -49,7 +49,7 @@ class Trendings_helper: to_save = json.dumps(data) else: to_save = data - self.serv_redis_db.zincrby(keyname, to_save, 1) + self.serv_redis_db.zincrby(keyname, 1, to_save) self.logger.debug('Added to redis: keyname={}, content={}'.format(keyname, to_save)) def addTrendingEvent(self, eventName, timestamp): diff --git a/helpers/users_helper.py b/helpers/users_helper.py index ba3ab22..a2726e9 100644 --- a/helpers/users_helper.py +++ b/helpers/users_helper.py @@ -41,11 +41,11 @@ class Users_helper: timestampDate_str = util.getDateStrFormat(timestampDate) keyname_timestamp = "{}:{}".format(self.keyTimestamp, org) - self.serv_redis_db.zadd(keyname_timestamp, timestamp, timestamp) + self.serv_redis_db.zadd(keyname_timestamp, {timestamp: timestamp}) self.logger.debug('Added to redis: keyname={}, org={}'.format(keyname_timestamp, timestamp)) keyname_org = "{}:{}".format(self.keyOrgLog, timestampDate_str) - self.serv_redis_db.zincrby(keyname_org, org, 1) + self.serv_redis_db.zincrby(keyname_org, 1, org) self.logger.debug('Added to redis: keyname={}, org={}'.format(keyname_org, org)) self.serv_redis_db.sadd(self.keyAllOrgLog, org) diff --git a/requirements.txt b/requirements.txt index 8970bf3..3cb0733 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,7 @@ argparse flask geoip2 -# Redis needs to be 2.10.6 due to a change in redis 3.10 client, see here: https://github.com/MISP/misp-dashboard/issues/76#issuecomment-439389621 -redis==2.10.6 +redis phonenumbers pip pycountry diff --git a/server.py b/server.py index 3ee4344..41ff751 100755 --- a/server.py +++ b/server.py @@ -95,13 +95,6 @@ class LogItem(): else: to_add = util.getFields(self.feed, field) to_ret[i] = to_add if to_add is not None else '' - - # Number to keep them sorted (jsonify sort keys) - for item in range(len(LogItem.FIELDNAME_ORDER)): - try: - to_ret[item] = self.fields[item] - except IndexError: # not enough field in rcv item - to_ret[item] = '' return to_ret diff --git a/util.py b/util.py index 8e0661f..7e6e011 100644 --- a/util.py +++ b/util.py @@ -18,7 +18,7 @@ def push_to_redis_zset(serv_redis_db, mainKey, toAdd, endSubkey="", count=1): now = datetime.datetime.now() today_str = getDateStrFormat(now) keyname = "{}:{}{}".format(mainKey, today_str, endSubkey) - serv_redis_db.zincrby(keyname, toAdd, count) + serv_redis_db.zincrby(keyname, count, toAdd) def getMonthSpan(date): ds = datetime.datetime(date.year, date.month, 1) diff --git a/zmq_dispatcher.py b/zmq_dispatcher.py index 9d77101..13492ad 100755 --- a/zmq_dispatcher.py +++ b/zmq_dispatcher.py @@ -240,7 +240,10 @@ def handler_attribute(zmq_name, jsonobj, hasAlreadyBeenContributed=False, parent live_helper.publish_log(zmq_name, attributeType, jsonobj) def handler_diagnostic_tool(zmq_name, jsonobj): - res = time.time() - float(jsonobj['content']) + try: + res = time.time() - float(jsonobj['content']) + except Exception as e: + logger.error(e) serv_list.set('diagnostic_tool_response', str(res)) ############### @@ -263,7 +266,8 @@ def main(sleeptime): while True: content = serv_list.rpop(LISTNAME) if content is None: - logger.info('Processed {} message(s) since last sleep.'.format(numMsg)) + log_text = 'Processed {} message(s) since last sleep.'.format(numMsg) + logger.info(log_text) numMsg = 0 time.sleep(sleeptime) continue