From 0e5b03fee6ee058e9a30e43e70d95c00556a68fd Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Fri, 3 Nov 2017 09:32:07 +0100 Subject: [PATCH] Draft of adding contribution info to redis --- config/config.cfg.default | 8 +++++-- server.py | 15 ++++++++----- zmq_subscriber.py | 47 +++++++++++++++++++++++++++++++++------ 3 files changed, 56 insertions(+), 14 deletions(-) diff --git a/config/config.cfg.default b/config/config.cfg.default index 988cc7b..4a568fb 100644 --- a/config/config.cfg.default +++ b/config/config.cfg.default @@ -20,9 +20,13 @@ zoomlevel = 11 clusteringDistance = 10 [CONTRIB] -#[1.5 -> +inf] +max_number_of_last_contributor = 10 +#How much harder it gets to rank up (exponential multiplier) [1.5 -> +inf] rankMultiplier = 2 -categories = ["internal_reference", "targeting_data", "antivirus_detection", "payload_delivery", "artifacts_dropped", "payload_installation", "persistence_mechanism", "network_activity", "payload_type", "attribution", "external_analysis", "financial_fraud", "support_Tool", "social_network", "person", "other" ] +categories_in_datatable = ["internal_reference", "targeting_data", "antivirus_detection", "payload_delivery", "artifacts_dropped", "payload_installation", "persistence_mechanism", "network_activity", "payload_type", "attribution", "external_analysis", "financial_fraud", "support_Tool", "social_network", "person", "other" ] +default_pnts_per_contribution = 1 +# array of the form [[category, pntsRcv], ...] +pnts_per_contribution = [["payload_delivery", 5], ["artifact_dropped", 20], ["network_activity", 5]] [Log] field_to_plot = Attribute.category diff --git a/server.py b/server.py index 28c880d..842871a 100755 --- a/server.py +++ b/server.py @@ -28,7 +28,7 @@ serv_redis_db = redis.StrictRedis( port=cfg.getint('RedisGlobal', 'port'), db=cfg.getint('RedisDB', 'db')) -categories = json.loads(cfg.get('CONTRIB', 'categories')) +categories_in_datatable = json.loads(cfg.get('CONTRIB', 'categories_in_datatable')) subscriber_log = redis_server_log.pubsub(ignore_subscribe_messages=True) subscriber_log.psubscribe(cfg.get('RedisLog', 'channel')) @@ -145,8 +145,8 @@ def geo(): @app.route("/contrib") def contrib(): - categ_list = json.loads(cfg.get('CONTRIB', 'categories')) - categ_list_str = [ s[0].upper() + s[1:].replace('_', ' ') for s in json.loads(cfg.get('CONTRIB', 'categories'))] + categ_list = categories_in_datatable + categ_list_str = [ s[0].upper() + s[1:].replace('_', ' ') for s in categories_in_datatable] return render_template('contrib.html', currOrg="", rankMultiplier=cfg.getfloat('CONTRIB' ,'rankMultiplier'), @@ -230,7 +230,7 @@ def getCategPerContrib(): dic['rank'] = random.randint(1,16) dic['logo_path'] = 'logo' dic['org'] = 'Org'+str(d) - for f in categories: + for f in categories_in_datatable: dic[f] = random.randint(0,1600) data.append(dic) @@ -250,7 +250,12 @@ def getOrgRank(): points = random.randint(1,math.floor(cfg.getfloat('CONTRIB' ,'rankMultiplier')**16)) #FIXME put 0 if org has no points remainingPts = getRemainingPoints(points) - data = {'org': org, 'points': points, 'rank': getRankLevel(points), 'remainingPts': remainingPts['remainingPts'], 'stepPts': remainingPts['stepPts']} + data = {'org': org, + 'points': points, + 'rank': getRankLevel(points), + 'remainingPts': remainingPts['remainingPts'], + 'stepPts': remainingPts['stepPts'], + } return jsonify(data) @app.route("/_getTopCoord") diff --git a/zmq_subscriber.py b/zmq_subscriber.py index e0f6b99..cd01536 100755 --- a/zmq_subscriber.py +++ b/zmq_subscriber.py @@ -24,6 +24,14 @@ CHANNELDISP = cfg.get('RedisMap', 'channelDisp') CHANNEL_PROC = cfg.get('RedisMap', 'channelProc') PATH_TO_DB = cfg.get('RedisMap', 'pathMaxMindDB') +DEFAULT_PNTS_REWARD = cfg.get('CONTRIB', 'default_pnts_per_contribution') +categories_in_datatable = json.loads(cfg.get('CONTRIB', 'categories_in_datatable')) +DICO_PNTS_REWARD = {} +temp = json.loads(cfg.get('CONTRIB', 'pnts_per_contribution')) +for categ, pnts in temp: + DICO_PNTS_REWARD[categ] = pnts +MAX_NUMBER_OF_LAST_CONTRIBUTOR = cfg.getint('CONTRIB', 'max_number_of_last_contributor') + serv_log = redis.StrictRedis( host=cfg.get('RedisGlobal', 'host'), port=cfg.getint('RedisGlobal', 'port'), @@ -44,19 +52,18 @@ def publish_log(zmq_name, name, content): to_send = { 'name': name, 'log': json.dumps(content), 'zmqName': zmq_name } serv_log.publish(CHANNEL, json.dumps(to_send)) -def push_to_redis_zset(keyCateg, toAdd): +def push_to_redis_zset(keyCateg, toAdd, endSubkey="", count=1): now = datetime.datetime.now() - today_str = str(now.year)+str(now.month)+str(now.day) - keyname = "{}:{}".format(keyCateg, today_str) - serv_redis_db.zincrby(keyname, toAdd) + today_str = str(now.year)+str(now.month).zfill(2)+str(now.day).zfill(2) + keyname = "{}:{}{}".format(keyCateg, today_str, endSubkey) + serv_redis_db.zincrby(keyname, toAdd, count) def push_to_redis_geo(keyCateg, lon, lat, content): now = datetime.datetime.now() - today_str = str(now.year)+str(now.month)+str(now.day) + today_str = str(now.year)+str(now.month).zfill(2)+str(now.day).zfill(2) keyname = "{}:{}".format(keyCateg, today_str) serv_redis_db.geoadd(keyname, lon, lat, content) - def ip_to_coord(ip): resp = reader.city(ip) lat = float(resp.location.latitude) @@ -109,6 +116,28 @@ def getFields(obj, fields): except KeyError as e: return "" +def noSpaceLower(str): + return str.lower().replace(' ', '_') + +#pntMultiplier if one contribution rewards more than others. (e.g. shighting may gives more points than editing) +def handleContribution(org, categ, action, pntMultiplier=1): + if action in ['edit']: + pass + #return #not a contribution? + # is a valid contribution + try: + pnts_to_add = DICO_PNTS_REWARD[noSpaceLower(categ)] + except KeyError: + pnts_to_add = DEFAULT_PNTS_REWARD + pnts_to_add *= pntMultiplier + + push_to_redis_zset('CONTRIB_DAY', org, count=pnts_to_add) + #CONTRIB_CATEG retain the contribution per category, not the point earned in this categ + push_to_redis_zset('CONTRIB_CATEG', org, count=DEFAULT_PNTS_REWARD, endSubkey=':'+noSpaceLower(categ)) + serv_redis_db.sadd('CONTRIB_ALL_ORG', org) + serv_redis_db.lpush('CONTRIB_LAST', org) + serv_redis_db.ltrim('CONTRIB_LAST', 0, MAX_NUMBER_OF_LAST_CONTRIBUTOR-1) # Limit list size + #serv_redis_db.lrange('CONTRIB_LAST', 0, MAX_NUMBER_OF_LAST_CONTRIBUTOR-1) # get the last 10 contributors ############## ## HANDLERS ## @@ -129,6 +158,10 @@ def handler_keepalive(zmq_name, jsonevent): def handler_sighting(zmq_name, jsonsight): print('sending' ,'sighting') + org = jsonsight['org'] + categ = jsonsight['categ'] + action = jsonsight['action'] + handleContribution(org, categ, action) return def handler_event(zmq_name, jsonobj): @@ -165,6 +198,7 @@ def handler_attribute(zmq_name, jsonobj): if jsonattr['category'] == "Network activity": getCoordAndPublish(zmq_name, jsonattr['value'], jsonattr['category']) + handleContribution(jsonobj['Event']['Orgc']['name'], jsonattr['category'], jsonobj['action']) # Push to log publish_log(zmq_name, 'Attribute', to_push) @@ -210,4 +244,3 @@ if __name__ == "__main__": main(args.zmqname) reader.close() -