Draft of adding contribution info to redis

config/config.cfg.default

@@ -20,9 +20,13 @@ zoomlevel = 11
 clusteringDistance = 10
 
 [CONTRIB]
-#[1.5 -> +inf]
+max_number_of_last_contributor = 10
+#How much harder it gets to rank up (exponential multiplier) [1.5 -> +inf]
 rankMultiplier = 2
-categories = ["internal_reference", "targeting_data", "antivirus_detection", "payload_delivery", "artifacts_dropped", "payload_installation", "persistence_mechanism", "network_activity", "payload_type", "attribution", "external_analysis", "financial_fraud", "support_Tool", "social_network", "person", "other" ]
+categories_in_datatable = ["internal_reference", "targeting_data", "antivirus_detection", "payload_delivery", "artifacts_dropped", "payload_installation", "persistence_mechanism", "network_activity", "payload_type", "attribution", "external_analysis", "financial_fraud", "support_Tool", "social_network", "person", "other" ]
+default_pnts_per_contribution = 1
+# array of the form [[category, pntsRcv], ...]
+pnts_per_contribution = [["payload_delivery", 5], ["artifact_dropped", 20], ["network_activity", 5]]
 
 [Log]
 field_to_plot = Attribute.category

server.py

@@ -28,7 +28,7 @@ serv_redis_db = redis.StrictRedis(
         port=cfg.getint('RedisGlobal', 'port'),
         db=cfg.getint('RedisDB', 'db'))
 
-categories = json.loads(cfg.get('CONTRIB', 'categories'))
+categories_in_datatable = json.loads(cfg.get('CONTRIB', 'categories_in_datatable'))
 
 subscriber_log = redis_server_log.pubsub(ignore_subscribe_messages=True)
 subscriber_log.psubscribe(cfg.get('RedisLog', 'channel'))
@@ -145,8 +145,8 @@ def geo():
 
 @app.route("/contrib")
 def contrib():
-    categ_list = json.loads(cfg.get('CONTRIB', 'categories'))
+    categ_list = categories_in_datatable
-    categ_list_str = [ s[0].upper() + s[1:].replace('_', ' ') for s in json.loads(cfg.get('CONTRIB', 'categories'))]
+    categ_list_str = [ s[0].upper() + s[1:].replace('_', ' ') for s in categories_in_datatable]
     return render_template('contrib.html',
             currOrg="",
             rankMultiplier=cfg.getfloat('CONTRIB' ,'rankMultiplier'),
@@ -230,7 +230,7 @@ def getCategPerContrib():
         dic['rank'] = random.randint(1,16)
         dic['logo_path'] = 'logo'
         dic['org'] = 'Org'+str(d)
-        for f in categories:
+        for f in categories_in_datatable:
             dic[f] = random.randint(0,1600)
         data.append(dic)
 
@@ -250,7 +250,12 @@ def getOrgRank():
     points = random.randint(1,math.floor(cfg.getfloat('CONTRIB' ,'rankMultiplier')**16))
     #FIXME put 0 if org has no points
     remainingPts = getRemainingPoints(points)
-    data = {'org': org, 'points': points, 'rank': getRankLevel(points), 'remainingPts': remainingPts['remainingPts'], 'stepPts': remainingPts['stepPts']}
+    data = {'org': org,
+    'points': points,
+    'rank': getRankLevel(points),
+    'remainingPts': remainingPts['remainingPts'],
+    'stepPts': remainingPts['stepPts'],
+    }
     return jsonify(data)
 
 @app.route("/_getTopCoord")

zmq_subscriber.py

@@ -24,6 +24,14 @@ CHANNELDISP = cfg.get('RedisMap', 'channelDisp')
 CHANNEL_PROC = cfg.get('RedisMap', 'channelProc')
 PATH_TO_DB = cfg.get('RedisMap', 'pathMaxMindDB')
 
+DEFAULT_PNTS_REWARD = cfg.get('CONTRIB', 'default_pnts_per_contribution')
+categories_in_datatable = json.loads(cfg.get('CONTRIB', 'categories_in_datatable'))
+DICO_PNTS_REWARD = {}
+temp = json.loads(cfg.get('CONTRIB', 'pnts_per_contribution'))
+for categ, pnts in temp:
+    DICO_PNTS_REWARD[categ] = pnts
+MAX_NUMBER_OF_LAST_CONTRIBUTOR = cfg.getint('CONTRIB', 'max_number_of_last_contributor')
+
 serv_log = redis.StrictRedis(
         host=cfg.get('RedisGlobal', 'host'),
         port=cfg.getint('RedisGlobal', 'port'),
@@ -44,19 +52,18 @@ def publish_log(zmq_name, name, content):
     to_send = { 'name': name, 'log': json.dumps(content), 'zmqName': zmq_name }
     serv_log.publish(CHANNEL, json.dumps(to_send))
 
-def push_to_redis_zset(keyCateg, toAdd):
+def push_to_redis_zset(keyCateg, toAdd, endSubkey="", count=1):
     now = datetime.datetime.now()
-    today_str = str(now.year)+str(now.month)+str(now.day)
+    today_str = str(now.year)+str(now.month).zfill(2)+str(now.day).zfill(2)
-    keyname = "{}:{}".format(keyCateg, today_str)
+    keyname = "{}:{}{}".format(keyCateg, today_str, endSubkey)
-    serv_redis_db.zincrby(keyname, toAdd)
+    serv_redis_db.zincrby(keyname, toAdd, count)
 
 def push_to_redis_geo(keyCateg, lon, lat, content):
     now = datetime.datetime.now()
-    today_str = str(now.year)+str(now.month)+str(now.day)
+    today_str = str(now.year)+str(now.month).zfill(2)+str(now.day).zfill(2)
     keyname = "{}:{}".format(keyCateg, today_str)
     serv_redis_db.geoadd(keyname, lon, lat, content)
 
-
 def ip_to_coord(ip):
     resp = reader.city(ip)
     lat = float(resp.location.latitude)
@@ -109,6 +116,28 @@ def getFields(obj, fields):
     except KeyError as e:
         return ""
 
+def noSpaceLower(str):
+    return str.lower().replace(' ', '_')
+
+#pntMultiplier if one contribution rewards more than others. (e.g. shighting may gives more points than editing)
+def handleContribution(org, categ, action, pntMultiplier=1):
+    if action in ['edit']:
+        pass
+        #return #not a contribution?
+    # is a valid contribution
+    try:
+        pnts_to_add = DICO_PNTS_REWARD[noSpaceLower(categ)]
+    except KeyError:
+        pnts_to_add = DEFAULT_PNTS_REWARD
+    pnts_to_add *= pntMultiplier
+
+    push_to_redis_zset('CONTRIB_DAY', org, count=pnts_to_add)
+    #CONTRIB_CATEG retain the contribution per category, not the point earned in this categ
+    push_to_redis_zset('CONTRIB_CATEG', org, count=DEFAULT_PNTS_REWARD, endSubkey=':'+noSpaceLower(categ))
+    serv_redis_db.sadd('CONTRIB_ALL_ORG', org)
+    serv_redis_db.lpush('CONTRIB_LAST', org)
+    serv_redis_db.ltrim('CONTRIB_LAST', 0, MAX_NUMBER_OF_LAST_CONTRIBUTOR-1) # Limit list size
+    #serv_redis_db.lrange('CONTRIB_LAST', 0, MAX_NUMBER_OF_LAST_CONTRIBUTOR-1) # get the last 10 contributors
 
 ##############
 ## HANDLERS ##
@@ -129,6 +158,10 @@ def handler_keepalive(zmq_name, jsonevent):
 
 def handler_sighting(zmq_name, jsonsight):
     print('sending' ,'sighting')
+    org = jsonsight['org']
+    categ = jsonsight['categ']
+    action = jsonsight['action']
+    handleContribution(org, categ, action)
     return
 
 def handler_event(zmq_name, jsonobj):
@@ -165,6 +198,7 @@ def handler_attribute(zmq_name, jsonobj):
     if jsonattr['category'] == "Network activity":
         getCoordAndPublish(zmq_name, jsonattr['value'], jsonattr['category'])
 
+    handleContribution(jsonobj['Event']['Orgc']['name'], jsonattr['category'], jsonobj['action'])
     # Push to log
     publish_log(zmq_name, 'Attribute', to_push)
 
@@ -210,4 +244,3 @@ if __name__ == "__main__":
 
     main(args.zmqname)
     reader.close()
-