MISP
/
misp-dashboard
mirror of https://github.com/MISP/misp-dashboard
2
0
Fork 0
Code Issues Releases Wiki Activity

added latestAwards + started trophy ranks

pull/7/head
Sami Mokaddem 2017-11-14 09:50:20 +01:00
parent 70561357c3
commit c13e39d5a6
6 changed files with 200 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
config/ranking.cfg
View File

@ -69,6 +69,9 @@ regularlyDays=7
4=Has published valuable content for the community
5=Has published loads of valuable content for the community
[TrophyDifficulty]
difficulty=1.5
[HonorTrophy]
0=No trophy
1=Novice

71
contributor_helper.py
View File

@ -22,6 +22,7 @@ class Contributor_helper:
for badgeNum in range(1, self.honorBadgeNum+1): #get Num of honorBadge
self.org_honor_badge_title[badgeNum] = self.cfg_org_rank.get('HonorBadge', str(badgeNum))
self.trophyDifficulty = self.cfg_org_rank.getfloat('TrophyDifficulty', 'difficulty')
self.trophyNum = len(self.cfg_org_rank.options('HonorTrophyCateg'))
self.categories_in_trophy = json.loads(self.cfg_org_rank.get('HonorTrophyCateg', 'categ'))
self.trophy_title = {}
@ -131,7 +132,11 @@ class Contributor_helper:
to_ret[i] = -1
return {'rank': final_rank, 'status': to_ret, 'totPoints': self.getOrgContributionTotalPoints(org)}
# return the awards given to the organisation
def updateOrgContributionRank(self, orgName, pnts_to_add, action, contribType, eventTime, isLabeled):
ContributionStatus = chelper.getCurrentContributionStatus(org)
oldRank = ContributionStatus['final_rank']
oldContributionStatus = ContributionStatus['status']
keyname = 'CONTRIB_ORG:{org}:{orgCateg}'
# update total points
totOrgPnts = self.serv_redis_db.incrby(keyname.format(org=orgName, orgCateg='points'), pnts_to_add)
@ -167,8 +172,6 @@ class Contributor_helper:
regularlyDays = self.regularlyDays
isRecent = (datetime.datetime.now() - eventTime).days > recentDays
print("contribType: {}, action: {}".format(contribType, action))
print("isLabeled: {}, isRecent: {}, totOrgPnts".format(isLabeled, isRecent, totOrgPnts))
#update contribution Requirement
contrib = [] #[[contrib_level, contrib_ttl], [], ...]
if totOrgPnts >= self.org_rank_requirement_pnts[1] and contribType == 'Sighting':
@ -206,6 +209,19 @@ class Contributor_helper:
self.serv_redis_db.set(keyname.format(org=orgName, orgCateg='CONTRIB_REQ_'+str(rankReq)), 1)
self.serv_redis_db.expire(keyname.format(org=orgName, orgCateg='CONTRIB_REQ_'+str(rankReq)), ttl)
ContributionStatus = chelper.getCurrentContributionStatus(org)
newRank = ContributionStatus['final_rank']
newContributionStatus = ContributionStatus['status']
awards_given = []
if newRank > oldRank:
awards_given.append(['rank', newRank])
for i in range(len(oldContributionStatus)):
if oldContributionStatus[i] != newContributionStatus[i]:
awards_given.append(['contribution_status', i])
print(awards_given)
return awards_given
''' HONOR BADGES '''
def getOrgHonorBadges(self, org):
keyname = 'CONTRIB_ORG:{org}:{orgCateg}'
@ -226,15 +242,48 @@ class Contributor_helper:
''' TROPHIES '''
def getOrgTrophies(self, org):
keyname = 'CONTRIB_ORG:{org}:{orgCateg}'
keyname = 'CONTRIB_TROPHY:{org}:{orgCateg}'
trophy = []
for i in range(1, self.trophyNum+1):
key = keyname.format(org=org, orgCateg='TROPHY_'+str(i))
for categ in self.categories_in_trophy:
key = keyname.format(org=org, orgCateg=categ)
trophy_Pnts = self.serv_redis_db.get(key)
if trophy_Pnts is not None: #existing
trophy.append(trophy_Pnts)
trophy_Pnts = float(trophy_Pnts.decode('utf8'))
trophy_rank = self.getRankTrophy(trophy_Pnts)
trophy_true_rank = self.getTrueRankTrophy(trophy_Pnts)
trophy.append({ 'categ': categ, 'trophy_points': trophy_Pnts, 'trophy_rank': trophy_rank, 'trophy_true_rank': trophy_true_rank, 'trophy_title': self.trophy_title[trophy_true_rank]})
return trophy
def giveTrophyPointsToOrg(self, org, categ, points):
keyname = 'CONTRIB_TROPHY:{org}:{orgCateg}'
self.serv_redis_db.incrby(keyname.format(org=org, orgCateg=categ), points)
def removeTrophyPointsFromOrg(self, org, categ, points):
keyname = 'CONTRIB_TROPHY:{org}:{orgCateg}'
self.serv_redis_db.incrby(keyname.format(org=org, orgCateg=categ), -points)
''' AWARDS HELPER '''
def getLastAwardsFromRedis(self):
date = datetime.datetime.now()
keyname = "CONTRIB_LAST_AWARDS"
prev_days = 7
topNum = self.MAX_NUMBER_OF_LAST_CONTRIBUTOR # default Num
addedOrg = []
data = []
for curDate in util.getXPrevDaysSpan(date, prev_days):
last_awards = self.getZrange(keyname, curDate, topNum)
for dico_award, sec in last_awards:
dico_award = json.loads(dico_award)
org = dico_award['org']
dic = {}
dic['orgRank'] = self.getOrgContributionRank(org)['final_rank']
dic['logo_path'] = self.getOrgLogoFromMISP(org)
dic['org'] = org
dic['epoch'] = sec
dic['award'] = dico_award['award']
data.append(dic)
return data
''' MONTHLY CONTRIBUTION '''
def getOrgPntFromRedis(self, org, date):
keyCateg = 'CONTRIB_DAY'
@ -398,6 +447,16 @@ class Contributor_helper:
prev = i
return { 'remainingPts': 0, 'stepPts': self.rankMultiplier**self.levelMax }
def getRankTrophy(self, points):
if points == 0:
return 0
elif points == 1:
return 1
else:
return float("{:.2f}".format(math.log(points, self.trophyDifficulty)))
def getTrueRankTrophy(self, ptns):
return int(self.getRankTrophy(ptns))
''' '''
''' TEST DATA '''

102
give_honors_to_org.py
View File

@ -1,27 +1,41 @@
#!/usr/bin/env python3.5
import os, sys
import os, sys, json
import datetime, time
import redis
import configparser
import util
import contributor_helper
ONE_DAY = 60*60*24
configfile = os.path.join(os.environ['DASH_CONFIG'], 'config.cfg')
cfg = configparser.ConfigParser()
cfg.read(configfile)
serv_log = redis.StrictRedis(
host=cfg.get('RedisGlobal', 'host'),
port=cfg.getint('RedisGlobal', 'port'),
db=cfg.getint('RedisLog', 'db'))
serv_redis_db = redis.StrictRedis(
host=cfg.get('RedisGlobal', 'host'),
port=cfg.getint('RedisGlobal', 'port'),
db=cfg.getint('RedisDB', 'db'))
CHANNEL_LASTAWARDS = cfg.get('RedisLog', 'channelLastAwards')
chelper = contributor_helper.Contributor_helper(serv_redis_db, cfg)
def publish_log(zmq_name, name, content, channel):
to_send = { 'name': name, 'data': json.dumps(content), 'zmqName': zmq_name }
serv_log.publish(channel, json.dumps(to_send))
def printOrgInfo(org):
org_pnts = chelper.getOrgContributionTotalPoints(org)
org_c_rank = chelper.getOrgContributionRank(org)
org_c_status = chelper.getCurrentContributionStatus(org)
org_honor_badge = chelper.getOrgHonorBadges(org)
org_trophy = chelper.getOrgTrophies(org)
os.system('clear')
print()
print("Organisation points: {}".format(org_pnts))
print("Organisation contribution rank: {}".format(org_c_status['rank']))
@ -39,23 +53,37 @@ Organisation honor badges:
for badgeNum, text in chelper.org_honor_badge_title.items():
acq = 'x' if badgeNum in org_honor_badge else ' '
print("{}.\t[{}]\t{}".format(badgeNum, acq, text))
print()
print()
print('''
Organisation trophy:
--------------------------''')
for dic in org_trophy:
categ = dic['categ']
trophyRank = dic['trophy_true_rank']
trophyPnts = dic['trophy_points']
print("{}\t{} [{}]".format(categ, trophyRank, trophyPnts))
print()
def main():
if len(sys.argv) > 1:
org = sys.argv[1]
else:
org = input('Enter the organisation name: ')
printOrgInfo(org)
ContributionStatus = chelper.getCurrentContributionStatus(org)
OLD_org_c_status = ContributionStatus['status']
OLD_org_honor_badge = chelper.getOrgHonorBadges(org)
# ranks
while True:
org_pnts = chelper.getOrgContributionTotalPoints(org)
org_c_rank = chelper.getOrgContributionRank(org)
org_c_status = chelper.getCurrentContributionStatus(org)
org_honor_badge = chelper.getOrgHonorBadges(org)
org_trophy = chelper.getOrgTrophies(org)
userRep = input("Enter the organisation RANK to give/remove to {} (<ENTER> to finish): ".format(org))
if userRep == '':
@ -67,7 +95,7 @@ def main():
except:
print('Not an integer')
continue
if rankNum < 1 and rankNum > chelper.org_rank_maxLevel:
if rankNum < 1 or rankNum > chelper.org_rank_maxLevel:
print('Not a valid rank')
continue
@ -77,13 +105,14 @@ def main():
chelper.giveContribRankToOrg(org, rankNum)
printOrgInfo(org)
# badges
while True:
org_pnts = chelper.getOrgContributionTotalPoints(org)
org_c_rank = chelper.getOrgContributionRank(org)
org_c_status = chelper.getCurrentContributionStatus(org)
org_honor_badge = chelper.getOrgHonorBadges(org)
org_trophy = chelper.getOrgTrophies(org)
userRep = input("Enter the organisation BADGE to give/remove to {} (<ENTER> to finish): ".format(org))
if userRep == '':
@ -106,6 +135,67 @@ def main():
printOrgInfo(org)
# trophy
while True:
org_pnts = chelper.getOrgContributionTotalPoints(org)
org_c_rank = chelper.getOrgContributionRank(org)
org_c_status = chelper.getCurrentContributionStatus(org)
org_honor_badge = chelper.getOrgHonorBadges(org)
org_trophy = chelper.getOrgTrophies(org)
print()
for i, categ in enumerate(chelper.categories_in_trophy):
print("{}. {}".format(i, categ))
userCateg = input("Enter the CATEGORY in which to add/remove trophy points: ")
if userCateg == '':
break
try: #not int
userCateg = int(userCateg)
except:
print('Not an integer')
continue
if userCateg < 1 and userCateg > len(chelper.categories_in_trophy):
print('Not a valid rank')
continue
categ = chelper.categories_in_trophy[userCateg]
userRep = input("Enter the TROPHY POINTS to give/remove to {} (<ENTER> to finish) in {}: ".format(org, categ))
if userRep == '':
break
else:
# validate input
try: #not int
trophyPnts = int(userRep)
except:
print('Not an integer')
continue
chelper.giveTrophyPointsToOrg(org, categ, trophyPnts)
printOrgInfo(org)
now = datetime.datetime.now()
nowSec = int(time.time())
ContributionStatus = chelper.getCurrentContributionStatus(org)
NEW_org_c_status = ContributionStatus['status']
NEW_org_honor_badge = chelper.getOrgHonorBadges(org)
awards_given = []
for i in NEW_org_c_status.keys():
if OLD_org_c_status[i] < NEW_org_c_status[i] and i != ContributionStatus['rank']:
awards_given.append(['contribution_status', i])
for badgeNum in NEW_org_honor_badge:
if badgeNum not in OLD_org_honor_badge:
awards_given.append(['badge', badgeNum])
for award in awards_given:
# update awards given
serv_redis_db.zadd('CONTRIB_LAST_AWARDS:'+util.getDateStrFormat(now), nowSec, json.dumps({'org': org, 'award': award, 'epoch': nowSec }))
serv_redis_db.expire('CONTRIB_LAST_AWARDS:'+util.getDateStrFormat(now), ONE_DAY*7) #expire after 7 day
# publish
publish_log('GIVE_HONOR_ZMQ', 'CONTRIBUTION', {'org': org, 'award': award, 'epoch': nowSec }, CHANNEL_LASTAWARDS)
if __name__ == '__main__':
main()

12
server.py
View File

@ -322,11 +322,12 @@ def eventStreamAwards():
for msg in subscriber_lastAwards.listen():
content = msg['data'].decode('utf8')
contentJson = json.loads(content)
lastContribJson = json.loads(contentJson['log'])
org = lastContribJson['org']
data = json.loads(contentJson['data'])
org = data['org']
to_return = contributor_helper.getContributorFromRedis(org)
epoch = lastContribJson['epoch']
epoch = data['epoch']
to_return['epoch'] = epoch
to_return['award'] = data['award']
yield 'data: {}\n\n'.format(json.dumps(to_return))
@app.route("/_getTopContributor")
@ -381,8 +382,7 @@ def getLatestAwards():
except:
date = datetime.datetime.now()
return getLastContributors()
#return jsonify(contributor_helper.getCategPerContribFromRedis(date))
return jsonify(contributor_helper.getLastAwardsFromRedis())
@app.route("/_getAllOrg")
def getAllOrg():
@ -418,7 +418,7 @@ def getTrophies():
org = request.args.get('org')
except:
org = ''
return jsonify(contributor_helper.TEST_getOrgTrophies(org))
return jsonify(contributor_helper.getOrgTrophies(org))
if __name__ == '__main__':
app.run(host='localhost', port=8001, threaded=True)

42
static/js/contrib.js
View File

@ -97,17 +97,14 @@ var optionDatatable_Categ = {
{ className: "centerCellPicOrgLogo", "targets": [ 4 ]}
]
};
var optionDatatable_awards = jQuery.extend({}, optionDatatable_light)
var optionDatatable_awards = jQuery.extend({}, optionDatatable_light);
optionDatatable_awards["ordering"] = true;
optionDatatable_awards["order"] = [[ 0, "dec" ]];
optionDatatable_awards.columnDefs = [
{ className: "small", "targets": [ 0 ] },
{ className: "centerCellPicOrgLogo", "targets": [ 1 ] },
{ className: "centerCellPicOrgLogo verticalAlign", "targets": [ 2 ] },
{ 'orderData':[1], 'targets': [0] },
{
'targets': [1],
'searchable': false
},
]
{ className: "centerCellPicOrgLogo verticalAlign", "targets": [ 2 ] }
];
var typeaheadOption = {
source: function (query, process) {
@ -339,19 +336,28 @@ function addLastContributor(datatable, data, update) {
datatable.rows().every( function() {
if($(this.data()[6])[0].text == data.org) {
datatable.row( this ).data( to_add );
$(this).addClass( "warning" );
}
});
}
}
function addAwards(datatableAwards, json) {
if(json.award[0] == 'contribution_status') {
var award = getOrgRankIcon(json.award[1], 60)
} else if (json.award[0] == 'badge') {
var award = createHonorImg([json.award[1]], 20)
} else if (json.award[0] == 'trophy') {
var award = createHonorImg(json.award[1], 20)
}
var date = new Date(json.epoch*1000);
date.toString = function() {return this.toTimeString().slice(0,-15) +' '+ this.toLocaleDateString(); };
var to_add = [
date.toTimeString().slice(0,-15) +' '+ date.toLocaleDateString(),
date,
getOrgRankIcon(json.orgRank, 60),
createImg(json.logo_path, 32),
createOrgLink(json.org),
createHonorImg(json.honorBadge, 20),
award,
];
datatableAwards.row.add(to_add);
}
@ -488,10 +494,12 @@ function updateProgressHeader(org) {
categ = trophy_categ_list[i];
$('#trophy_'+categ).attr('src', source);
$('#trophy_'+categ).attr('title', "");
$('#trophy_'+categ).popover("destroy")
}
for(var i=0; i<data.length; i++) { // add
categ = data[i].categName;
rank = data[i].rank;
categ = data[i].categ;
rank = data[i].trophy_true_rank;
trophy_points = data[i].trophy_points
source = url_baseTrophyLogo+rank+'.png'
$('#trophy_'+categ).attr('src', source);
$('#trophy_'+categ).attr('title', trophy_title[rank]);
@ -575,15 +583,7 @@ $(document).ready(function() {
// latest awards
$.getJSON( url_getLatestAwards, function( data ) {
for (i in data) {
var date = new Date(data[i].epoch*1000);
var to_add = [
date.toTimeString().slice(0,-15) +' '+ date.toLocaleDateString(),
getOrgRankIcon(data[i].orgRank, 60),
createImg(data[i].logo_path, 32),
createOrgLink(data[i].org),
createHonorImg(data[i].honorBadge, 20),
];
datatableAwards.row.add(to_add);
addAwards(datatableAwards, data[i]);
}
datatableAwards.draw();
});

11
zmq_subscriber.py
View File

@ -25,6 +25,7 @@ ONE_DAY = 60*60*24
ZMQ_URL = cfg.get('RedisGlobal', 'zmq_url')
CHANNEL = cfg.get('RedisLog', 'channel')
CHANNEL_LASTCONTRIB = cfg.get('RedisLog', 'channelLastContributor')
CHANNEL_LASTAWARDS = cfg.get('RedisLog', 'channelLastAwards')
CHANNELDISP = cfg.get('RedisMap', 'channelDisp')
CHANNEL_PROC = cfg.get('RedisMap', 'channelProc')
PATH_TO_DB = cfg.get('RedisMap', 'pathMaxMindDB')
@ -155,10 +156,16 @@ def handleContribution(zmq_name, org, contribType, categ, action, pntMultiplier=
serv_redis_db.sadd('CONTRIB_ALL_ORG', org)
serv_redis_db.zadd('CONTRIB_LAST:'+util.getDateStrFormat(now), nowSec, org)
serv_redis_db.expire('CONTRIB_LAST:'+util.getDateStrFormat(now), ONE_DAY) #expire after 1 day
serv_redis_db.expire('CONTRIB_LAST:'+util.getDateStrFormat(now), ONE_DAY*7) #expire after 7 day
contributor_helper.updateOrgContributionRank(org, pnts_to_add, action, contribType, eventTime=datetime.datetime.now(), isLabeled=isLabeled)
awards_given = contributor_helper.updateOrgContributionRank(org, pnts_to_add, action, contribType, eventTime=datetime.datetime.now(), isLabeled=isLabeled)
for award in awards_given:
# update awards given
serv_redis_db.zadd('CONTRIB_LAST_AWARDS:'+util.getDateStrFormat(now), nowSec, json.dumps({'org': org, 'award': award, 'epoch': nowSec }))
serv_redis_db.expire('CONTRIB_LAST_AWARDS:'+util.getDateStrFormat(now), ONE_DAY*7) #expire after 7 day
# publish
publish_log(zmq_name, 'CONTRIBUTION', {'org': org, 'award': award, 'epoch': nowSec }, channel=CHANNEL_LASTAWARDS)
##############