MISP
/
misp-dashboard
mirror of https://github.com/MISP/misp-dashboard
2
0
Fork 0
Code Issues Releases Wiki Activity

fixed bugs if requesting data from non-existing org + refined draft updateOrgContributionRank + Fixed non existing org logo + try in zmq_subscribe

pull/3/head
Sami Mokaddem 2017-11-09 13:43:02 +01:00
parent 139c7dd54f
commit 1550f25b30
6 changed files with 94 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
config/ranking.cfg
View File

@ -32,8 +32,8 @@ maxLevel=16
12=4096 12=4096
13=8192 13=8192
14=16384 14=16384
15=32768 15=0
16=65536 16=0
[rankRequirementsMisc] [rankRequirementsMisc]
heavilyCount=10 heavilyCount=10

91
contributor_helper.py
View File

@ -69,6 +69,15 @@ class Contributor_helper:
return data return data
''' CONTRIBUTION RANK ''' ''' CONTRIBUTION RANK '''
def getOrgContributionTotalPoints(self, org):
keyname = 'CONTRIB_ORG:{org}:{orgCateg}'
pnts = self.serv_redis_db.get(keyname.format(org=org, orgCateg='points'))
if pnts is None:
pnts = 0
else:
pnts = int(pnts.decode('utf8'))
return pnts
# return: [final_rank, requirement_fulfilled, requirement_not_fulfilled] # return: [final_rank, requirement_fulfilled, requirement_not_fulfilled]
def getOrgContributionRank(self, org): def getOrgContributionRank(self, org):
keyname = 'CONTRIB_ORG:{org}:{orgCateg}' keyname = 'CONTRIB_ORG:{org}:{orgCateg}'
@ -108,53 +117,55 @@ class Contributor_helper:
to_ret[i] = 0 to_ret[i] = 0
else: else:
to_ret[i] = -1 to_ret[i] = -1
return to_ret return {'rank': final_rank, 'status': to_ret, 'totPoints': self.getOrgContributionTotalPoints(org)}
def updateOrgContributionRank(self, orgName, pnts_to_add, contribType, eventTime, isLabeled): def updateOrgContributionRank(self, orgName, pnts_to_add, contribType, eventTime, isLabeled):
keyname = 'CONTRIB_ORG:{org}:{orgCateg}' keyname = 'CONTRIB_ORG:{org}:{orgCateg}'
#update total points #update total points
self.serv_redis_db.set(keyname.format(org=orgName, orgCateg='points'), pnts_to_add) totOrgPnts = self.serv_redis_db.incrby(keyname.format(org=orgName, orgCateg='points'), pnts_to_add)
#update contribution Requirement # getRequirement parameters
heavilyCount = self.heavilyCount heavilyCount = self.heavilyCount
recentDays = self.recentDays recentDays = self.recentDays
regularlyDays = self.regularlyDays regularlyDays = self.regularlyDays
isRecent = (datetime.datetime.now() - eventTime).days > recentDays isRecent = (datetime.datetime.now() - eventTime).days > recentDays
print("isLabeled: {}, isRecent: {}, totOrgPnts".format(isLabeled, isRecent, totOrgPnts))
#update contribution Requirement
contrib = [] #[[contrib_level, contrib_ttl], [], ...] contrib = [] #[[contrib_level, contrib_ttl], [], ...]
if contribType == 'sighting': if totOrgPnts >= self.org_rank_requirement_pnts[1] and contribType == 'Sighting':
#[contrib_level, contrib_ttl] #[contrib_level, contrib_ttl]
contrib.append([1, ONE_DAY*365]) contrib.append([1, util.ONE_DAY*365])
if contribType == 'attribute' or contribType == 'object': if totOrgPnts >= self.org_rank_requirement_pnts[2] and contribType == 'Attribute' or contribType == 'Object':
contrib.append([2, ONE_DAY*365]) contrib.append([2, util.ONE_DAY*365])
if contribType == 'proposal' or contribType == 'discussion': if totOrgPnts >= self.org_rank_requirement_pnts[3] and contribType == 'Proposal' or contribType == 'Discussion':
contrib.append([3, ONE_DAY*365]) contrib.append([3, util.ONE_DAY*365])
if contribType == 'sighting' and isRecent: if totOrgPnts >= self.org_rank_requirement_pnts[4] and contribType == 'Sighting' and isRecent:
contrib.append([4, ONE_DAY*recentDays]) contrib.append([4, util.ONE_DAY*recentDays])
if contribType == 'proposal' and isRecent: if totOrgPnts >= self.org_rank_requirement_pnts[5] and contribType == 'Proposal' and isRecent:
contrib.append([5, ONE_DAY*recentDays]) contrib.append([5, util.ONE_DAY*recentDays])
if contribType == 'event': if totOrgPnts >= self.org_rank_requirement_pnts[6] and contribType == 'Event':
contrib.append([6, ONE_DAY*365]) contrib.append([6, util.ONE_DAY*365])
if contribType == 'event': if totOrgPnts >= self.org_rank_requirement_pnts[7] and contribType == 'Event':
contrib.append([7, ONE_DAY*recentDays]) contrib.append([7, util.ONE_DAY*recentDays])
if contribType == 'event': if totOrgPnts >= self.org_rank_requirement_pnts[8] and contribType == 'Event':
contrib.append([8, ONE_DAY*regularlyDays]) contrib.append([8, util.ONE_DAY*regularlyDays])
if contribType == 'event' and isLabeled: if totOrgPnts >= self.org_rank_requirement_pnts[9] and contribType == 'Event' and isLabeled:
contrib.append([9, ONE_DAY*regularlyDays]) contrib.append([9, util.ONE_DAY*regularlyDays])
if contribType == 'sighting' and sightingWeekCount>heavilyCount: if totOrgPnts >= self.org_rank_requirement_pnts[10] and contribType == 'Sighting' and sightingWeekCount>heavilyCount:
contrib.append([10, ONE_DAY*regularlyDays]) contrib.append([10, util.ONE_DAY*regularlyDays])
if (contribType == 'attribute' or contribType == 'object') and attributeWeekCount>heavilyCount: if totOrgPnts >= self.org_rank_requirement_pnts[11] and (contribType == 'Attribute' or contribType == 'Object') and attributeWeekCount>heavilyCount:
contrib.append([11, ONE_DAY*regularlyDays]) contrib.append([11, util.ONE_DAY*regularlyDays])
if contribType == 'proposal' and proposalWeekCount>heavilyCount: if totOrgPnts >= self.org_rank_requirement_pnts[12] and contribType == 'Proposal' and proposalWeekCount>heavilyCount:
contrib.append([12, ONE_DAY*regularlyDays]) contrib.append([12, util.ONE_DAY*regularlyDays])
if contribType == 'event' and eventWeekCount>heavilyCount: if totOrgPnts >= self.org_rank_requirement_pnts[13] and contribType == 'Event' and eventWeekCount>heavilyCount:
contrib.append([13, ONE_DAY*regularlyDays]) contrib.append([13, util.ONE_DAY*regularlyDays])
if contribType == 'event' and eventWeekCount>heavilyCount and isLabeled: if totOrgPnts >= self.org_rank_requirement_pnts[14] and contribType == 'Event' and eventWeekCount>heavilyCount and isLabeled:
contrib.append([14, ONE_DAY*regularlyDays]) contrib.append([14, util.ONE_DAY*regularlyDays])
print([r for r, ttl in contrib])
for rankReq, ttl in contrib: for rankReq, ttl in contrib:
self.serv_redis_db.set(keyname.format(org=orgName, orgCateg='CONTRIB_REQ_'+str(rankReq)), 1) self.serv_redis_db.set(keyname.format(org=orgName, orgCateg='CONTRIB_REQ_'+str(rankReq)), 1)
self.serv_redis_db.expire(keyname.format(org=orgName, orgCateg='CONTRIB_REQ_'+str(i)), ttl) self.serv_redis_db.expire(keyname.format(org=orgName, orgCateg='CONTRIB_REQ_'+str(rankReq)), ttl)
''' HONOR BADGES ''' ''' HONOR BADGES '''
def getOrgHonorBadges(self, org): def getOrgHonorBadges(self, org):
@ -357,22 +368,20 @@ class Contributor_helper:
def TEST_getTop5OvertimeFromRedis(self): def TEST_getTop5OvertimeFromRedis(self):
import time import time
now = time.time() now = time.time()
ONE_DAY = 60*60*24
data2 = [ data2 = [
{'label': 'CIRCL', 'data': [[now, random.randint(1,50)], [now-ONE_DAY, random.randint(1,50)], [now-ONE_DAY*2, random.randint(1,50)], [now-ONE_DAY*3, random.randint(1,50)], [now-ONE_DAY*4, random.randint(1,50)]]}, {'label': 'CIRCL', 'data': [[now, random.randint(1,50)], [now-util.ONE_DAY, random.randint(1,50)], [now-util.ONE_DAY*2, random.randint(1,50)], [now-util.ONE_DAY*3, random.randint(1,50)], [now-util.ONE_DAY*4, random.randint(1,50)]]},
{'label': 'CASES', 'data': [[now, random.randint(1,50)], [now-ONE_DAY, random.randint(1,50)], [now-ONE_DAY*2, random.randint(1,50)], [now-ONE_DAY*3, random.randint(1,50)], [now-ONE_DAY*4, random.randint(1,50)]]}, {'label': 'CASES', 'data': [[now, random.randint(1,50)], [now-util.ONE_DAY, random.randint(1,50)], [now-util.ONE_DAY*2, random.randint(1,50)], [now-util.ONE_DAY*3, random.randint(1,50)], [now-util.ONE_DAY*4, random.randint(1,50)]]},
{'label': 'Org1', 'data': [[now, random.randint(1,50)], [now-ONE_DAY, random.randint(1,50)], [now-ONE_DAY*2, random.randint(1,50)], [now-ONE_DAY*3, random.randint(1,50)], [now-ONE_DAY*4, random.randint(1,50)]]}, {'label': 'Org1', 'data': [[now, random.randint(1,50)], [now-util.ONE_DAY, random.randint(1,50)], [now-util.ONE_DAY*2, random.randint(1,50)], [now-util.ONE_DAY*3, random.randint(1,50)], [now-util.ONE_DAY*4, random.randint(1,50)]]},
{'label': 'Org2', 'data': [[now, random.randint(1,50)], [now-ONE_DAY, random.randint(1,50)], [now-ONE_DAY*2, random.randint(1,50)], [now-ONE_DAY*3, random.randint(1,50)], [now-ONE_DAY*4, random.randint(1,50)]]}, {'label': 'Org2', 'data': [[now, random.randint(1,50)], [now-util.ONE_DAY, random.randint(1,50)], [now-util.ONE_DAY*2, random.randint(1,50)], [now-util.ONE_DAY*3, random.randint(1,50)], [now-util.ONE_DAY*4, random.randint(1,50)]]},
{'label': 'SMILE', 'data': [[now, random.randint(1,50)], [now-ONE_DAY, random.randint(1,50)], [now-ONE_DAY*2, random.randint(1,50)], [now-ONE_DAY*3, random.randint(1,50)], [now-ONE_DAY*4, random.randint(1,50)]]}, {'label': 'SMILE', 'data': [[now, random.randint(1,50)], [now-util.ONE_DAY, random.randint(1,50)], [now-util.ONE_DAY*2, random.randint(1,50)], [now-util.ONE_DAY*3, random.randint(1,50)], [now-util.ONE_DAY*4, random.randint(1,50)]]},
] ]
return data2 return data2
def TEST_getOrgOvertime(self, org): def TEST_getOrgOvertime(self, org):
import time import time
now = time.time() now = time.time()
ONE_DAY = 60*60*24
data = [ data = [
{'label': org, 'data': [[now, random.randint(1,30)], [now-ONE_DAY, random.randint(1,30)], [now-ONE_DAY*2, random.randint(1,30)], [now-ONE_DAY*3, random.randint(1,30)], [now-ONE_DAY*4, random.randint(1,40)]]} {'label': org, 'data': [[now, random.randint(1,30)], [now-util.ONE_DAY, random.randint(1,30)], [now-util.ONE_DAY*2, random.randint(1,30)], [now-util.ONE_DAY*3, random.randint(1,30)], [now-util.ONE_DAY*4, random.randint(1,40)]]}
] ]
return data return data

4
static/css/ranking.css
View File

@ -3,6 +3,10 @@
height: auto; height: auto;
} }
.successCell {
background-color: #dff0d8 !important
}
.circleBadge { .circleBadge {
width: 80px; width: 80px;
height: 80px; height: 80px;

26
static/js/contrib.js
View File

@ -156,11 +156,13 @@ function getOrgRankIcon(rank, size) {
} }
function createImg(source, size) { function createImg(source, size) {
var obj = document.createElement('object'); var obj = document.createElement('img');
obj.height = size; obj.height = size;
obj.width = size; obj.width = size;
obj.style.margin = 'auto'; obj.style.margin = 'auto';
obj.data = source; obj.src = source;
obj.type = "image/png"
obj.alt = ""
return obj.outerHTML; return obj.outerHTML;
} }
@ -372,19 +374,22 @@ function updateProgressHeader(org) {
} }
for (var row of $('#bodyTablerankingModal')[0].children) { for (var row of $('#bodyTablerankingModal')[0].children) {
row = $(row); row = $(row);
var firstCell = $(row.children()[0]);
var rank = row.data('rank'); var rank = row.data('rank');
//remove all classes //remove all classes
row.removeClass("warning"); row.removeClass("warning");
row.removeClass("danger"); row.removeClass("danger");
row.removeClass("success"); row.removeClass("success");
firstCell.removeClass("successCell");
//add correct class //add correct class
if(status[rank] == 0){ if(status[rank] == 0){
row.addClass("danger"); row.addClass("danger");
} else if(status[rank] == 1 && rank == curContributionOrgRank) {
row.addClass("success");
} else if(status[rank] == 1) { } else if(status[rank] == 1) {
row.addClass("warning"); row.addClass("warning");
} }
if(rank == curContributionOrgRank) {
firstCell.addClass("successCell");
}
} }
}); });
@ -424,14 +429,13 @@ function updateProgressHeader(org) {
$.getJSON( url_getOrgOvertime+'?org='+org, function( data ) { $.getJSON( url_getOrgOvertime+'?org='+org, function( data ) {
var toPlot = dataTop5Overtime.slice(0); //cloning data var toPlot = dataTop5Overtime.slice(0); //cloning data
// transform secs into date // transform secs into date
for(i in data){ var new_data = [];
var new_data = []; for(list of data['data']) {
for(list of data[i]['data']) { new_data.push([new Date(list[0]*1000), list[1]]);
new_data.push([new Date(list[0]*1000), list[1]]);
}
data[i]['data'] = new_data;
toPlot.push(data[i]);
} }
data['data'] = new_data;
toPlot.push(data);
plotLineChart.setData(toPlot); plotLineChart.setData(toPlot);
plotLineChart.setupGrid(); plotLineChart.setupGrid();
plotLineChart.draw(); plotLineChart.draw();

2
util.py
View File

@ -1,5 +1,7 @@
import datetime import datetime
ONE_DAY = 60*60*24
def getMonthSpan(date): def getMonthSpan(date):
ds = datetime.datetime(date.year, date.month, 1) ds = datetime.datetime(date.year, date.month, 1)
dyear = 1 if ds.month+1 > 12 else 0 dyear = 1 if ds.month+1 > 12 else 0

29
zmq_subscriber.py
View File

@ -130,8 +130,8 @@ def noSpaceLower(str):
return str.lower().replace(' ', '_') return str.lower().replace(' ', '_')
#pntMultiplier if one contribution rewards more than others. (e.g. shighting may gives more points than editing) #pntMultiplier if one contribution rewards more than others. (e.g. shighting may gives more points than editing)
def handleContribution(zmq_name, org, categ, action, pntMultiplier=1, eventTime=datetime.datetime.now(), isLabeled=False): def handleContribution(zmq_name, org, contribType, categ, action, pntMultiplier=1, eventTime=datetime.datetime.now(), isLabeled=False):
if action in ['edit']: if action in ['edit', None]:
pass pass
#return #not a contribution? #return #not a contribution?
# is a valid contribution # is a valid contribution
@ -151,7 +151,7 @@ def handleContribution(zmq_name, org, categ, action, pntMultiplier=1, eventTime=
serv_redis_db.zadd('CONTRIB_LAST:'+util.getDateStrFormat(now), nowSec, org) serv_redis_db.zadd('CONTRIB_LAST:'+util.getDateStrFormat(now), nowSec, org)
serv_redis_db.expire('CONTRIB_LAST:'+util.getDateStrFormat(now), ONE_DAY) #expire after 1 day serv_redis_db.expire('CONTRIB_LAST:'+util.getDateStrFormat(now), ONE_DAY) #expire after 1 day
contributor_helper.updateOrgContributionRank(org, pnts_to_add, eventTime, eventTime=datetime.datetime.now(), isLabeled=isLabeled) contributor_helper.updateOrgContributionRank(org, pnts_to_add, contribType, eventTime=datetime.datetime.now(), isLabeled=isLabeled)
publish_log(zmq_name, 'CONTRIBUTION', {'org': org, 'categ': categ, 'action': action, 'epoch': nowSec }, channel=CHANNEL_LASTCONTRIB) publish_log(zmq_name, 'CONTRIBUTION', {'org': org, 'categ': categ, 'action': action, 'epoch': nowSec }, channel=CHANNEL_LASTCONTRIB)
@ -178,8 +178,11 @@ def handler_sighting(zmq_name, jsondata):
jsonsight = jsondata['Sighting'] jsonsight = jsondata['Sighting']
org = jsonsight['Event']['Orgc']['name'] org = jsonsight['Event']['Orgc']['name']
categ = jsonsight['Attribute']['category'] categ = jsonsight['Attribute']['category']
action = jsondata['action'] try:
handleContribution(zmq_name, org, categ, action, pntMultiplier=2) action = jsondata['action']
except KeyError:
action = None
handleContribution(zmq_name, org, 'Sighting', categ, action, pntMultiplier=2)
handler_attribute(zmq_name, jsonsight, hasAlreadyBeenContributed=True) handler_attribute(zmq_name, jsonsight, hasAlreadyBeenContributed=True)
def handler_event(zmq_name, jsonobj): def handler_event(zmq_name, jsonobj):
@ -217,9 +220,19 @@ def handler_attribute(zmq_name, jsonobj, hasAlreadyBeenContributed=False):
getCoordAndPublish(zmq_name, jsonattr['value'], jsonattr['category']) getCoordAndPublish(zmq_name, jsonattr['value'], jsonattr['category'])
if not hasAlreadyBeenContributed: if not hasAlreadyBeenContributed:
eventLabeled = False try:
#eventLabeled = len(jsonattr[]) > 0 eventLabeled = len(jsonattr['Tag']) > 0
handleContribution(zmq_name, jsonobj['Event']['Orgc']['name'], jsonattr['category'], jsonobj['action'], isLabeled=eventLabeled) except KeyError:
eventLabeled = False
try:
action = jsonobj['action']
except KeyError:
action = None
handleContribution(zmq_name, jsonobj['Event']['Orgc']['name'],
'Attribute',
jsonattr['category'],
action,
isLabeled=eventLabeled)
# Push to log # Push to log
publish_log(zmq_name, 'Attribute', to_push) publish_log(zmq_name, 'Attribute', to_push)