MISP
/
misp-dashboard
mirror of https://github.com/MISP/misp-dashboard
2
0
Fork 0
Code Issues Releases Wiki Activity

fixed bugs if requesting data from non-existing org + refined draft updateOrgContributionRank + Fixed non existing org logo + try in zmq_subscribe

pull/3/head
Sami Mokaddem 2017-11-09 13:43:02 +01:00
parent 139c7dd54f
commit 1550f25b30
6 changed files with 94 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
config/ranking.cfg
View File

@ -32,8 +32,8 @@ maxLevel=16
12=4096
13=8192
14=16384
15=32768
16=65536
15=0
16=0
[rankRequirementsMisc]
heavilyCount=10

91
contributor_helper.py
View File

@ -69,6 +69,15 @@ class Contributor_helper:
return data
''' CONTRIBUTION RANK '''
def getOrgContributionTotalPoints(self, org):
keyname = 'CONTRIB_ORG:{org}:{orgCateg}'
pnts = self.serv_redis_db.get(keyname.format(org=org, orgCateg='points'))
if pnts is None:
pnts = 0
else:
pnts = int(pnts.decode('utf8'))
return pnts
# return: [final_rank, requirement_fulfilled, requirement_not_fulfilled]
def getOrgContributionRank(self, org):
keyname = 'CONTRIB_ORG:{org}:{orgCateg}'
@ -108,53 +117,55 @@ class Contributor_helper:
to_ret[i] = 0
else:
to_ret[i] = -1
return to_ret
return {'rank': final_rank, 'status': to_ret, 'totPoints': self.getOrgContributionTotalPoints(org)}
def updateOrgContributionRank(self, orgName, pnts_to_add, contribType, eventTime, isLabeled):
keyname = 'CONTRIB_ORG:{org}:{orgCateg}'
#update total points
self.serv_redis_db.set(keyname.format(org=orgName, orgCateg='points'), pnts_to_add)
#update contribution Requirement
totOrgPnts = self.serv_redis_db.incrby(keyname.format(org=orgName, orgCateg='points'), pnts_to_add)
# getRequirement parameters
heavilyCount = self.heavilyCount
recentDays = self.recentDays
regularlyDays = self.regularlyDays
isRecent = (datetime.datetime.now() - eventTime).days > recentDays
print("isLabeled: {}, isRecent: {}, totOrgPnts".format(isLabeled, isRecent, totOrgPnts))
#update contribution Requirement
contrib = [] #[[contrib_level, contrib_ttl], [], ...]
if contribType == 'sighting':
if totOrgPnts >= self.org_rank_requirement_pnts[1] and contribType == 'Sighting':
#[contrib_level, contrib_ttl]
contrib.append([1, ONE_DAY*365])
if contribType == 'attribute' or contribType == 'object':
contrib.append([2, ONE_DAY*365])
if contribType == 'proposal' or contribType == 'discussion':
contrib.append([3, ONE_DAY*365])
if contribType == 'sighting' and isRecent:
contrib.append([4, ONE_DAY*recentDays])
if contribType == 'proposal' and isRecent:
contrib.append([5, ONE_DAY*recentDays])
if contribType == 'event':
contrib.append([6, ONE_DAY*365])
if contribType == 'event':
contrib.append([7, ONE_DAY*recentDays])
if contribType == 'event':
contrib.append([8, ONE_DAY*regularlyDays])
if contribType == 'event' and isLabeled:
contrib.append([9, ONE_DAY*regularlyDays])
if contribType == 'sighting' and sightingWeekCount>heavilyCount:
contrib.append([10, ONE_DAY*regularlyDays])
if (contribType == 'attribute' or contribType == 'object') and attributeWeekCount>heavilyCount:
contrib.append([11, ONE_DAY*regularlyDays])
if contribType == 'proposal' and proposalWeekCount>heavilyCount:
contrib.append([12, ONE_DAY*regularlyDays])
if contribType == 'event' and eventWeekCount>heavilyCount:
contrib.append([13, ONE_DAY*regularlyDays])
if contribType == 'event' and eventWeekCount>heavilyCount and isLabeled:
contrib.append([14, ONE_DAY*regularlyDays])
contrib.append([1, util.ONE_DAY*365])
if totOrgPnts >= self.org_rank_requirement_pnts[2] and contribType == 'Attribute' or contribType == 'Object':
contrib.append([2, util.ONE_DAY*365])
if totOrgPnts >= self.org_rank_requirement_pnts[3] and contribType == 'Proposal' or contribType == 'Discussion':
contrib.append([3, util.ONE_DAY*365])
if totOrgPnts >= self.org_rank_requirement_pnts[4] and contribType == 'Sighting' and isRecent:
contrib.append([4, util.ONE_DAY*recentDays])
if totOrgPnts >= self.org_rank_requirement_pnts[5] and contribType == 'Proposal' and isRecent:
contrib.append([5, util.ONE_DAY*recentDays])
if totOrgPnts >= self.org_rank_requirement_pnts[6] and contribType == 'Event':
contrib.append([6, util.ONE_DAY*365])
if totOrgPnts >= self.org_rank_requirement_pnts[7] and contribType == 'Event':
contrib.append([7, util.ONE_DAY*recentDays])
if totOrgPnts >= self.org_rank_requirement_pnts[8] and contribType == 'Event':
contrib.append([8, util.ONE_DAY*regularlyDays])
if totOrgPnts >= self.org_rank_requirement_pnts[9] and contribType == 'Event' and isLabeled:
contrib.append([9, util.ONE_DAY*regularlyDays])
if totOrgPnts >= self.org_rank_requirement_pnts[10] and contribType == 'Sighting' and sightingWeekCount>heavilyCount:
contrib.append([10, util.ONE_DAY*regularlyDays])
if totOrgPnts >= self.org_rank_requirement_pnts[11] and (contribType == 'Attribute' or contribType == 'Object') and attributeWeekCount>heavilyCount:
contrib.append([11, util.ONE_DAY*regularlyDays])
if totOrgPnts >= self.org_rank_requirement_pnts[12] and contribType == 'Proposal' and proposalWeekCount>heavilyCount:
contrib.append([12, util.ONE_DAY*regularlyDays])
if totOrgPnts >= self.org_rank_requirement_pnts[13] and contribType == 'Event' and eventWeekCount>heavilyCount:
contrib.append([13, util.ONE_DAY*regularlyDays])
if totOrgPnts >= self.org_rank_requirement_pnts[14] and contribType == 'Event' and eventWeekCount>heavilyCount and isLabeled:
contrib.append([14, util.ONE_DAY*regularlyDays])
print([r for r, ttl in contrib])
for rankReq, ttl in contrib:
self.serv_redis_db.set(keyname.format(org=orgName, orgCateg='CONTRIB_REQ_'+str(rankReq)), 1)
self.serv_redis_db.expire(keyname.format(org=orgName, orgCateg='CONTRIB_REQ_'+str(i)), ttl)
self.serv_redis_db.expire(keyname.format(org=orgName, orgCateg='CONTRIB_REQ_'+str(rankReq)), ttl)
''' HONOR BADGES '''
def getOrgHonorBadges(self, org):
@ -357,22 +368,20 @@ class Contributor_helper:
def TEST_getTop5OvertimeFromRedis(self):
import time
now = time.time()
ONE_DAY = 60*60*24
data2 = [
{'label': 'CIRCL', 'data': [[now, random.randint(1,50)], [now-ONE_DAY, random.randint(1,50)], [now-ONE_DAY*2, random.randint(1,50)], [now-ONE_DAY*3, random.randint(1,50)], [now-ONE_DAY*4, random.randint(1,50)]]},
{'label': 'CASES', 'data': [[now, random.randint(1,50)], [now-ONE_DAY, random.randint(1,50)], [now-ONE_DAY*2, random.randint(1,50)], [now-ONE_DAY*3, random.randint(1,50)], [now-ONE_DAY*4, random.randint(1,50)]]},
{'label': 'Org1', 'data': [[now, random.randint(1,50)], [now-ONE_DAY, random.randint(1,50)], [now-ONE_DAY*2, random.randint(1,50)], [now-ONE_DAY*3, random.randint(1,50)], [now-ONE_DAY*4, random.randint(1,50)]]},
{'label': 'Org2', 'data': [[now, random.randint(1,50)], [now-ONE_DAY, random.randint(1,50)], [now-ONE_DAY*2, random.randint(1,50)], [now-ONE_DAY*3, random.randint(1,50)], [now-ONE_DAY*4, random.randint(1,50)]]},
{'label': 'SMILE', 'data': [[now, random.randint(1,50)], [now-ONE_DAY, random.randint(1,50)], [now-ONE_DAY*2, random.randint(1,50)], [now-ONE_DAY*3, random.randint(1,50)], [now-ONE_DAY*4, random.randint(1,50)]]},
{'label': 'CIRCL', 'data': [[now, random.randint(1,50)], [now-util.ONE_DAY, random.randint(1,50)], [now-util.ONE_DAY*2, random.randint(1,50)], [now-util.ONE_DAY*3, random.randint(1,50)], [now-util.ONE_DAY*4, random.randint(1,50)]]},
{'label': 'CASES', 'data': [[now, random.randint(1,50)], [now-util.ONE_DAY, random.randint(1,50)], [now-util.ONE_DAY*2, random.randint(1,50)], [now-util.ONE_DAY*3, random.randint(1,50)], [now-util.ONE_DAY*4, random.randint(1,50)]]},
{'label': 'Org1', 'data': [[now, random.randint(1,50)], [now-util.ONE_DAY, random.randint(1,50)], [now-util.ONE_DAY*2, random.randint(1,50)], [now-util.ONE_DAY*3, random.randint(1,50)], [now-util.ONE_DAY*4, random.randint(1,50)]]},
{'label': 'Org2', 'data': [[now, random.randint(1,50)], [now-util.ONE_DAY, random.randint(1,50)], [now-util.ONE_DAY*2, random.randint(1,50)], [now-util.ONE_DAY*3, random.randint(1,50)], [now-util.ONE_DAY*4, random.randint(1,50)]]},
{'label': 'SMILE', 'data': [[now, random.randint(1,50)], [now-util.ONE_DAY, random.randint(1,50)], [now-util.ONE_DAY*2, random.randint(1,50)], [now-util.ONE_DAY*3, random.randint(1,50)], [now-util.ONE_DAY*4, random.randint(1,50)]]},
]
return data2
def TEST_getOrgOvertime(self, org):
import time
now = time.time()
ONE_DAY = 60*60*24
data = [
{'label': org, 'data': [[now, random.randint(1,30)], [now-ONE_DAY, random.randint(1,30)], [now-ONE_DAY*2, random.randint(1,30)], [now-ONE_DAY*3, random.randint(1,30)], [now-ONE_DAY*4, random.randint(1,40)]]}
{'label': org, 'data': [[now, random.randint(1,30)], [now-util.ONE_DAY, random.randint(1,30)], [now-util.ONE_DAY*2, random.randint(1,30)], [now-util.ONE_DAY*3, random.randint(1,30)], [now-util.ONE_DAY*4, random.randint(1,40)]]}
]
return data

4
static/css/ranking.css
View File

@ -3,6 +3,10 @@
height: auto;
}
.successCell {
background-color: #dff0d8 !important
}
.circleBadge {
width: 80px;
height: 80px;

22
static/js/contrib.js
View File

@ -156,11 +156,13 @@ function getOrgRankIcon(rank, size) {
}
function createImg(source, size) {
var obj = document.createElement('object');
var obj = document.createElement('img');
obj.height = size;
obj.width = size;
obj.style.margin = 'auto';
obj.data = source;
obj.src = source;
obj.type = "image/png"
obj.alt = ""
return obj.outerHTML;
}
@ -372,19 +374,22 @@ function updateProgressHeader(org) {
}
for (var row of $('#bodyTablerankingModal')[0].children) {
row = $(row);
var firstCell = $(row.children()[0]);
var rank = row.data('rank');
//remove all classes
row.removeClass("warning");
row.removeClass("danger");
row.removeClass("success");
firstCell.removeClass("successCell");
//add correct class
if(status[rank] == 0){
row.addClass("danger");
} else if(status[rank] == 1 && rank == curContributionOrgRank) {
row.addClass("success");
} else if(status[rank] == 1) {
row.addClass("warning");
}
if(rank == curContributionOrgRank) {
firstCell.addClass("successCell");
}
}
});
@ -424,14 +429,13 @@ function updateProgressHeader(org) {
$.getJSON( url_getOrgOvertime+'?org='+org, function( data ) {
var toPlot = dataTop5Overtime.slice(0); //cloning data
// transform secs into date
for(i in data){
var new_data = [];
for(list of data[i]['data']) {
for(list of data['data']) {
new_data.push([new Date(list[0]*1000), list[1]]);
}
data[i]['data'] = new_data;
toPlot.push(data[i]);
}
data['data'] = new_data;
toPlot.push(data);
plotLineChart.setData(toPlot);
plotLineChart.setupGrid();
plotLineChart.draw();

2
util.py
View File

@ -1,5 +1,7 @@
import datetime
ONE_DAY = 60*60*24
def getMonthSpan(date):
ds = datetime.datetime(date.year, date.month, 1)
dyear = 1 if ds.month+1 > 12 else 0

25
zmq_subscriber.py
View File

@ -130,8 +130,8 @@ def noSpaceLower(str):
return str.lower().replace(' ', '_')
#pntMultiplier if one contribution rewards more than others. (e.g. shighting may gives more points than editing)
def handleContribution(zmq_name, org, categ, action, pntMultiplier=1, eventTime=datetime.datetime.now(), isLabeled=False):
if action in ['edit']:
def handleContribution(zmq_name, org, contribType, categ, action, pntMultiplier=1, eventTime=datetime.datetime.now(), isLabeled=False):
if action in ['edit', None]:
pass
#return #not a contribution?
# is a valid contribution
@ -151,7 +151,7 @@ def handleContribution(zmq_name, org, categ, action, pntMultiplier=1, eventTime=
serv_redis_db.zadd('CONTRIB_LAST:'+util.getDateStrFormat(now), nowSec, org)
serv_redis_db.expire('CONTRIB_LAST:'+util.getDateStrFormat(now), ONE_DAY) #expire after 1 day
contributor_helper.updateOrgContributionRank(org, pnts_to_add, eventTime, eventTime=datetime.datetime.now(), isLabeled=isLabeled)
contributor_helper.updateOrgContributionRank(org, pnts_to_add, contribType, eventTime=datetime.datetime.now(), isLabeled=isLabeled)
publish_log(zmq_name, 'CONTRIBUTION', {'org': org, 'categ': categ, 'action': action, 'epoch': nowSec }, channel=CHANNEL_LASTCONTRIB)
@ -178,8 +178,11 @@ def handler_sighting(zmq_name, jsondata):
jsonsight = jsondata['Sighting']
org = jsonsight['Event']['Orgc']['name']
categ = jsonsight['Attribute']['category']
try:
action = jsondata['action']
handleContribution(zmq_name, org, categ, action, pntMultiplier=2)
except KeyError:
action = None
handleContribution(zmq_name, org, 'Sighting', categ, action, pntMultiplier=2)
handler_attribute(zmq_name, jsonsight, hasAlreadyBeenContributed=True)
def handler_event(zmq_name, jsonobj):
@ -217,9 +220,19 @@ def handler_attribute(zmq_name, jsonobj, hasAlreadyBeenContributed=False):
getCoordAndPublish(zmq_name, jsonattr['value'], jsonattr['category'])
if not hasAlreadyBeenContributed:
try:
eventLabeled = len(jsonattr['Tag']) > 0
except KeyError:
eventLabeled = False
#eventLabeled = len(jsonattr[]) > 0
handleContribution(zmq_name, jsonobj['Event']['Orgc']['name'], jsonattr['category'], jsonobj['action'], isLabeled=eventLabeled)
try:
action = jsonobj['action']
except KeyError:
action = None
handleContribution(zmq_name, jsonobj['Event']['Orgc']['name'],
'Attribute',
jsonattr['category'],
action,
isLabeled=eventLabeled)
# Push to log
publish_log(zmq_name, 'Attribute', to_push)