MISP
/
misp-dashboard
mirror of https://github.com/MISP/misp-dashboard
2
0
Fork 0
Code Issues Releases Wiki Activity

Added overtake remaining points + moved updateOrgContributionRank from zmq_subscribe to contributor_helper

pull/3/head
Sami Mokaddem 2017-11-08 17:37:03 +01:00
parent 93ce25946d
commit 10c59fc220
5 changed files with 103 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
contributor_helper.py
View File

@ -88,11 +88,11 @@ class Contributor_helper:
def giveContribRankToOrg(self, org, rankNum):
keyname = 'CONTRIB_ORG:{org}:{orgCateg}'
serv_redis_db.set(keyname.format(org=orgName, orgCateg='CONTRIB_REQ_'+str(rankNum)), 1)
self.serv_redis_db.set(keyname.format(org=orgName, orgCateg='CONTRIB_REQ_'+str(rankNum)), 1)
def removeContribRankFromOrg(self, org, rankNum):
keyname = 'CONTRIB_ORG:{org}:{orgCateg}'
serv_redis_db.delete(keyname.format(org=orgName, orgCateg='CONTRIB_REQ_'+str(rankNum)))
self.serv_redis_db.delete(keyname.format(org=orgName, orgCateg='CONTRIB_REQ_'+str(rankNum)))
# 1 for fulfilled, 0 for not fulfilled, -1 for not relevant
def getCurrentContributionStatus(self, org):
@ -107,6 +107,52 @@ class Contributor_helper:
to_ret[i] = -1
return to_ret
def updateOrgContributionRank(self, orgName, pnts_to_add, contribType, eventTime, isLabeled):
keyname = 'CONTRIB_ORG:{org}:{orgCateg}'
#update total points
self.serv_redis_db.set(keyname.format(org=orgName, orgCateg='points'), pnts_to_add)
#update contribution Requirement
heavilyCount = self.heavilyCount
recentDays = self.recentDays
regularlyDays = self.regularlyDays
isRecent = (datetime.datetime.now() - eventTime).days > recentDays
contrib = [] #[[contrib_level, contrib_ttl], [], ...]
if contribType == 'sighting':
#[contrib_level, contrib_ttl]
contrib.append([1, ONE_DAY*365])
if contribType == 'attribute' or contribType == 'object':
contrib.append([2, ONE_DAY*365])
if contribType == 'proposal' or contribType == 'discussion':
contrib.append([3, ONE_DAY*365])
if contribType == 'sighting' and isRecent:
contrib.append([4, ONE_DAY*recentDays])
if contribType == 'proposal' and isRecent:
contrib.append([5, ONE_DAY*recentDays])
if contribType == 'event':
contrib.append([6, ONE_DAY*365])
if contribType == 'event':
contrib.append([7, ONE_DAY*recentDays])
if contribType == 'event':
contrib.append([8, ONE_DAY*regularlyDays])
if contribType == 'event' and isLabeled:
contrib.append([9, ONE_DAY*regularlyDays])
if contribType == 'sighting' and sightingWeekCount>heavilyCount:
contrib.append([10, ONE_DAY*regularlyDays])
if (contribType == 'attribute' or contribType == 'object') and attributeWeekCount>heavilyCount:
contrib.append([11, ONE_DAY*regularlyDays])
if contribType == 'proposal' and proposalWeekCount>heavilyCount:
contrib.append([12, ONE_DAY*regularlyDays])
if contribType == 'event' and eventWeekCount>heavilyCount:
contrib.append([13, ONE_DAY*regularlyDays])
if contribType == 'event' and eventWeekCount>heavilyCount and isLabeled:
contrib.append([14, ONE_DAY*regularlyDays])
for rankReq, ttl in contrib:
self.serv_redis_db.set(keyname.format(org=orgName, orgCateg='CONTRIB_REQ_'+str(rankReq)), 1)
self.serv_redis_db.expire(keyname.format(org=orgName, orgCateg='CONTRIB_REQ_'+str(i)), ttl)
''' HONOR BADGES '''
def getOrgHonorBadges(self, org):
keyname = 'CONTRIB_ORG:{org}:{orgCateg}'
@ -119,11 +165,11 @@ class Contributor_helper:
def giveBadgeToOrg(self, org, badgeNum):
keyname = 'CONTRIB_ORG:{org}:{orgCateg}'
serv_redis_db.set(keyname.format(org=orgName, orgCateg='BADGE_'+str(badgeNum)), 1)
self.serv_redis_db.set(keyname.format(org=orgName, orgCateg='BADGE_'+str(badgeNum)), 1)
def removeBadgeFromOrg(self, org, badgeNum):
keyname = 'CONTRIB_ORG:{org}:{orgCateg}'
serv_redis_db.delete(keyname.format(org=orgName, orgCateg='BADGE_'+str(badgeNum)))
self.serv_redis_db.delete(keyname.format(org=orgName, orgCateg='BADGE_'+str(badgeNum)))
''' MONTHLY CONTRIBUTION '''
def getOrgPntFromRedis(self, org, date):
@ -270,7 +316,9 @@ class Contributor_helper:
return { 'remainingPts': 0, 'stepPts': self.rankMultiplier**self.levelMax }
''' '''
''' TEST DATA '''
''' '''
def TEST_getCategPerContribFromRedis(self, date):
data2 = []

22
static/js/contrib.js
View File

@ -308,6 +308,7 @@ function addLastContributor(datatable, data, update) {
}
function updateProgressHeader(org) {
currOrg = org;
// get Org rank
$.getJSON( url_getOrgRank+'?org='+org, function( data ) {
datatableTop.draw();
@ -383,6 +384,25 @@ function updateProgressHeader(org) {
}
}
});
//update overtake points
var prevOrg = "";
var prevOrgPnts = 0;
datatableTop.rows().every( function() {
var row = this.node();
if(this.data()[5] == currOrg) {
if(prevOrg == ""){ //already first
$('#orgToOverTake').text(this.data()[5]);
$('#pntsToOvertakeNext').text(0);
} else {
$('#orgToOverTake').text(prevOrg);
$('#pntsToOvertakeNext').text(parseInt(prevOrgPnts)-this.data()[0]);
}
} else {
prevOrg = this.data()[5];
prevOrgPnts = this.data()[0];
}
});
}
function showOnlyOrg() {
@ -458,7 +478,7 @@ $(document).ready(function() {
});
if(currOrg != "") // currOrg selected
//FIXME: timeout used to wait that all datatables are draw.
setTimeout( function() { updateProgressHeader(currOrg); }, 200);
setTimeout( function() { updateProgressHeader(currOrg); }, 400);
source_lastContrib = new EventSource(url_eventStreamLastContributor);
source_lastContrib.onmessage = function(event) {

8
templates/contrib.html
View File

@ -201,6 +201,14 @@
</div>
<div id="orgNextRankDiv" class='textTopHeader' style="padding-top: 0px; position: relative; width: 40px; height: 40px;"></div>
</button>
<div class='leftSepa textTopHeader'>
<span class="label label-primary">
<strong style="">Points to overtake</strong>
<strong style="font-size: medium;" id='orgToOverTake'></strong>
<strong style="">: </strong>
<strong style="font-size: large;" id='pntsToOvertakeNext'></strong>
</span>
</div>
</div>
</div>
<!-- /.navbar-header -->

11
templates/index.html
View File

@ -107,7 +107,7 @@ table {
}
.textTopHeader {
height: 50px;
height: 50px;
text-align: center;
margin-left: 8px;
float: left;
@ -132,6 +132,7 @@ small {
<ul class="dropdown-menu">
<li><a href="{{ url_for('index') }}">MISP Live Dashboard</a></li>
<li><a href="{{ url_for('geo') }}">MISP Geolocalisation</a></li>
<li><a href="{{ url_for('contrib') }}">MISP Contributors</a></li>
</ul>
<div id="ledsHolder" style="float: right; height: 50px;"></div>
@ -146,7 +147,7 @@ small {
<div class="row">
<div class="col-lg-12">
<div class="col-lg-{{ size_dashboard_width[0] }}">
<div id="panelOpenMap" class="panel panel-default" style="margin-top: 15px; height: {{ pannelSize[0] }}vh;">
<div class="panel-heading bg-info" style="font-weight: bold;">
<b><a id="textMap1" href="#panelbody">No map </a></b>
@ -191,7 +192,7 @@ small {
<!-- /.col-lg-6 -->
<!-- /.col-lg-6 -->
<div class="col-lg-{{ size_dashboard_width[1] }}">
<div class="panel panel-default" style="margin-top: 15px; height: {{ pannelSize[2] }}vh;">
<div id="panelbody" class="panel-body" style="height: 100%;">
<div id="feedDiv2" style="width:100%; height: 100%; position: relative;"></div>
@ -231,7 +232,7 @@ small {
<div class="row">
<div class="col-lg-12">
</div>
<!-- /.col-lg-12 -->
</div>
@ -247,7 +248,7 @@ small {
<!-- /#wrapper -->
<!-- Index -->
<script>
<script>
/* URL */
var urlForLogs = "{{ url_for('logs') }}";
var urlForHead = "{{ url_for('getLogHead') }}";

71
zmq_subscriber.py
View File

@ -14,6 +14,9 @@ import sys
import json
import geoip2.database
import util
import contributor_helper
configfile = os.path.join(os.environ['DASH_CONFIG'], 'config.cfg')
cfg = configparser.ConfigParser()
cfg.read(configfile)
@ -52,22 +55,19 @@ contributor_helper = contributor_helper.Contributor_helper(serv_redis_db, cfg)
reader = geoip2.database.Reader(PATH_TO_DB)
def getDateStrFormat(date):
return str(date.year)+str(date.month).zfill(2)+str(date.day).zfill(2)
def publish_log(zmq_name, name, content, channel=CHANNEL):
to_send = { 'name': name, 'log': json.dumps(content), 'zmqName': zmq_name }
serv_log.publish(channel, json.dumps(to_send))
def push_to_redis_zset(keyCateg, toAdd, endSubkey="", count=1):
now = datetime.datetime.now()
today_str = getDateStrFormat(now)
today_str = util.getDateStrFormat(now)
keyname = "{}:{}{}".format(keyCateg, today_str, endSubkey)
serv_redis_db.zincrby(keyname, toAdd, count)
def push_to_redis_geo(keyCateg, lon, lat, content):
now = datetime.datetime.now()
today_str = getDateStrFormat(now)
today_str = util.getDateStrFormat(now)
keyname = "{}:{}".format(keyCateg, today_str)
serv_redis_db.geoadd(keyname, lon, lat, content)
@ -127,7 +127,7 @@ def noSpaceLower(str):
return str.lower().replace(' ', '_')
#pntMultiplier if one contribution rewards more than others. (e.g. shighting may gives more points than editing)
def handleContribution(zmq_name, org, categ, action, pntMultiplier=1):
def handleContribution(zmq_name, org, categ, action, pntMultiplier=1, eventTime=datetime.datetime.now(), isLabeled=False):
if action in ['edit']:
pass
#return #not a contribution?
@ -145,58 +145,13 @@ def handleContribution(zmq_name, org, categ, action, pntMultiplier=1):
now = datetime.datetime.now()
nowSec = int(time.time())
serv_redis_db.zadd('CONTRIB_LAST:'+getDateStrFormat(now), nowSec, org)
serv_redis_db.expire('CONTRIB_LAST:'+getDateStrFormat(now), ONE_DAY) #expire after 1 day
serv_redis_db.zadd('CONTRIB_LAST:'+util.getDateStrFormat(now), nowSec, org)
serv_redis_db.expire('CONTRIB_LAST:'+util.getDateStrFormat(now), ONE_DAY) #expire after 1 day
updateOrgRank(org, pnts_to_add, eventTime, eventClassification)
contributor_helper.updateOrgContributionRank(org, pnts_to_add, eventTime, eventTime=datetime.datetime.now(), isLabeled=isLabeled)
publish_log(zmq_name, 'CONTRIBUTION', {'org': org, 'categ': categ, 'action': action, 'epoch': nowSec }, channel=CHANNEL_LASTCONTRIB)
def updateOrgRank(orgName, pnts_to_add, contribType, eventTime, isClassified):
keyname = 'CONTRIB_ORG:{org}:{orgCateg}'
#update total points
serv_redis_db.set(keyname.format(org=orgName, orgCateg='points'), pnts_to_add)
#update contribution Requirement
heavilyCount = contributor_helper.heavilyCount
recentDays = contributor_helper.recentDays
regularlyDays = contributor_helper.regularlyDays
isRecent = True if (datetime.datetime.now() - eventTime).days > recentDays
contrib = [] #[[contrib_level, contrib_ttl], [], ...]
if contribType == 'sighting':
#[contrib_level, contrib_ttl]
contrib.append([1, ONE_DAY*365]])
if contribType == 'attribute' or contribType == 'object':
contrib.append([2, ONE_DAY*365])
if contribType == 'proposal' or contribType == 'discussion':
contrib.append([3, ONE_DAY*365])
if contribType == 'sighting' and isRecent:
contrib.append([4, ONE_DAY*recentDays])
if contribType == 'proposal' and isRecent:
contrib.append([5, ONE_DAY*recentDays])
if contribType == 'event':
contrib.append([6, ONE_DAY*365])
if contribType == 'event':
contrib.append([7, ONE_DAY*recentDays])
if contribType == 'event':
contrib.append([8, ONE_DAY*regularlyDays])
if contribType == 'event' and isClassified:
contrib.append([9, ONE_DAY*regularlyDays])
if contribType == 'sighting' and sightingWeekCount>heavilyCount:
contrib.append([10, ONE_DAY*regularlyDays])
if (contribType == 'attribute' or contribType == 'object') and attributeWeekCount>heavilyCount:
contrib.append([11, ONE_DAY*regularlyDays])
if contribType == 'proposal' and proposalWeekCount>heavilyCount:
contrib.append([12, ONE_DAY*regularlyDays])
if contribType == 'event' and eventWeekCount>heavilyCount:
contrib.append([13, ONE_DAY*regularlyDays])
if contribType == 'event' and eventWeekCount>heavilyCount and isClassified:
contrib.append([14, ONE_DAY*regularlyDays])
for rankReq, ttl in contrib:
serv_redis_db.set(keyname.format(org=orgName, orgCateg='CONTRIB_REQ_'+str(rankReq)), 1)
serv_redis_db.expire(keyname.format(org=orgName, orgCateg='CONTRIB_REQ_'+str(i)), ttl)
##############
## HANDLERS ##
@ -257,11 +212,17 @@ def handler_attribute(zmq_name, jsonobj):
if jsonattr['category'] == "Network activity":
getCoordAndPublish(zmq_name, jsonattr['value'], jsonattr['category'])
handleContribution(zmq_name, jsonobj['Event']['Orgc']['name'], jsonattr['category'], jsonobj['action'])
eventLabeled = False
#eventLabeled = len(jsonattr[]) > 0
handleContribution(zmq_name, jsonobj['Event']['Orgc']['name'], jsonattr['category'], jsonobj['action'], isLabeled=eventLabeled)
# Push to log
publish_log(zmq_name, 'Attribute', to_push)
###############
## MAIN LOOP ##
###############
def process_log(zmq_name, event):
event = event.decode('utf8')
topic, eventdata = event.split(' ', maxsplit=1)