mirror of https://github.com/MISP/misp-dashboard
Draft getting contribution info from redis
Sami Mokaddem
parent
9f6638bf29
commit
264a4e52e2
228
server.py
228
server.py
|
|
@ -29,6 +29,7 @@ serv_redis_db = redis.StrictRedis(
|
|||
db=cfg.getint('RedisDB', 'db'))
|
||||
|
||||
categories_in_datatable = json.loads(cfg.get('CONTRIB', 'categories_in_datatable'))
|
||||
MAX_NUMBER_OF_LAST_CONTRIBUTOR = cfg.getint('CONTRIB', 'max_number_of_last_contributor')
|
||||
|
||||
subscriber_log = redis_server_log.pubsub(ignore_subscribe_messages=True)
|
||||
subscriber_log.psubscribe(cfg.get('RedisLog', 'channel'))
|
||||
|
|
@ -104,9 +105,14 @@ class EventMessage():
|
|||
def getRankLevel(points):
|
||||
if points == 0:
|
||||
return 0
|
||||
elif points == 1:
|
||||
return 1
|
||||
else:
|
||||
return float("{:.2f}".format(math.log(points, cfg.getfloat('CONTRIB' ,'rankMultiplier'))))
|
||||
|
||||
def getTrueRank(ptns):
|
||||
return int(getRankLevel(ptns))
|
||||
|
||||
def getRemainingPoints(points):
|
||||
prev = 0
|
||||
for i in [math.floor(cfg.getfloat('CONTRIB' ,'rankMultiplier')**x) for x in range(1,17)]:
|
||||
|
|
@ -117,14 +123,99 @@ def getRemainingPoints(points):
|
|||
|
||||
|
||||
''' GENERAL '''
|
||||
def getMonthSpan(date):
|
||||
ds = datetime.datetime(date.year, date.month, 1)
|
||||
dyear = 1 if ds.month+1 > 12 else 0
|
||||
dmonth = -12 if ds.month+1 > 12 else 0
|
||||
de = datetime.datetime(ds.year + dyear, ds.month+1 + dmonth, 1)
|
||||
|
||||
def getZrange(keyCateg, date, topNum):
|
||||
date_str = str(date.year)+str(date.month).zfill(2)+str(date.day).zfill(2)
|
||||
keyname = "{}:{}".format(keyCateg, date_str)
|
||||
data = serv_redis_db.zrange(keyname, 0, 5, desc=True, withscores=True)
|
||||
delta = de - ds
|
||||
to_return = []
|
||||
for i in range(delta.days):
|
||||
to_return.append(ds + datetime.timedelta(days=i))
|
||||
return to_return
|
||||
|
||||
def getDateStrFormat(date):
|
||||
return str(date.year)+str(date.month).zfill(2)+str(date.day).zfill(2)
|
||||
|
||||
def getZrange(keyCateg, date, topNum, endSubkey=""):
|
||||
date_str = getDateStrFormat(date)
|
||||
keyname = "{}:{}{}".format(keyCateg, date_str, endSubkey)
|
||||
data = serv_redis_db.zrange(keyname, 0, topNum-1, desc=True, withscores=True)
|
||||
data = [ [record[0].decode('utf8'), record[1]] for record in data ]
|
||||
return data
|
||||
|
||||
def getOrgPntFromRedis(org, date):
|
||||
keyCateg = 'CONTRIB_DAY'
|
||||
scoreSum = 0
|
||||
for curDate in getMonthSpan(date):
|
||||
date_str = getDateStrFormat(curDate)
|
||||
keyname = "{}:{}".format(keyCateg, date_str)
|
||||
data = serv_redis_db.zscore(keyname, org)
|
||||
if data is None:
|
||||
data = 0
|
||||
scoreSum += data
|
||||
return scoreSum
|
||||
|
||||
def getOrgRankFromRedis(org, date):
|
||||
ptns = getOrgPntFromRedis(org, date)
|
||||
return getTrueRank(ptns)
|
||||
|
||||
def getOrgLogoFromRedis(org):
|
||||
return 'logo_'+org
|
||||
|
||||
def getTopContributor_fromRedis(date):
|
||||
data2 = [
|
||||
{
|
||||
'rank': random.randint(1,16),
|
||||
'logo_path': 'logo1',
|
||||
'org': 'CIRCL',
|
||||
},
|
||||
{
|
||||
'rank': random.randint(1,16),
|
||||
'logo_path': 'logo2',
|
||||
'org': 'CASES',
|
||||
},
|
||||
{
|
||||
'rank': random.randint(1,16),
|
||||
'logo_path': 'logo3',
|
||||
'org': 'SMILE',
|
||||
},
|
||||
{
|
||||
'rank': random.randint(1,16),
|
||||
'logo_path': 'logo4',
|
||||
'org': 'ORG4',
|
||||
},
|
||||
{
|
||||
'rank': random.randint(1,16),
|
||||
'logo_path': 'logo5',
|
||||
'org': 'ORG5',
|
||||
},
|
||||
]
|
||||
|
||||
orgDicoPnts = {}
|
||||
for curDate in getMonthSpan(date):
|
||||
keyCateg = "CONTRIB_DAY"
|
||||
topNum = 0 # all
|
||||
contrib_org = getZrange(keyCateg, curDate, topNum)
|
||||
for org, pnts in contrib_org:
|
||||
if org not in orgDicoPnts:
|
||||
orgDicoPnts[org] = 0
|
||||
orgDicoPnts[org] += pnts
|
||||
|
||||
data = []
|
||||
for org, pnts in orgDicoPnts.items():
|
||||
dic = {}
|
||||
dic['rank'] = getTrueRank(pnts)
|
||||
dic['logo_path'] = getOrgLogoFromRedis(org)
|
||||
dic['org'] = org
|
||||
dic['pnts'] = pnts
|
||||
data.append(dic)
|
||||
data.sort(key=lambda x: x['pnts'], reverse=True)
|
||||
|
||||
return data
|
||||
#return data2
|
||||
|
||||
###########
|
||||
## ROUTE ##
|
||||
###########
|
||||
|
|
@ -215,7 +306,7 @@ def getHitMap():
|
|||
except:
|
||||
date = datetime.datetime.now()
|
||||
keyCateg = "GEO_COUNTRY"
|
||||
topNum = -1 # default Num
|
||||
topNum = 0 # all
|
||||
data = getZrange(keyCateg, date, topNum)
|
||||
return jsonify(data)
|
||||
|
||||
|
|
@ -243,7 +334,7 @@ def getCoordsByRadius():
|
|||
delta = dateEnd - dateStart
|
||||
for i in range(delta.days+1):
|
||||
correctDatetime = dateStart + datetime.timedelta(days=i)
|
||||
date_str = str(correctDatetime.year)+str(correctDatetime.month).zfill(2)+str(correctDatetime.day).zfill(2)
|
||||
date_str = getDateStrFormat(correctDatetime)
|
||||
keyCateg = 'GEO_RAD'
|
||||
keyname = "{}:{}".format(keyCateg, date_str)
|
||||
res = serv_redis_db.georadius(keyname, centerLon, centerLat, radius, unit='km', withcoord=True)
|
||||
|
|
@ -275,7 +366,7 @@ def getCoordsByRadius():
|
|||
|
||||
@app.route("/_getLastContributor")
|
||||
def getLastContributor():
|
||||
data = [
|
||||
data2 = [
|
||||
{
|
||||
'rank': random.randint(1,16),
|
||||
'logo_path': 'logo1',
|
||||
|
|
@ -302,48 +393,77 @@ def getLastContributor():
|
|||
'org': 'ORG5',
|
||||
},
|
||||
]
|
||||
return jsonify(data*2)
|
||||
try:
|
||||
date = datetime.datetime.fromtimestamp(float(request.args.get('date')))
|
||||
except:
|
||||
date = datetime.datetime.now()
|
||||
keyCateg = "CONTRIB_LAST"
|
||||
topNum = MAX_NUMBER_OF_LAST_CONTRIBUTOR # default Num
|
||||
last_contrib_org = getZrange(keyCateg, date, topNum)
|
||||
data = []
|
||||
for org, sec in last_contrib_org:
|
||||
dic = {}
|
||||
dic['rank'] = getOrgRankFromRedis(org, datetime.datetime.now())
|
||||
dic['logo_path'] = getOrgLogoFromRedis(org)
|
||||
dic['org'] = org
|
||||
dic['pnts'] = getOrgPntFromRedis(org, date)
|
||||
data.append(dic)
|
||||
return jsonify(data)
|
||||
#return jsonify(data2*2)
|
||||
|
||||
@app.route("/_getTopContributor")
|
||||
def getTopContributor():
|
||||
data = [
|
||||
{
|
||||
'rank': random.randint(1,16),
|
||||
'logo_path': 'logo1',
|
||||
'org': 'CIRCL',
|
||||
},
|
||||
{
|
||||
'rank': random.randint(1,16),
|
||||
'logo_path': 'logo2',
|
||||
'org': 'CASES',
|
||||
},
|
||||
{
|
||||
'rank': random.randint(1,16),
|
||||
'logo_path': 'logo3',
|
||||
'org': 'SMILE',
|
||||
},
|
||||
{
|
||||
'rank': random.randint(1,16),
|
||||
'logo_path': 'logo4',
|
||||
'org': 'ORG4',
|
||||
},
|
||||
{
|
||||
'rank': random.randint(1,16),
|
||||
'logo_path': 'logo5',
|
||||
'org': 'ORG5',
|
||||
},
|
||||
]
|
||||
return jsonify(data*2)
|
||||
def getTopContributor(suppliedDate=None):
|
||||
if suppliedDate is None:
|
||||
try:
|
||||
date = datetime.datetime.fromtimestamp(float(request.args.get('date')))
|
||||
except:
|
||||
date = datetime.datetime.now()
|
||||
else:
|
||||
date = suppliedDate
|
||||
|
||||
data = getTopContributor_fromRedis(date)
|
||||
return jsonify(data)
|
||||
|
||||
@app.route("/_getFameContributor")
|
||||
def getFameContributor():
|
||||
try:
|
||||
date = datetime.datetime.fromtimestamp(float(request.args.get('date')))
|
||||
except:
|
||||
today = datetime.datetime.now()
|
||||
# get previous month
|
||||
date = (datetime.datetime(today.year, today.month, 1) - datetime.timedelta(days=1))
|
||||
return getTopContributor(date)
|
||||
|
||||
|
||||
@app.route("/_getTop5Overtime")
|
||||
def getTop5Overtime():
|
||||
data = [{'label': 'CIRCL', 'data': [[0, 4], [1, 7], [2,14]]}, {'label': 'CASES', 'data': [[0, 1], [1, 5], [2,2]]}]
|
||||
data2 = [
|
||||
{'label': 'CIRCL', 'data': [[0, 4], [1, 7], [2,14]]},
|
||||
{'label': 'CASES', 'data': [[0, 1], [1, 5], [2,2]]}
|
||||
]
|
||||
data = []
|
||||
today = datetime.datetime.now()
|
||||
topSortedOrg = getTopContributor_fromRedis(today) #Get current top
|
||||
# show current top 5 org points overtime (last 5 days)
|
||||
for dic in topSortedOrg[0:5]:
|
||||
org = dic['org']
|
||||
overtime = []
|
||||
for deltaD in range(1,6,1):
|
||||
date = (datetime.datetime(today.year, today.month, today.day) - datetime.timedelta(days=deltaD))
|
||||
keyname = 'CONTRIB_DAY:'+getDateStrFormat(date)
|
||||
org_score = serv_redis_db.zscore(keyname, org)
|
||||
if org_score is None:
|
||||
org_score = 0
|
||||
overtime.append([deltaD, org_score])
|
||||
to_append = {'label': org, 'data': overtime}
|
||||
data.append(to_append)
|
||||
return jsonify(data)
|
||||
#return jsonify(data2)
|
||||
|
||||
@app.route("/_getCategPerContrib")
|
||||
def getCategPerContrib():
|
||||
|
||||
data = []
|
||||
data2 = []
|
||||
for d in range(15):
|
||||
dic = {}
|
||||
dic['rank'] = random.randint(1,16)
|
||||
|
|
@ -351,14 +471,40 @@ def getCategPerContrib():
|
|||
dic['org'] = 'Org'+str(d)
|
||||
for f in categories_in_datatable:
|
||||
dic[f] = random.randint(0,1600)
|
||||
data2.append(dic)
|
||||
|
||||
try:
|
||||
date = datetime.datetime.fromtimestamp(float(request.args.get('date')))
|
||||
except:
|
||||
date = datetime.datetime.now()
|
||||
keyCateg = "CONTRIB_DAY"
|
||||
topNum = 0 # all
|
||||
contrib_org = getZrange(keyCateg, date, topNum)
|
||||
data = []
|
||||
for org, pnts in contrib_org:
|
||||
dic = {}
|
||||
dic['rank'] = getTrueRank(pnts)
|
||||
dic['logo_path'] = getOrgLogoFromRedis(org)
|
||||
dic['org'] = org
|
||||
dic['pnts'] = pnts
|
||||
for categ in categories_in_datatable:
|
||||
keyname = 'CONTRIB_CATEG:'+getDateStrFormat(date)+':'+categ
|
||||
categ_score = serv_redis_db.zscore(keyname, org)
|
||||
if categ_score is None:
|
||||
categ_score = 0
|
||||
dic[categ] = categ_score
|
||||
data.append(dic)
|
||||
|
||||
return jsonify(data)
|
||||
return jsonify(data2)
|
||||
|
||||
@app.route("/_getAllOrg")
|
||||
def getAllOrg():
|
||||
data = ['CIRCL', 'CASES', 'SMILE' ,'ORG4' ,'ORG5', 'SUPER HYPER LONG ORGINZATION NAME', 'Org3']
|
||||
data = serv_redis_db.smembers('CONTRIB_ALL_ORG')
|
||||
data = [x.decode('utf8') for x in data]
|
||||
data2 = ['CIRCL', 'CASES', 'SMILE' ,'ORG4' ,'ORG5', 'SUPER HYPER LONG ORGINZATION NAME', 'Org3']
|
||||
return jsonify(data)
|
||||
#return jsonify(data2)
|
||||
|
||||
@app.route("/_getOrgRank")
|
||||
def getOrgRank():
|
||||
|
|
@ -366,7 +512,9 @@ def getOrgRank():
|
|||
org = request.args.get('org')
|
||||
except:
|
||||
org = ''
|
||||
date = datetime.datetime.now()
|
||||
points = random.randint(1,math.floor(cfg.getfloat('CONTRIB' ,'rankMultiplier')**16))
|
||||
points = getOrgPntFromRedis(org, date)
|
||||
#FIXME put 0 if org has no points
|
||||
remainingPts = getRemainingPoints(points)
|
||||
data = {'org': org,
|
||||
|
|
|
|||
|
|
@ -197,7 +197,7 @@ function addToTableFromJson(datatable, url) {
|
|||
var row = data[i];
|
||||
i = parseInt(i);
|
||||
var to_add = [
|
||||
i+1,
|
||||
row.pnts,
|
||||
getRankIcon(row.rank),
|
||||
row.logo_path,
|
||||
row.org
|
||||
|
|
@ -211,7 +211,6 @@ function addToTableFromJson(datatable, url) {
|
|||
function updateProgressHeader(org) {
|
||||
// get Org rank
|
||||
$.getJSON( url_getOrgRank+'?org='+org, function( data ) {
|
||||
console.log(data);
|
||||
datatableTop.draw();
|
||||
var rank = Math.floor(data.rank);
|
||||
var rankDec = data.rank-rank;
|
||||
|
|
@ -262,9 +261,9 @@ $(document).ready(function() {
|
|||
// top contributors
|
||||
addToTableFromJson(datatableTop, url_getTopContributor);
|
||||
// hall of fame
|
||||
addToTableFromJson(datatableFame, url_getTopContributor);
|
||||
addToTableFromJson(datatableFame, url_getFameContributor);
|
||||
// last contributors
|
||||
addToTableFromJson(datatableLast, url_getTopContributor);
|
||||
addToTableFromJson(datatableLast, url_getLastContributor);
|
||||
// category per contributors
|
||||
$.getJSON( url_getCategPerContrib, function( data ) {
|
||||
for (i in data) {
|
||||
|
|
@ -286,6 +285,7 @@ $(document).ready(function() {
|
|||
});
|
||||
// top 5 contrib overtime
|
||||
$.getJSON( url_getTop5Overtime, function( data ) {
|
||||
console.log(data);
|
||||
var plotLineChart = $.plot("#divTop5Overtime", data, optionsLineChart);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -99,7 +99,7 @@
|
|||
<table id="topContribTable" class="table table-hover table-striped">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>#</th>
|
||||
<th>Points</th>
|
||||
<th>Rank</th>
|
||||
<th>Logo</th>
|
||||
<th>Organisation</th>
|
||||
|
|
@ -126,7 +126,7 @@
|
|||
<table id="lastTable" class="table table-hover table-striped">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>#</th>
|
||||
<th>Points</th>
|
||||
<th>Rank</th>
|
||||
<th>Logo</th>
|
||||
<th>Organisation</th>
|
||||
|
|
@ -152,7 +152,7 @@
|
|||
<table id="categTable" class="table table-hover table-striped">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>#</th>
|
||||
<th>Points</th>
|
||||
<th>Rank</th>
|
||||
<th>Logo</th>
|
||||
<th>Organisation</th>
|
||||
|
|
@ -187,7 +187,7 @@
|
|||
<table id="fameTable" class="table table-hover table-striped" style="margin-bottom: 0px;">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>#</th>
|
||||
<th>Points</th>
|
||||
<th>Prev. rank</th>
|
||||
<th>Logo</th>
|
||||
<th>Organisation</th>
|
||||
|
|
@ -229,8 +229,10 @@
|
|||
<script>
|
||||
/* URL */
|
||||
var url_getTopContributor = "{{ url_for('getTopContributor') }}";
|
||||
var url_getFameContributor = "{{ url_for('getFameContributor') }}";
|
||||
var url_getCategPerContrib = "{{ url_for('getCategPerContrib') }}";
|
||||
var url_getTop5Overtime = "{{ url_for('getTop5Overtime') }}";
|
||||
var url_getLastContributor = "{{ url_for('getLastContributor') }}";
|
||||
var url_getAllOrg = "{{ url_for('getAllOrg') }}";
|
||||
var url_getOrgRank = "{{ url_for('getOrgRank') }}";
|
||||
var url_baseRankLogo = "{{ url_for('static', filename='pics/rankingMISP/1.png') }}";
|
||||
|
|
|
|||
|
|
@ -47,6 +47,8 @@ serv_redis_db = redis.StrictRedis(
|
|||
|
||||
reader = geoip2.database.Reader(PATH_TO_DB)
|
||||
|
||||
def getDateStrFormat(date):
|
||||
return str(date.year)+str(date.month).zfill(2)+str(date.day).zfill(2)
|
||||
|
||||
def publish_log(zmq_name, name, content):
|
||||
to_send = { 'name': name, 'log': json.dumps(content), 'zmqName': zmq_name }
|
||||
|
|
@ -54,13 +56,13 @@ def publish_log(zmq_name, name, content):
|
|||
|
||||
def push_to_redis_zset(keyCateg, toAdd, endSubkey="", count=1):
|
||||
now = datetime.datetime.now()
|
||||
today_str = str(now.year)+str(now.month).zfill(2)+str(now.day).zfill(2)
|
||||
today_str = getDateStrFormat(now)
|
||||
keyname = "{}:{}{}".format(keyCateg, today_str, endSubkey)
|
||||
serv_redis_db.zincrby(keyname, toAdd, count)
|
||||
|
||||
def push_to_redis_geo(keyCateg, lon, lat, content):
|
||||
now = datetime.datetime.now()
|
||||
today_str = str(now.year)+str(now.month).zfill(2)+str(now.day).zfill(2)
|
||||
today_str = getDateStrFormat(now)
|
||||
keyname = "{}:{}".format(keyCateg, today_str)
|
||||
serv_redis_db.geoadd(keyname, lon, lat, content)
|
||||
|
||||
|
|
@ -135,9 +137,11 @@ def handleContribution(org, categ, action, pntMultiplier=1):
|
|||
#CONTRIB_CATEG retain the contribution per category, not the point earned in this categ
|
||||
push_to_redis_zset('CONTRIB_CATEG', org, count=DEFAULT_PNTS_REWARD, endSubkey=':'+noSpaceLower(categ))
|
||||
serv_redis_db.sadd('CONTRIB_ALL_ORG', org)
|
||||
serv_redis_db.lpush('CONTRIB_LAST', org)
|
||||
serv_redis_db.ltrim('CONTRIB_LAST', 0, MAX_NUMBER_OF_LAST_CONTRIBUTOR-1) # Limit list size
|
||||
#serv_redis_db.lrange('CONTRIB_LAST', 0, MAX_NUMBER_OF_LAST_CONTRIBUTOR-1) # get the last 10 contributors
|
||||
|
||||
now = datetime.datetime.now()
|
||||
nowSec = int(time.time())
|
||||
serv_redis_db.zadd('CONTRIB_LAST:'+getDateStrFormat(now), nowSec, org)
|
||||
serv_redis_db.expire('CONTRIB_LAST:'+getDateStrFormat(now), 60*60*24) #expire after 1 day
|
||||
|
||||
##############
|
||||
## HANDLERS ##
|
||||
|
|
|
|||
Loading…
Reference in New Issue