mirror of https://github.com/MISP/misp-dashboard
Merge branch 'usersUpdate' into testScripts
Sami Mokaddem
commit
15e199d86c
10
doc/DOC.md
10
doc/DOC.md
|
|
@ -33,11 +33,11 @@ Real time data are sent to their respective server's Redis pubsub channel
|
|||
|
||||
## Users
|
||||
|
||||
| Module | Feature | Key name | Key type | Key content |
|
||||
|---------------------|---------------------------------------|------------------------------------|----------|--------------------------|
|
||||
| users_helper | Use to consider only one org per hour | ```LOGIN_TIMESTAMPSET:date_hour``` | set | org (TTL = 1 hour) |
|
||||
| users_helper | Use to get when users connect to MISP | ```LOGIN_TIMESTAMP:date``` | set | timestamp |
|
||||
| users_helper | When an org connects to MISP | ```LOGIN_ORG:date``` | zset | org |
|
||||
| Module | Feature | Key name | Key type | Key content |
|
||||
|---------------------|------------------------------------------|------------------------------------|----------|--------------------------|
|
||||
| users_helper | Use to get when orgs connect to MISP | ```LOGIN_TIMESTAMP:org``` | zset | timestamp |
|
||||
| users_helper | Num. of time an org connected on a date | ```LOGIN_ORG:date``` | zset | org |
|
||||
| users_helper | All ORG that has logged | ```LOGIN_ALL_ORG``` | set | org |
|
||||
|
||||
## Trendings
|
||||
| Module | Feature | Key name | Key type | Key content |
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ import redis
|
|||
import util
|
||||
from . import users_helper
|
||||
KEYDAY = "CONTRIB_DAY" # To be used by other module
|
||||
KEYALLORG = "CONTRIB_ALL_ORG" # To be used by other module
|
||||
|
||||
class Contributor_helper:
|
||||
def __init__(self, serv_redis_db, cfg):
|
||||
|
|
@ -91,7 +92,7 @@ class Contributor_helper:
|
|||
self.keyDay = KEYDAY
|
||||
self.keyCateg = "CONTRIB_CATEG"
|
||||
self.keyLastContrib = "CONTRIB_LAST"
|
||||
self.keyAllOrg = "CONTRIB_ALL_ORG"
|
||||
self.keyAllOrg = KEYALLORG
|
||||
self.keyContribReq = "CONTRIB_ORG"
|
||||
self.keyTrophy = "CONTRIB_TROPHY"
|
||||
self.keyLastAward = "CONTRIB_LAST_AWARDS"
|
||||
|
|
@ -123,8 +124,8 @@ class Contributor_helper:
|
|||
nowSec = int(time.time())
|
||||
pnts_to_add = self.default_pnts_per_contribution
|
||||
|
||||
# if there is a contribution, there is a login (even if it comes from the API)
|
||||
self.users_helper.add_user_login(nowSec, org)
|
||||
# Do not consider contribution as login anymore
|
||||
#self.users_helper.add_user_login(nowSec, org)
|
||||
|
||||
# is a valid contribution
|
||||
if categ is not None:
|
||||
|
|
|
|||
|
|
@ -13,10 +13,10 @@ class Users_helper:
|
|||
self.serv_redis_db = serv_redis_db
|
||||
self.cfg = cfg
|
||||
# REDIS keys
|
||||
self.keyTimestamp = "LOGIN_TIMESTAMP"
|
||||
self.keyTimestampSet = "LOGIN_TIMESTAMPSET"
|
||||
self.keyOrgLog = "LOGIN_ORG"
|
||||
self.keyContribDay = contributor_helper.KEYDAY # Key to get monthly contribution
|
||||
self.keyTimestamp = "LOGIN_TIMESTAMP"
|
||||
self.keyOrgLog = "LOGIN_ORG"
|
||||
self.keyContribDay = contributor_helper.KEYDAY # Key to get monthly contribution
|
||||
self.keyAllOrgLog = "LOGIN_ALL_ORG" # Key to get all organisation that logged in
|
||||
|
||||
#logger
|
||||
logDir = cfg.get('Log', 'directory')
|
||||
|
|
@ -27,58 +27,64 @@ class Users_helper:
|
|||
logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO)
|
||||
self.logger = logging.getLogger(__name__)
|
||||
|
||||
def addTemporary(self, org, timestamp):
|
||||
timestampDate = datetime.datetime.fromtimestamp(float(timestamp))
|
||||
timestampDate_str = util.getDateHoursStrFormat(timestampDate)
|
||||
keyname_timestamp = "{}:{}".format(self.keyTimestampSet, timestampDate_str)
|
||||
self.serv_redis_db.sadd(keyname_timestamp, org)
|
||||
self.logger.debug('Added to redis: keyname={}, org={}'.format(keyname_timestamp, org))
|
||||
self.serv_redis_db.expire(keyname_timestamp, 60*60)
|
||||
|
||||
def hasAlreadyBeenAdded(self, org, timestamp):
|
||||
timestampDate = datetime.datetime.fromtimestamp(float(timestamp))
|
||||
timestampDate_str = util.getDateHoursStrFormat(timestampDate)
|
||||
keyname_timestamp = "{}:{}".format(self.keyTimestampSet, timestampDate_str)
|
||||
orgs = [ org.decode('utf8') for org in self.serv_redis_db.smembers(keyname_timestamp) ]
|
||||
if orgs is None:
|
||||
return False
|
||||
return (org in orgs)
|
||||
|
||||
def add_user_login(self, timestamp, org):
|
||||
timestampDate = datetime.datetime.fromtimestamp(float(timestamp))
|
||||
timestampDate_str = util.getDateStrFormat(timestampDate)
|
||||
|
||||
if not self.hasAlreadyBeenAdded(org, timestamp):
|
||||
keyname_timestamp = "{}:{}".format(self.keyTimestamp, timestampDate_str)
|
||||
self.serv_redis_db.sadd(keyname_timestamp, timestamp)
|
||||
self.logger.debug('Added to redis: keyname={}, org={}'.format(keyname_timestamp, timestamp))
|
||||
self.addTemporary(org, timestamp)
|
||||
keyname_timestamp = "{}:{}".format(self.keyTimestamp, org)
|
||||
self.serv_redis_db.zadd(keyname_timestamp, timestamp, timestamp)
|
||||
self.logger.debug('Added to redis: keyname={}, org={}'.format(keyname_timestamp, timestamp))
|
||||
|
||||
keyname_org = "{}:{}".format(self.keyOrgLog, timestampDate_str)
|
||||
self.serv_redis_db.zincrby(keyname_org, org, 1)
|
||||
self.logger.debug('Added to redis: keyname={}, org={}'.format(keyname_org, org))
|
||||
|
||||
def getUserLogins(self, date):
|
||||
keyname = "{}:{}".format(self.keyTimestamp, util.getDateStrFormat(date))
|
||||
timestamps = self.serv_redis_db.smembers(keyname)
|
||||
timestamps = [int(timestamp.decode('utf8')) for timestamp in timestamps]
|
||||
return timestamps
|
||||
self.serv_redis_db.sadd(self.keyAllOrgLog, org)
|
||||
self.logger.debug('Added to redis: keyname={}, org={}'.format(self.keyAllOrgLog, org))
|
||||
|
||||
def getOrgslogin(self, date, topNum=12):
|
||||
keyname = "{}:{}".format(self.keyOrgLog, util.getDateStrFormat(date))
|
||||
data = self.serv_redis_db.zrange(keyname, 0, topNum-1, desc=True, withscores=True)
|
||||
data = [ [record[0].decode('utf8'), record[1]] for record in data ]
|
||||
return data
|
||||
def getAllOrg(self):
|
||||
temp = self.serv_redis_db.smembers(self.keyAllOrgLog)
|
||||
return [ org.decode('utf8') for org in temp ]
|
||||
|
||||
# return: All timestamps for one org for the spanned time or not
|
||||
def getDates(self, org, date=None):
|
||||
keyname = "{}:{}".format(self.keyTimestamp, org)
|
||||
timestamps = self.serv_redis_db.zrange(keyname, 0, -1, desc=True, withscores=True)
|
||||
if date is None:
|
||||
to_return = [ t[1] for t in timestamps ]
|
||||
else:
|
||||
to_return = []
|
||||
for t in timestamps:
|
||||
t = datetime.datetime.fromtimestamp(float(t[1]))
|
||||
if util.getDateStrFormat(t) == util.getDateStrFormat(date): #same day
|
||||
to_return.append(t)
|
||||
elif util.getDateStrFormat(t) > util.getDateStrFormat(date):
|
||||
continue # timestamps should be sorted, skipping to reach wanted date
|
||||
else:
|
||||
break # timestamps should be sorted, no need to process anymore
|
||||
return to_return
|
||||
|
||||
|
||||
# return: All dates for all orgs, if date is not supplied, return for all dates
|
||||
def getUserLogins(self, date=None):
|
||||
# get all orgs and retreive their timestamps
|
||||
dates = []
|
||||
for org in self.getAllOrg():
|
||||
keyname = "{}:{}".format(self.keyOrgLog, org)
|
||||
dates += self.getDates(org, date)
|
||||
return dates
|
||||
|
||||
# return: All orgs that logged in for the time spanned
|
||||
def getAllLoggedInOrgs(self, date, prev_days=31):
|
||||
orgs = set()
|
||||
for curDate in util.getXPrevDaysSpan(date, prev_days):
|
||||
keyname = "{}:{}".format(self.keyOrgLog, util.getDateStrFormat(curDate))
|
||||
data = self.serv_redis_db.zrange(keyname, 0, -1, desc=True, withscores=True)
|
||||
data = self.serv_redis_db.zrange(keyname, 0, -1, desc=True)
|
||||
for org in data:
|
||||
orgs.add(org[0].decode('utf8'))
|
||||
orgs.add(org.decode('utf8'))
|
||||
return list(orgs)
|
||||
|
||||
# return: list composed of the number of [log, contrib] for one org for the time spanned
|
||||
def getOrgContribAndLogin(self, date, org, prev_days=31):
|
||||
keyname_log = "{}:{}"
|
||||
keyname_contrib = "{}:{}"
|
||||
|
|
@ -87,10 +93,11 @@ class Users_helper:
|
|||
log = self.serv_redis_db.zscore(keyname_log.format(self.keyOrgLog, util.getDateStrFormat(curDate)), org)
|
||||
log = 0 if log is None else 1
|
||||
contrib = self.serv_redis_db.zscore(keyname_contrib.format(self.keyContribDay, util.getDateStrFormat(curDate)), org)
|
||||
contrib = 0 if contrib is None else 1
|
||||
contrib = 0 if contrib is None else contrib
|
||||
data.append([log, contrib])
|
||||
return data
|
||||
|
||||
# return: the computed ratio of contribution/login for a given array
|
||||
def getContribOverLoginScore(self, array):
|
||||
totLog = 0
|
||||
totContrib = 0
|
||||
|
|
@ -101,6 +108,7 @@ class Users_helper:
|
|||
totLog = 1
|
||||
return totContrib/totLog
|
||||
|
||||
# return: list of org having the greatest ContribOverLoginScore for the time spanned
|
||||
def getTopOrglogin(self, date, maxNum=12, prev_days=7):
|
||||
all_logged_in_orgs = self.getAllLoggedInOrgs(date, prev_days)
|
||||
data = []
|
||||
|
|
@ -112,11 +120,13 @@ class Users_helper:
|
|||
return data[:maxNum]
|
||||
|
||||
|
||||
# return: array composed of [number of org that contributed, number of org that logged in without contribution]
|
||||
# for the spanned time
|
||||
def getLoginVSCOntribution(self, date):
|
||||
keyname = "{}:{}".format(self.keyContribDay, util.getDateStrFormat(date))
|
||||
orgs_contri = self.serv_redis_db.zrange(keyname, 0, -1, desc=True, withscores=False)
|
||||
orgs_contri = [ org.decode('utf8') for org in orgs_contri ]
|
||||
orgs_login = [ org[0] for org in self.getOrgslogin(date, topNum=0) ]
|
||||
orgs_login = [ org for org in self.getAllLoggedInOrgs(date, prev_days=0) ]
|
||||
contributed_num = 0
|
||||
non_contributed_num = 0
|
||||
for org in orgs_login:
|
||||
|
|
@ -127,13 +137,16 @@ class Users_helper:
|
|||
return [contributed_num, non_contributed_num]
|
||||
|
||||
|
||||
def getUserLoginsForPunchCard(self, date, prev_days=6):
|
||||
# return: list of day where day is a list of the number of time users logged in during an hour
|
||||
def getUserLoginsForPunchCard(self, date, org=None, prev_days=6):
|
||||
week = {}
|
||||
for curDate in util.getXPrevDaysSpan(date, prev_days):
|
||||
timestamps = self.getUserLogins(curDate)
|
||||
if org is None:
|
||||
dates = self.getUserLogins(curDate)
|
||||
else:
|
||||
dates = self.getDates(org, date=curDate)
|
||||
day = {}
|
||||
for timestamp in timestamps:
|
||||
date = datetime.datetime.fromtimestamp(float(timestamp))
|
||||
for date in dates:
|
||||
if date.hour not in day:
|
||||
day[date.hour] = 0
|
||||
day[date.hour] += 1
|
||||
|
|
@ -156,7 +169,9 @@ class Users_helper:
|
|||
data = [data[6]]+data[:6]
|
||||
return data
|
||||
|
||||
def getUserLoginsAndContribOvertime(self, date, prev_days=6):
|
||||
# return: a dico of the form {login: [[timestamp, count], ...], contrib: [[timestamp, 1/0], ...]}
|
||||
# either for all orgs or the supplied one
|
||||
def getUserLoginsAndContribOvertime(self, date, org=None, prev_days=6):
|
||||
dico_hours_contrib = {}
|
||||
dico_hours = {}
|
||||
for curDate in util.getXPrevHoursSpan(date, prev_days*24):
|
||||
|
|
@ -164,17 +179,26 @@ class Users_helper:
|
|||
dico_hours_contrib[util.getTimestamp(curDate)] = 0 # populate with empty data
|
||||
|
||||
for curDate in util.getXPrevDaysSpan(date, prev_days):
|
||||
timestamps = self.getUserLogins(curDate)
|
||||
if org is None:
|
||||
dates = self.getUserLogins(curDate)
|
||||
else:
|
||||
dates = self.getDates(org, date=curDate)
|
||||
keyname = "{}:{}".format(self.keyContribDay, util.getDateStrFormat(curDate))
|
||||
|
||||
orgs_contri = self.serv_redis_db.zrange(keyname, 0, -1, desc=True, withscores=False)
|
||||
orgs_contri_num = len(orgs_contri)
|
||||
if org is None:
|
||||
orgs_contri = self.serv_redis_db.zrange(keyname, 0, -1, desc=True, withscores=True)
|
||||
orgs_contri_num = 0
|
||||
for _, count in orgs_contri:
|
||||
orgs_contri_num += count
|
||||
else:
|
||||
orgs_contri_num = self.serv_redis_db.zscore(keyname, org)
|
||||
orgs_contri_num = orgs_contri_num if orgs_contri_num is not None else 0
|
||||
|
||||
for curDate in util.getHoursSpanOfDate(curDate, adaptToFitCurrentTime=True): #fill hole day
|
||||
dico_hours_contrib[util.getTimestamp(curDate)] = orgs_contri_num
|
||||
|
||||
for timestamp in timestamps: # sum occurence during the current hour
|
||||
dateTimestamp = datetime.datetime.fromtimestamp(float(timestamp))
|
||||
dateTimestamp = dateTimestamp.replace(minute=0, second=0, microsecond=0)
|
||||
for d in dates: # sum occurence during the current hour
|
||||
dateTimestamp = d.replace(minute=0, second=0, microsecond=0)
|
||||
try:
|
||||
dico_hours[util.getTimestamp(dateTimestamp)] += 1
|
||||
except KeyError: # timestamp out of bound (greater than 1 week)
|
||||
|
|
|
|||
16
server.py
16
server.py
|
|
@ -424,17 +424,8 @@ def getUserLogins():
|
|||
except:
|
||||
date = datetime.datetime.now()
|
||||
|
||||
data = users_helper.getUserLoginsForPunchCard(date)
|
||||
return jsonify(data)
|
||||
|
||||
@app.route("/_getUserLoginsOvertime")
|
||||
def getUserLoginsOvertime():
|
||||
try:
|
||||
date = datetime.datetime.fromtimestamp(float(request.args.get('date')))
|
||||
except:
|
||||
date = datetime.datetime.now()
|
||||
|
||||
data = users_helper.getUserLoginsOvertime(date)
|
||||
org = request.args.get('org', None)
|
||||
data = users_helper.getUserLoginsForPunchCard(date, org)
|
||||
return jsonify(data)
|
||||
|
||||
@app.route("/_getTopOrglogin")
|
||||
|
|
@ -464,7 +455,8 @@ def getUserLoginsAndContribOvertime():
|
|||
except:
|
||||
date = datetime.datetime.now()
|
||||
|
||||
data = users_helper.getUserLoginsAndContribOvertime(date)
|
||||
org = request.args.get('org', None)
|
||||
data = users_helper.getUserLoginsAndContribOvertime(date, org)
|
||||
return jsonify(data)
|
||||
|
||||
''' TRENDINGS '''
|
||||
|
|
|
|||
|
|
@ -3,6 +3,38 @@ var pieOrgWidget;
|
|||
var pieApiWidget;
|
||||
var overtimeWidget;
|
||||
var div_day;
|
||||
var allOrg;
|
||||
|
||||
var typeaheadOption_punch = {
|
||||
source: function (query, process) {
|
||||
if (allOrg === undefined) { // caching
|
||||
return $.getJSON(url_getTypeaheadData, function (orgs) {
|
||||
allOrg = orgs;
|
||||
return process(orgs);
|
||||
});
|
||||
} else {
|
||||
return process(allOrg);
|
||||
}
|
||||
},
|
||||
updater: function(org) {
|
||||
updateDatePunch(undefined, undefined, org);
|
||||
}
|
||||
}
|
||||
var typeaheadOption_overtime = {
|
||||
source: function (query, process) {
|
||||
if (allOrg === undefined) { // caching
|
||||
return $.getJSON(url_getTypeaheadData, function (orgs) {
|
||||
allOrg = orgs;
|
||||
return process(orgs);
|
||||
});
|
||||
} else {
|
||||
return process(allOrg);
|
||||
}
|
||||
},
|
||||
updater: function(org) {
|
||||
updateDateOvertime(undefined, undefined, org);
|
||||
}
|
||||
}
|
||||
|
||||
function legendFormatter(label, series) {
|
||||
// removing unwanted "
|
||||
|
|
@ -32,9 +64,16 @@ function highlight_punchDay() {
|
|||
div_day.addClass('highlightDay')
|
||||
}
|
||||
|
||||
function updateDatePunch() {
|
||||
function updateDatePunch(ignore1, igonre2, org) { //date picker sets ( String dateText, Object inst )
|
||||
var date = datePickerWidgetPunch.datepicker( "getDate" );
|
||||
$.getJSON( url_getUserLogins+"?date="+date.getTime()/1000, function( data ) {
|
||||
if (org === undefined){
|
||||
$('#typeaheadPunch').attr('placeholder', "Enter an organization");
|
||||
var url = url_getUserLogins+"?date="+date.getTime()/1000;
|
||||
} else {
|
||||
$('#typeaheadPunch').attr('placeholder', org);
|
||||
var url = url_getUserLogins+"?date="+date.getTime()/1000+"&org="+org;
|
||||
}
|
||||
$.getJSON(url, function( data ) {
|
||||
if (!(punchcardWidget === undefined)) {
|
||||
punchcardWidget.settings.data = data;
|
||||
punchcardWidget.refresh();
|
||||
|
|
@ -115,7 +154,7 @@ function updateDatePieApi() {
|
|||
}
|
||||
});
|
||||
}
|
||||
function updateDateOvertime() {
|
||||
function updateDateOvertime(ignore1, igonre2, org) { //date picker sets ( String dateText, Object inst )
|
||||
var date = datePickerWidgetOvertime.datepicker( "getDate" );
|
||||
var now = new Date();
|
||||
if (date.toDateString() == now.toDateString()) {
|
||||
|
|
@ -123,8 +162,14 @@ function updateDateOvertime() {
|
|||
} else {
|
||||
date.setTime(date.getTime() + (24*60*60*1000-1)); // include data of selected date
|
||||
}
|
||||
$.getJSON( url_getUserLoginsAndContribOvertime+"?date="+parseInt(date.getTime()/1000), function( data ) {
|
||||
console.log(data);
|
||||
if (org === undefined){
|
||||
var url = url_getUserLoginsAndContribOvertime+"?date="+parseInt(date.getTime()/1000)
|
||||
$('#typeaheadOvertime').attr('placeholder', "Enter an organization");
|
||||
} else {
|
||||
var url = url_getUserLoginsAndContribOvertime+"?date="+parseInt(date.getTime()/1000)+"&org="+org;
|
||||
$('#typeaheadOvertime').attr('placeholder', org);
|
||||
}
|
||||
$.getJSON( url, function( data ) {
|
||||
data_log = data['login'];
|
||||
data_contrib = data['contrib'];
|
||||
temp_log = [];
|
||||
|
|
@ -203,6 +248,9 @@ $(document).ready(function () {
|
|||
updateDatePieApi();
|
||||
updateDateOvertime();
|
||||
|
||||
$('#typeaheadPunch').typeahead(typeaheadOption_punch);
|
||||
$('#typeaheadOvertime').typeahead(typeaheadOption_overtime);
|
||||
|
||||
$("<div id='tooltip'></div>").css({
|
||||
position: "absolute",
|
||||
display: "none",
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@
|
|||
<script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
|
||||
<!-- Bootstrap Core JavaScript -->
|
||||
<script src="{{ url_for('static', filename='js/bootstrap.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/bootstrap3-typeahead.min.js') }}"></script>
|
||||
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="text/css">
|
||||
|
||||
<link rel="stylesheet" href="{{ url_for('static', filename='css/jquery-jvectormap-2.0.3.css') }}" type="text/css" media="screen"/>
|
||||
|
|
@ -130,6 +131,8 @@ small {
|
|||
<strong class='leftSepa textTopHeader' style="float: none; padding: 11px;">Dates:
|
||||
<input type="text" id="datepickerPunch" size="10" style="">
|
||||
</strong>
|
||||
<a href="#" style="margin-top: 5px; float: right;" onclick="updateDatePunch();"><span class="glyphicon glyphicon-trash"></span></a>
|
||||
<input type="text" id="typeaheadPunch" data-provide="typeahead" size="20" style="margin-bottom: 5px; float:right;" placeholder="Enter an organization">
|
||||
</div>
|
||||
<div id="panelbody" class="panel-body" style="">
|
||||
<div id="punchcard" style="width:100%; height: 100%;"></div>
|
||||
|
|
@ -171,6 +174,8 @@ small {
|
|||
<strong class='leftSepa textTopHeader' style="float: none; padding: 11px;">Dates:
|
||||
<input type="text" id="datepickerOvertimeLogin" size="10" style="">
|
||||
</strong>
|
||||
<a href="#" style="margin-top: 5px; float: right;" onclick="updateDateOvertime();"><span class="glyphicon glyphicon-trash"></span></a>
|
||||
<input type="text" id="typeaheadOvertime" data-provide="typeahead" size="20" style="margin-bottom: 5px; float:right;" placeholder="Enter an organization">
|
||||
</div>
|
||||
<div id="panelbody" class="panel-body" style="">
|
||||
<div id="lineChart" style="width:100%; height: 20vh;"></div>
|
||||
|
|
@ -196,6 +201,7 @@ small {
|
|||
var url_getTopOrglogin = "{{ url_for('getTopOrglogin') }}";
|
||||
var url_getLoginVSCOntribution = "{{ url_for('getLoginVSCOntribution') }}";
|
||||
var url_getUserLoginsAndContribOvertime = "{{ url_for('getUserLoginsAndContribOvertime') }}";
|
||||
var url_getTypeaheadData = "{{ url_for('getAllOrg') }}";
|
||||
|
||||
/* DATA FROM CONF */
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue