Merge branch 'usersUpdate' into testScripts

pull/29/head
Sami Mokaddem 2017-12-07 10:10:27 +01:00
commit 15e199d86c
6 changed files with 147 additions and 76 deletions

View File

@ -33,11 +33,11 @@ Real time data are sent to their respective server's Redis pubsub channel
## Users
| Module | Feature | Key name | Key type | Key content |
|---------------------|---------------------------------------|------------------------------------|----------|--------------------------|
| users_helper | Use to consider only one org per hour | ```LOGIN_TIMESTAMPSET:date_hour``` | set | org (TTL = 1 hour) |
| users_helper | Use to get when users connect to MISP | ```LOGIN_TIMESTAMP:date``` | set | timestamp |
| users_helper | When an org connects to MISP | ```LOGIN_ORG:date``` | zset | org |
| Module | Feature | Key name | Key type | Key content |
|---------------------|------------------------------------------|------------------------------------|----------|--------------------------|
| users_helper | Use to get when orgs connect to MISP | ```LOGIN_TIMESTAMP:org``` | zset | timestamp |
| users_helper | Num. of time an org connected on a date | ```LOGIN_ORG:date``` | zset | org |
| users_helper | All ORG that has logged | ```LOGIN_ALL_ORG``` | set | org |
## Trendings
| Module | Feature | Key name | Key type | Key content |

View File

@ -12,6 +12,7 @@ import redis
import util
from . import users_helper
KEYDAY = "CONTRIB_DAY" # To be used by other module
KEYALLORG = "CONTRIB_ALL_ORG" # To be used by other module
class Contributor_helper:
def __init__(self, serv_redis_db, cfg):
@ -91,7 +92,7 @@ class Contributor_helper:
self.keyDay = KEYDAY
self.keyCateg = "CONTRIB_CATEG"
self.keyLastContrib = "CONTRIB_LAST"
self.keyAllOrg = "CONTRIB_ALL_ORG"
self.keyAllOrg = KEYALLORG
self.keyContribReq = "CONTRIB_ORG"
self.keyTrophy = "CONTRIB_TROPHY"
self.keyLastAward = "CONTRIB_LAST_AWARDS"
@ -123,8 +124,8 @@ class Contributor_helper:
nowSec = int(time.time())
pnts_to_add = self.default_pnts_per_contribution
# if there is a contribution, there is a login (even if it comes from the API)
self.users_helper.add_user_login(nowSec, org)
# Do not consider contribution as login anymore
#self.users_helper.add_user_login(nowSec, org)
# is a valid contribution
if categ is not None:

View File

@ -13,10 +13,10 @@ class Users_helper:
self.serv_redis_db = serv_redis_db
self.cfg = cfg
# REDIS keys
self.keyTimestamp = "LOGIN_TIMESTAMP"
self.keyTimestampSet = "LOGIN_TIMESTAMPSET"
self.keyOrgLog = "LOGIN_ORG"
self.keyContribDay = contributor_helper.KEYDAY # Key to get monthly contribution
self.keyTimestamp = "LOGIN_TIMESTAMP"
self.keyOrgLog = "LOGIN_ORG"
self.keyContribDay = contributor_helper.KEYDAY # Key to get monthly contribution
self.keyAllOrgLog = "LOGIN_ALL_ORG" # Key to get all organisation that logged in
#logger
logDir = cfg.get('Log', 'directory')
@ -27,58 +27,64 @@ class Users_helper:
logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO)
self.logger = logging.getLogger(__name__)
def addTemporary(self, org, timestamp):
timestampDate = datetime.datetime.fromtimestamp(float(timestamp))
timestampDate_str = util.getDateHoursStrFormat(timestampDate)
keyname_timestamp = "{}:{}".format(self.keyTimestampSet, timestampDate_str)
self.serv_redis_db.sadd(keyname_timestamp, org)
self.logger.debug('Added to redis: keyname={}, org={}'.format(keyname_timestamp, org))
self.serv_redis_db.expire(keyname_timestamp, 60*60)
def hasAlreadyBeenAdded(self, org, timestamp):
timestampDate = datetime.datetime.fromtimestamp(float(timestamp))
timestampDate_str = util.getDateHoursStrFormat(timestampDate)
keyname_timestamp = "{}:{}".format(self.keyTimestampSet, timestampDate_str)
orgs = [ org.decode('utf8') for org in self.serv_redis_db.smembers(keyname_timestamp) ]
if orgs is None:
return False
return (org in orgs)
def add_user_login(self, timestamp, org):
timestampDate = datetime.datetime.fromtimestamp(float(timestamp))
timestampDate_str = util.getDateStrFormat(timestampDate)
if not self.hasAlreadyBeenAdded(org, timestamp):
keyname_timestamp = "{}:{}".format(self.keyTimestamp, timestampDate_str)
self.serv_redis_db.sadd(keyname_timestamp, timestamp)
self.logger.debug('Added to redis: keyname={}, org={}'.format(keyname_timestamp, timestamp))
self.addTemporary(org, timestamp)
keyname_timestamp = "{}:{}".format(self.keyTimestamp, org)
self.serv_redis_db.zadd(keyname_timestamp, timestamp, timestamp)
self.logger.debug('Added to redis: keyname={}, org={}'.format(keyname_timestamp, timestamp))
keyname_org = "{}:{}".format(self.keyOrgLog, timestampDate_str)
self.serv_redis_db.zincrby(keyname_org, org, 1)
self.logger.debug('Added to redis: keyname={}, org={}'.format(keyname_org, org))
def getUserLogins(self, date):
keyname = "{}:{}".format(self.keyTimestamp, util.getDateStrFormat(date))
timestamps = self.serv_redis_db.smembers(keyname)
timestamps = [int(timestamp.decode('utf8')) for timestamp in timestamps]
return timestamps
self.serv_redis_db.sadd(self.keyAllOrgLog, org)
self.logger.debug('Added to redis: keyname={}, org={}'.format(self.keyAllOrgLog, org))
def getOrgslogin(self, date, topNum=12):
keyname = "{}:{}".format(self.keyOrgLog, util.getDateStrFormat(date))
data = self.serv_redis_db.zrange(keyname, 0, topNum-1, desc=True, withscores=True)
data = [ [record[0].decode('utf8'), record[1]] for record in data ]
return data
def getAllOrg(self):
temp = self.serv_redis_db.smembers(self.keyAllOrgLog)
return [ org.decode('utf8') for org in temp ]
# return: All timestamps for one org for the spanned time or not
def getDates(self, org, date=None):
keyname = "{}:{}".format(self.keyTimestamp, org)
timestamps = self.serv_redis_db.zrange(keyname, 0, -1, desc=True, withscores=True)
if date is None:
to_return = [ t[1] for t in timestamps ]
else:
to_return = []
for t in timestamps:
t = datetime.datetime.fromtimestamp(float(t[1]))
if util.getDateStrFormat(t) == util.getDateStrFormat(date): #same day
to_return.append(t)
elif util.getDateStrFormat(t) > util.getDateStrFormat(date):
continue # timestamps should be sorted, skipping to reach wanted date
else:
break # timestamps should be sorted, no need to process anymore
return to_return
# return: All dates for all orgs, if date is not supplied, return for all dates
def getUserLogins(self, date=None):
# get all orgs and retreive their timestamps
dates = []
for org in self.getAllOrg():
keyname = "{}:{}".format(self.keyOrgLog, org)
dates += self.getDates(org, date)
return dates
# return: All orgs that logged in for the time spanned
def getAllLoggedInOrgs(self, date, prev_days=31):
orgs = set()
for curDate in util.getXPrevDaysSpan(date, prev_days):
keyname = "{}:{}".format(self.keyOrgLog, util.getDateStrFormat(curDate))
data = self.serv_redis_db.zrange(keyname, 0, -1, desc=True, withscores=True)
data = self.serv_redis_db.zrange(keyname, 0, -1, desc=True)
for org in data:
orgs.add(org[0].decode('utf8'))
orgs.add(org.decode('utf8'))
return list(orgs)
# return: list composed of the number of [log, contrib] for one org for the time spanned
def getOrgContribAndLogin(self, date, org, prev_days=31):
keyname_log = "{}:{}"
keyname_contrib = "{}:{}"
@ -87,10 +93,11 @@ class Users_helper:
log = self.serv_redis_db.zscore(keyname_log.format(self.keyOrgLog, util.getDateStrFormat(curDate)), org)
log = 0 if log is None else 1
contrib = self.serv_redis_db.zscore(keyname_contrib.format(self.keyContribDay, util.getDateStrFormat(curDate)), org)
contrib = 0 if contrib is None else 1
contrib = 0 if contrib is None else contrib
data.append([log, contrib])
return data
# return: the computed ratio of contribution/login for a given array
def getContribOverLoginScore(self, array):
totLog = 0
totContrib = 0
@ -101,6 +108,7 @@ class Users_helper:
totLog = 1
return totContrib/totLog
# return: list of org having the greatest ContribOverLoginScore for the time spanned
def getTopOrglogin(self, date, maxNum=12, prev_days=7):
all_logged_in_orgs = self.getAllLoggedInOrgs(date, prev_days)
data = []
@ -112,11 +120,13 @@ class Users_helper:
return data[:maxNum]
# return: array composed of [number of org that contributed, number of org that logged in without contribution]
# for the spanned time
def getLoginVSCOntribution(self, date):
keyname = "{}:{}".format(self.keyContribDay, util.getDateStrFormat(date))
orgs_contri = self.serv_redis_db.zrange(keyname, 0, -1, desc=True, withscores=False)
orgs_contri = [ org.decode('utf8') for org in orgs_contri ]
orgs_login = [ org[0] for org in self.getOrgslogin(date, topNum=0) ]
orgs_login = [ org for org in self.getAllLoggedInOrgs(date, prev_days=0) ]
contributed_num = 0
non_contributed_num = 0
for org in orgs_login:
@ -127,13 +137,16 @@ class Users_helper:
return [contributed_num, non_contributed_num]
def getUserLoginsForPunchCard(self, date, prev_days=6):
# return: list of day where day is a list of the number of time users logged in during an hour
def getUserLoginsForPunchCard(self, date, org=None, prev_days=6):
week = {}
for curDate in util.getXPrevDaysSpan(date, prev_days):
timestamps = self.getUserLogins(curDate)
if org is None:
dates = self.getUserLogins(curDate)
else:
dates = self.getDates(org, date=curDate)
day = {}
for timestamp in timestamps:
date = datetime.datetime.fromtimestamp(float(timestamp))
for date in dates:
if date.hour not in day:
day[date.hour] = 0
day[date.hour] += 1
@ -156,7 +169,9 @@ class Users_helper:
data = [data[6]]+data[:6]
return data
def getUserLoginsAndContribOvertime(self, date, prev_days=6):
# return: a dico of the form {login: [[timestamp, count], ...], contrib: [[timestamp, 1/0], ...]}
# either for all orgs or the supplied one
def getUserLoginsAndContribOvertime(self, date, org=None, prev_days=6):
dico_hours_contrib = {}
dico_hours = {}
for curDate in util.getXPrevHoursSpan(date, prev_days*24):
@ -164,17 +179,26 @@ class Users_helper:
dico_hours_contrib[util.getTimestamp(curDate)] = 0 # populate with empty data
for curDate in util.getXPrevDaysSpan(date, prev_days):
timestamps = self.getUserLogins(curDate)
if org is None:
dates = self.getUserLogins(curDate)
else:
dates = self.getDates(org, date=curDate)
keyname = "{}:{}".format(self.keyContribDay, util.getDateStrFormat(curDate))
orgs_contri = self.serv_redis_db.zrange(keyname, 0, -1, desc=True, withscores=False)
orgs_contri_num = len(orgs_contri)
if org is None:
orgs_contri = self.serv_redis_db.zrange(keyname, 0, -1, desc=True, withscores=True)
orgs_contri_num = 0
for _, count in orgs_contri:
orgs_contri_num += count
else:
orgs_contri_num = self.serv_redis_db.zscore(keyname, org)
orgs_contri_num = orgs_contri_num if orgs_contri_num is not None else 0
for curDate in util.getHoursSpanOfDate(curDate, adaptToFitCurrentTime=True): #fill hole day
dico_hours_contrib[util.getTimestamp(curDate)] = orgs_contri_num
for timestamp in timestamps: # sum occurence during the current hour
dateTimestamp = datetime.datetime.fromtimestamp(float(timestamp))
dateTimestamp = dateTimestamp.replace(minute=0, second=0, microsecond=0)
for d in dates: # sum occurence during the current hour
dateTimestamp = d.replace(minute=0, second=0, microsecond=0)
try:
dico_hours[util.getTimestamp(dateTimestamp)] += 1
except KeyError: # timestamp out of bound (greater than 1 week)

View File

@ -424,17 +424,8 @@ def getUserLogins():
except:
date = datetime.datetime.now()
data = users_helper.getUserLoginsForPunchCard(date)
return jsonify(data)
@app.route("/_getUserLoginsOvertime")
def getUserLoginsOvertime():
try:
date = datetime.datetime.fromtimestamp(float(request.args.get('date')))
except:
date = datetime.datetime.now()
data = users_helper.getUserLoginsOvertime(date)
org = request.args.get('org', None)
data = users_helper.getUserLoginsForPunchCard(date, org)
return jsonify(data)
@app.route("/_getTopOrglogin")
@ -464,7 +455,8 @@ def getUserLoginsAndContribOvertime():
except:
date = datetime.datetime.now()
data = users_helper.getUserLoginsAndContribOvertime(date)
org = request.args.get('org', None)
data = users_helper.getUserLoginsAndContribOvertime(date, org)
return jsonify(data)
''' TRENDINGS '''

View File

@ -3,6 +3,38 @@ var pieOrgWidget;
var pieApiWidget;
var overtimeWidget;
var div_day;
var allOrg;
var typeaheadOption_punch = {
source: function (query, process) {
if (allOrg === undefined) { // caching
return $.getJSON(url_getTypeaheadData, function (orgs) {
allOrg = orgs;
return process(orgs);
});
} else {
return process(allOrg);
}
},
updater: function(org) {
updateDatePunch(undefined, undefined, org);
}
}
var typeaheadOption_overtime = {
source: function (query, process) {
if (allOrg === undefined) { // caching
return $.getJSON(url_getTypeaheadData, function (orgs) {
allOrg = orgs;
return process(orgs);
});
} else {
return process(allOrg);
}
},
updater: function(org) {
updateDateOvertime(undefined, undefined, org);
}
}
function legendFormatter(label, series) {
// removing unwanted "
@ -32,9 +64,16 @@ function highlight_punchDay() {
div_day.addClass('highlightDay')
}
function updateDatePunch() {
function updateDatePunch(ignore1, igonre2, org) { //date picker sets ( String dateText, Object inst )
var date = datePickerWidgetPunch.datepicker( "getDate" );
$.getJSON( url_getUserLogins+"?date="+date.getTime()/1000, function( data ) {
if (org === undefined){
$('#typeaheadPunch').attr('placeholder', "Enter an organization");
var url = url_getUserLogins+"?date="+date.getTime()/1000;
} else {
$('#typeaheadPunch').attr('placeholder', org);
var url = url_getUserLogins+"?date="+date.getTime()/1000+"&org="+org;
}
$.getJSON(url, function( data ) {
if (!(punchcardWidget === undefined)) {
punchcardWidget.settings.data = data;
punchcardWidget.refresh();
@ -115,7 +154,7 @@ function updateDatePieApi() {
}
});
}
function updateDateOvertime() {
function updateDateOvertime(ignore1, igonre2, org) { //date picker sets ( String dateText, Object inst )
var date = datePickerWidgetOvertime.datepicker( "getDate" );
var now = new Date();
if (date.toDateString() == now.toDateString()) {
@ -123,8 +162,14 @@ function updateDateOvertime() {
} else {
date.setTime(date.getTime() + (24*60*60*1000-1)); // include data of selected date
}
$.getJSON( url_getUserLoginsAndContribOvertime+"?date="+parseInt(date.getTime()/1000), function( data ) {
console.log(data);
if (org === undefined){
var url = url_getUserLoginsAndContribOvertime+"?date="+parseInt(date.getTime()/1000)
$('#typeaheadOvertime').attr('placeholder', "Enter an organization");
} else {
var url = url_getUserLoginsAndContribOvertime+"?date="+parseInt(date.getTime()/1000)+"&org="+org;
$('#typeaheadOvertime').attr('placeholder', org);
}
$.getJSON( url, function( data ) {
data_log = data['login'];
data_contrib = data['contrib'];
temp_log = [];
@ -203,6 +248,9 @@ $(document).ready(function () {
updateDatePieApi();
updateDateOvertime();
$('#typeaheadPunch').typeahead(typeaheadOption_punch);
$('#typeaheadOvertime').typeahead(typeaheadOption_overtime);
$("<div id='tooltip'></div>").css({
position: "absolute",
display: "none",

View File

@ -27,6 +27,7 @@
<script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
<!-- Bootstrap Core JavaScript -->
<script src="{{ url_for('static', filename='js/bootstrap.js') }}"></script>
<script src="{{ url_for('static', filename='js/bootstrap3-typeahead.min.js') }}"></script>
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="text/css">
<link rel="stylesheet" href="{{ url_for('static', filename='css/jquery-jvectormap-2.0.3.css') }}" type="text/css" media="screen"/>
@ -130,6 +131,8 @@ small {
<strong class='leftSepa textTopHeader' style="float: none; padding: 11px;">Dates:
<input type="text" id="datepickerPunch" size="10" style="">
</strong>
<a href="#" style="margin-top: 5px; float: right;" onclick="updateDatePunch();"><span class="glyphicon glyphicon-trash"></span></a>
<input type="text" id="typeaheadPunch" data-provide="typeahead" size="20" style="margin-bottom: 5px; float:right;" placeholder="Enter an organization">
</div>
<div id="panelbody" class="panel-body" style="">
<div id="punchcard" style="width:100%; height: 100%;"></div>
@ -171,6 +174,8 @@ small {
<strong class='leftSepa textTopHeader' style="float: none; padding: 11px;">Dates:
<input type="text" id="datepickerOvertimeLogin" size="10" style="">
</strong>
<a href="#" style="margin-top: 5px; float: right;" onclick="updateDateOvertime();"><span class="glyphicon glyphicon-trash"></span></a>
<input type="text" id="typeaheadOvertime" data-provide="typeahead" size="20" style="margin-bottom: 5px; float:right;" placeholder="Enter an organization">
</div>
<div id="panelbody" class="panel-body" style="">
<div id="lineChart" style="width:100%; height: 20vh;"></div>
@ -196,6 +201,7 @@ small {
var url_getTopOrglogin = "{{ url_for('getTopOrglogin') }}";
var url_getLoginVSCOntribution = "{{ url_for('getLoginVSCOntribution') }}";
var url_getUserLoginsAndContribOvertime = "{{ url_for('getUserLoginsAndContribOvertime') }}";
var url_getTypeaheadData = "{{ url_for('getAllOrg') }}";
/* DATA FROM CONF */