From 73fb3324d3ea2ea8c96b913c07eff2ebf4014785 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Tue, 5 Dec 2017 16:04:28 +0100 Subject: [PATCH 1/5] feature/refacto: Simplified users_helper script. Added possibility to get data from particular org (in helper and with typeahead). /!\ CHANGED DATABASE ORGANISATION --- helpers/contributor_helper.py | 3 +- helpers/users_helper.py | 120 ++++++++++++++++++++-------------- server.py | 16 ++--- static/js/users.js | 58 ++++++++++++++-- templates/users.html | 6 ++ 5 files changed, 135 insertions(+), 68 deletions(-) diff --git a/helpers/contributor_helper.py b/helpers/contributor_helper.py index 42ff5cf..b2b94d1 100644 --- a/helpers/contributor_helper.py +++ b/helpers/contributor_helper.py @@ -12,6 +12,7 @@ import redis import util from . import users_helper KEYDAY = "CONTRIB_DAY" # To be used by other module +KEYALLORG = "CONTRIB_ALL_ORG" # To be used by other module class Contributor_helper: def __init__(self, serv_redis_db, cfg): @@ -91,7 +92,7 @@ class Contributor_helper: self.keyDay = KEYDAY self.keyCateg = "CONTRIB_CATEG" self.keyLastContrib = "CONTRIB_LAST" - self.keyAllOrg = "CONTRIB_ALL_ORG" + self.keyAllOrg = KEYALLORG self.keyContribReq = "CONTRIB_ORG" self.keyTrophy = "CONTRIB_TROPHY" self.keyLastAward = "CONTRIB_LAST_AWARDS" diff --git a/helpers/users_helper.py b/helpers/users_helper.py index c865f21..c74cf36 100644 --- a/helpers/users_helper.py +++ b/helpers/users_helper.py @@ -13,10 +13,10 @@ class Users_helper: self.serv_redis_db = serv_redis_db self.cfg = cfg # REDIS keys - self.keyTimestamp = "LOGIN_TIMESTAMP" - self.keyTimestampSet = "LOGIN_TIMESTAMPSET" - self.keyOrgLog = "LOGIN_ORG" - self.keyContribDay = contributor_helper.KEYDAY # Key to get monthly contribution + self.keyTimestamp = "LOGIN_TIMESTAMP" + self.keyOrgLog = "LOGIN_ORG" + self.keyContribDay = contributor_helper.KEYDAY # Key to get monthly contribution + self.keyAllOrgLog = "LOGIN_ALL_ORG" # Key to get all organisation that logged in #logger logDir = cfg.get('Log', 'directory') @@ -27,58 +27,62 @@ class Users_helper: logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO) self.logger = logging.getLogger(__name__) - def addTemporary(self, org, timestamp): - timestampDate = datetime.datetime.fromtimestamp(float(timestamp)) - timestampDate_str = util.getDateHoursStrFormat(timestampDate) - keyname_timestamp = "{}:{}".format(self.keyTimestampSet, timestampDate_str) - self.serv_redis_db.sadd(keyname_timestamp, org) - self.logger.debug('Added to redis: keyname={}, org={}'.format(keyname_timestamp, org)) - self.serv_redis_db.expire(keyname_timestamp, 60*60) - - def hasAlreadyBeenAdded(self, org, timestamp): - timestampDate = datetime.datetime.fromtimestamp(float(timestamp)) - timestampDate_str = util.getDateHoursStrFormat(timestampDate) - keyname_timestamp = "{}:{}".format(self.keyTimestampSet, timestampDate_str) - orgs = [ org.decode('utf8') for org in self.serv_redis_db.smembers(keyname_timestamp) ] - if orgs is None: - return False - return (org in orgs) - def add_user_login(self, timestamp, org): timestampDate = datetime.datetime.fromtimestamp(float(timestamp)) timestampDate_str = util.getDateStrFormat(timestampDate) - if not self.hasAlreadyBeenAdded(org, timestamp): - keyname_timestamp = "{}:{}".format(self.keyTimestamp, timestampDate_str) - self.serv_redis_db.sadd(keyname_timestamp, timestamp) - self.logger.debug('Added to redis: keyname={}, org={}'.format(keyname_timestamp, timestamp)) - self.addTemporary(org, timestamp) + keyname_timestamp = "{}:{}".format(self.keyTimestamp, org) + self.serv_redis_db.zadd(keyname_timestamp, timestamp, timestamp) + self.logger.debug('Added to redis: keyname={}, org={}'.format(keyname_timestamp, timestamp)) keyname_org = "{}:{}".format(self.keyOrgLog, timestampDate_str) self.serv_redis_db.zincrby(keyname_org, org, 1) self.logger.debug('Added to redis: keyname={}, org={}'.format(keyname_org, org)) - def getUserLogins(self, date): - keyname = "{}:{}".format(self.keyTimestamp, util.getDateStrFormat(date)) - timestamps = self.serv_redis_db.smembers(keyname) - timestamps = [int(timestamp.decode('utf8')) for timestamp in timestamps] + self.serv_redis_db.sadd(self.keyAllOrgLog, org) + self.logger.debug('Added to redis: keyname={}, org={}'.format(self.keyAllOrgLog, org)) + + def getAllOrg(self): + temp = self.serv_redis_db.smembers(self.keyAllOrgLog) + return [ org.decode('utf8') for org in temp ] + + # return: All timestamps for one org for the spanned time or not + def getDates(self, org, date=None): + keyname = "{}:{}".format(self.keyTimestamp, org) + timestamps = self.serv_redis_db.zrange(keyname, 0, -1, desc=True, withscores=True) + if date is None: + to_return = [ t[1] for t in timestamps ] + else: + to_return = [] + for t in timestamps: + t = datetime.datetime.fromtimestamp(float(t[1])) + if util.getDateStrFormat(t) == util.getDateStrFormat(date): #same day + to_return.append(t) + else: + break # timestamps should be sorted, no need to process anymore + return to_return + + + # return: All dates for all orgs, if date is not supplied, return for all dates + def getUserLogins(self, date=None): + # get all orgs and retreive their timestamps + timestamps = [] + for org in self.getAllOrg(): + keyname = "{}:{}".format(self.keyOrgLog, org) + timestamps += self.getDates(org, date) return timestamps - def getOrgslogin(self, date, topNum=12): - keyname = "{}:{}".format(self.keyOrgLog, util.getDateStrFormat(date)) - data = self.serv_redis_db.zrange(keyname, 0, topNum-1, desc=True, withscores=True) - data = [ [record[0].decode('utf8'), record[1]] for record in data ] - return data - + # return: All orgs that logged in for the time spanned def getAllLoggedInOrgs(self, date, prev_days=31): orgs = set() for curDate in util.getXPrevDaysSpan(date, prev_days): keyname = "{}:{}".format(self.keyOrgLog, util.getDateStrFormat(curDate)) - data = self.serv_redis_db.zrange(keyname, 0, -1, desc=True, withscores=True) + data = self.serv_redis_db.zrange(keyname, 0, -1, desc=True) for org in data: - orgs.add(org[0].decode('utf8')) + orgs.add(org.decode('utf8')) return list(orgs) + # return: list composed of the number of [log, contrib] for one org for the time spanned def getOrgContribAndLogin(self, date, org, prev_days=31): keyname_log = "{}:{}" keyname_contrib = "{}:{}" @@ -91,6 +95,7 @@ class Users_helper: data.append([log, contrib]) return data + # return: the computed ratio of contribution/login for a given array def getContribOverLoginScore(self, array): totLog = 0 totContrib = 0 @@ -101,6 +106,7 @@ class Users_helper: totLog = 1 return totContrib/totLog + # return: list of org having the greatest ContribOverLoginScore for the time spanned def getTopOrglogin(self, date, maxNum=12, prev_days=7): all_logged_in_orgs = self.getAllLoggedInOrgs(date, prev_days) data = [] @@ -112,11 +118,13 @@ class Users_helper: return data[:maxNum] + # return: array composed of [number of org that contributed, number of org that logged in without contribution] + # for the spanned time def getLoginVSCOntribution(self, date): keyname = "{}:{}".format(self.keyContribDay, util.getDateStrFormat(date)) orgs_contri = self.serv_redis_db.zrange(keyname, 0, -1, desc=True, withscores=False) orgs_contri = [ org.decode('utf8') for org in orgs_contri ] - orgs_login = [ org[0] for org in self.getOrgslogin(date, topNum=0) ] + orgs_login = [ org for org in self.getAllLoggedInOrgs(date, prev_days=0) ] contributed_num = 0 non_contributed_num = 0 for org in orgs_login: @@ -127,13 +135,16 @@ class Users_helper: return [contributed_num, non_contributed_num] - def getUserLoginsForPunchCard(self, date, prev_days=6): + # return: list of day where day is a list of the number of time users logged in during an hour + def getUserLoginsForPunchCard(self, date, org=None, prev_days=6): week = {} for curDate in util.getXPrevDaysSpan(date, prev_days): - timestamps = self.getUserLogins(curDate) + if org is None: + dates = self.getUserLogins(curDate) + else: + dates = self.getDates(org, date=curDate) day = {} - for timestamp in timestamps: - date = datetime.datetime.fromtimestamp(float(timestamp)) + for date in dates: if date.hour not in day: day[date.hour] = 0 day[date.hour] += 1 @@ -156,7 +167,9 @@ class Users_helper: data = [data[6]]+data[:6] return data - def getUserLoginsAndContribOvertime(self, date, prev_days=6): + # return: a dico of the form {login: [[timestamp, count], ...], contrib: [[timestamp, 1/0], ...]} + # either for all orgs or the supplied one + def getUserLoginsAndContribOvertime(self, date, org=None, prev_days=6): dico_hours_contrib = {} dico_hours = {} for curDate in util.getXPrevHoursSpan(date, prev_days*24): @@ -164,17 +177,24 @@ class Users_helper: dico_hours_contrib[util.getTimestamp(curDate)] = 0 # populate with empty data for curDate in util.getXPrevDaysSpan(date, prev_days): - timestamps = self.getUserLogins(curDate) + if org is None: + dates = self.getUserLogins(curDate) + else: + dates = self.getDates(org, date=curDate) keyname = "{}:{}".format(self.keyContribDay, util.getDateStrFormat(curDate)) - orgs_contri = self.serv_redis_db.zrange(keyname, 0, -1, desc=True, withscores=False) - orgs_contri_num = len(orgs_contri) + if org is None: + orgs_contri = self.serv_redis_db.zrange(keyname, 0, -1, desc=True, withscores=False) + orgs_contri_num = len(orgs_contri) + else: + orgs_contri_num = self.serv_redis_db.zscore(keyname, org) + orgs_contri_num = 1 if orgs_contri_num is not None else 0 + for curDate in util.getHoursSpanOfDate(curDate, adaptToFitCurrentTime=True): #fill hole day dico_hours_contrib[util.getTimestamp(curDate)] = orgs_contri_num - for timestamp in timestamps: # sum occurence during the current hour - dateTimestamp = datetime.datetime.fromtimestamp(float(timestamp)) - dateTimestamp = dateTimestamp.replace(minute=0, second=0, microsecond=0) + for d in dates: # sum occurence during the current hour + dateTimestamp = d.replace(minute=0, second=0, microsecond=0) try: dico_hours[util.getTimestamp(dateTimestamp)] += 1 except KeyError: # timestamp out of bound (greater than 1 week) diff --git a/server.py b/server.py index f0ebb99..ee57001 100755 --- a/server.py +++ b/server.py @@ -424,17 +424,8 @@ def getUserLogins(): except: date = datetime.datetime.now() - data = users_helper.getUserLoginsForPunchCard(date) - return jsonify(data) - -@app.route("/_getUserLoginsOvertime") -def getUserLoginsOvertime(): - try: - date = datetime.datetime.fromtimestamp(float(request.args.get('date'))) - except: - date = datetime.datetime.now() - - data = users_helper.getUserLoginsOvertime(date) + org = request.args.get('org', None) + data = users_helper.getUserLoginsForPunchCard(date, org) return jsonify(data) @app.route("/_getTopOrglogin") @@ -464,7 +455,8 @@ def getUserLoginsAndContribOvertime(): except: date = datetime.datetime.now() - data = users_helper.getUserLoginsAndContribOvertime(date) + org = request.args.get('org', None) + data = users_helper.getUserLoginsAndContribOvertime(date, org) return jsonify(data) ''' TRENDINGS ''' diff --git a/static/js/users.js b/static/js/users.js index 9bc625a..22ff0c3 100644 --- a/static/js/users.js +++ b/static/js/users.js @@ -3,6 +3,38 @@ var pieOrgWidget; var pieApiWidget; var overtimeWidget; var div_day; +var allOrg; + +var typeaheadOption_punch = { + source: function (query, process) { + if (allOrg === undefined) { // caching + return $.getJSON(url_getTypeaheadData, function (orgs) { + allOrg = orgs; + return process(orgs); + }); + } else { + return process(allOrg); + } + }, + updater: function(org) { + updateDatePunch(undefined, undefined, org); + } +} +var typeaheadOption_overtime = { + source: function (query, process) { + if (allOrg === undefined) { // caching + return $.getJSON(url_getTypeaheadData, function (orgs) { + allOrg = orgs; + return process(orgs); + }); + } else { + return process(allOrg); + } + }, + updater: function(org) { + updateDateOvertime(undefined, undefined, org); + } +} function legendFormatter(label, series) { // removing unwanted " @@ -32,9 +64,16 @@ function highlight_punchDay() { div_day.addClass('highlightDay') } -function updateDatePunch() { +function updateDatePunch(ignore1, igonre2, org) { //date picker sets ( String dateText, Object inst ) var date = datePickerWidgetPunch.datepicker( "getDate" ); - $.getJSON( url_getUserLogins+"?date="+date.getTime()/1000, function( data ) { + if (org === undefined){ + $('#typeaheadPunch').attr('placeholder', "Enter an organization"); + var url = url_getUserLogins+"?date="+date.getTime()/1000; + } else { + $('#typeaheadPunch').attr('placeholder', org); + var url = url_getUserLogins+"?date="+date.getTime()/1000+"&org="+org; + } + $.getJSON(url, function( data ) { if (!(punchcardWidget === undefined)) { punchcardWidget.settings.data = data; punchcardWidget.refresh(); @@ -115,7 +154,7 @@ function updateDatePieApi() { } }); } -function updateDateOvertime() { +function updateDateOvertime(ignore1, igonre2, org) { //date picker sets ( String dateText, Object inst ) var date = datePickerWidgetOvertime.datepicker( "getDate" ); var now = new Date(); if (date.toDateString() == now.toDateString()) { @@ -123,8 +162,14 @@ function updateDateOvertime() { } else { date.setTime(date.getTime() + (24*60*60*1000-1)); // include data of selected date } - $.getJSON( url_getUserLoginsAndContribOvertime+"?date="+parseInt(date.getTime()/1000), function( data ) { - console.log(data); + if (org === undefined){ + var url = url_getUserLoginsAndContribOvertime+"?date="+parseInt(date.getTime()/1000) + $('#typeaheadOvertime').attr('placeholder', "Enter an organization"); + } else { + var url = url_getUserLoginsAndContribOvertime+"?date="+parseInt(date.getTime()/1000)+"&org="+org; + $('#typeaheadOvertime').attr('placeholder', org); + } + $.getJSON( url, function( data ) { data_log = data['login']; data_contrib = data['contrib']; temp_log = []; @@ -203,6 +248,9 @@ $(document).ready(function () { updateDatePieApi(); updateDateOvertime(); + $('#typeaheadPunch').typeahead(typeaheadOption_punch); + $('#typeaheadOvertime').typeahead(typeaheadOption_overtime); + $("
").css({ position: "absolute", display: "none", diff --git a/templates/users.html b/templates/users.html index 224b5b0..a77096b 100644 --- a/templates/users.html +++ b/templates/users.html @@ -27,6 +27,7 @@ + @@ -130,6 +131,8 @@ small { Dates: + +
@@ -171,6 +174,8 @@ small { Dates: + +
@@ -196,6 +201,7 @@ small { var url_getTopOrglogin = "{{ url_for('getTopOrglogin') }}"; var url_getLoginVSCOntribution = "{{ url_for('getLoginVSCOntribution') }}"; var url_getUserLoginsAndContribOvertime = "{{ url_for('getUserLoginsAndContribOvertime') }}"; + var url_getTypeaheadData = "{{ url_for('getAllOrg') }}"; /* DATA FROM CONF */ From 2e74c418fb6714e8e03cc4e465fbeb88dedc1d0d Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Tue, 5 Dec 2017 16:28:58 +0100 Subject: [PATCH 2/5] update: database organisation in doc --- doc/DOC.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/DOC.md b/doc/DOC.md index 10ba74f..d2c9fbd 100644 --- a/doc/DOC.md +++ b/doc/DOC.md @@ -33,11 +33,11 @@ Real time data are sent to their respective server's Redis pubsub channel ## Users -| Module | Feature | Key name | Key type | Key content | -|---------------------|---------------------------------------|------------------------------------|----------|--------------------------| -| users_helper | Use to consider only one org per hour | ```LOGIN_TIMESTAMPSET:date_hour``` | set | org (TTL = 1 hour) | -| users_helper | Use to get when users connect to MISP | ```LOGIN_TIMESTAMP:date``` | set | timestamp | -| users_helper | When an org connects to MISP | ```LOGIN_ORG:date``` | zset | org | +| Module | Feature | Key name | Key type | Key content | +|---------------------|------------------------------------------|------------------------------------|----------|--------------------------| +| users_helper | Use to get when orgs connect to MISP | ```LOGIN_TIMESTAMP:org``` | zset | timestamp | +| users_helper | Num. of time an org connected on a date | ```LOGIN_ORG:date``` | zset | org | +| users_helper | All ORG that has logged | ```LOGIN_ALL_ORG``` | set | org | ## Trendings | Module | Feature | Key name | Key type | Key content | From a52bf6ced5d5db04498b094323f503d5a2198251 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Tue, 5 Dec 2017 16:31:08 +0100 Subject: [PATCH 3/5] update: Clear separation between login and contribution to have meaningful query and contribution/login score. --- helpers/contributor_helper.py | 4 ++-- helpers/users_helper.py | 16 +++++++++------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/helpers/contributor_helper.py b/helpers/contributor_helper.py index b2b94d1..10dac9a 100644 --- a/helpers/contributor_helper.py +++ b/helpers/contributor_helper.py @@ -124,8 +124,8 @@ class Contributor_helper: nowSec = int(time.time()) pnts_to_add = self.default_pnts_per_contribution - # if there is a contribution, there is a login (even if it comes from the API) - self.users_helper.add_user_login(nowSec, org) + # Do not consider contribution as login anymore + #self.users_helper.add_user_login(nowSec, org) # is a valid contribution if categ is not None: diff --git a/helpers/users_helper.py b/helpers/users_helper.py index c74cf36..551a351 100644 --- a/helpers/users_helper.py +++ b/helpers/users_helper.py @@ -66,11 +66,11 @@ class Users_helper: # return: All dates for all orgs, if date is not supplied, return for all dates def getUserLogins(self, date=None): # get all orgs and retreive their timestamps - timestamps = [] + dates = [] for org in self.getAllOrg(): keyname = "{}:{}".format(self.keyOrgLog, org) - timestamps += self.getDates(org, date) - return timestamps + dates += self.getDates(org, date) + return dates # return: All orgs that logged in for the time spanned def getAllLoggedInOrgs(self, date, prev_days=31): @@ -91,7 +91,7 @@ class Users_helper: log = self.serv_redis_db.zscore(keyname_log.format(self.keyOrgLog, util.getDateStrFormat(curDate)), org) log = 0 if log is None else 1 contrib = self.serv_redis_db.zscore(keyname_contrib.format(self.keyContribDay, util.getDateStrFormat(curDate)), org) - contrib = 0 if contrib is None else 1 + contrib = 0 if contrib is None else contrib data.append([log, contrib]) return data @@ -184,11 +184,13 @@ class Users_helper: keyname = "{}:{}".format(self.keyContribDay, util.getDateStrFormat(curDate)) if org is None: - orgs_contri = self.serv_redis_db.zrange(keyname, 0, -1, desc=True, withscores=False) - orgs_contri_num = len(orgs_contri) + orgs_contri = self.serv_redis_db.zrange(keyname, 0, -1, desc=True, withscores=True) + orgs_contri_num = 0 + for org, count in orgs_contri: + orgs_contri_num += count else: orgs_contri_num = self.serv_redis_db.zscore(keyname, org) - orgs_contri_num = 1 if orgs_contri_num is not None else 0 + orgs_contri_num = orgs_contri_num if orgs_contri_num is not None else 0 for curDate in util.getHoursSpanOfDate(curDate, adaptToFitCurrentTime=True): #fill hole day dico_hours_contrib[util.getTimestamp(curDate)] = orgs_contri_num From bb3ccf28158c07fc8ef9c448df9aee769590d572 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Wed, 6 Dec 2017 16:51:29 +0100 Subject: [PATCH 4/5] fix: contribution is correctly taken for the entire time span instead of just one day if no org is specified --- helpers/users_helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/users_helper.py b/helpers/users_helper.py index 551a351..459708b 100644 --- a/helpers/users_helper.py +++ b/helpers/users_helper.py @@ -186,7 +186,7 @@ class Users_helper: if org is None: orgs_contri = self.serv_redis_db.zrange(keyname, 0, -1, desc=True, withscores=True) orgs_contri_num = 0 - for org, count in orgs_contri: + for _, count in orgs_contri: orgs_contri_num += count else: orgs_contri_num = self.serv_redis_db.zscore(keyname, org) From d8d9d5a145f0c4ed57806ea9f8674f423c75abd3 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Thu, 7 Dec 2017 09:17:22 +0100 Subject: [PATCH 5/5] fix: getDates was omitting dates --- helpers/users_helper.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/helpers/users_helper.py b/helpers/users_helper.py index 459708b..815a8e3 100644 --- a/helpers/users_helper.py +++ b/helpers/users_helper.py @@ -58,6 +58,8 @@ class Users_helper: t = datetime.datetime.fromtimestamp(float(t[1])) if util.getDateStrFormat(t) == util.getDateStrFormat(date): #same day to_return.append(t) + elif util.getDateStrFormat(t) > util.getDateStrFormat(date): + continue # timestamps should be sorted, skipping to reach wanted date else: break # timestamps should be sorted, no need to process anymore return to_return