Merge pull request #24 from mokaddem/usersUpdate

Update of users module

doc/DOC.md

@@ -34,10 +34,10 @@ Real time data are sent to their respective server's Redis pubsub channel
 ## Users
 
 | Module              | Feature                                  | Key name                           | Key type | Key content              |
-|---------------------|---------------------------------------|------------------------------------|----------|--------------------------|
+|---------------------|------------------------------------------|------------------------------------|----------|--------------------------|
-| users_helper        | Use to consider only one org per hour | ```LOGIN_TIMESTAMPSET:date_hour``` | set      | org (TTL = 1 hour)       |
+| users_helper        | Use to get when orgs connect to MISP     | ```LOGIN_TIMESTAMP:org```          | zset     | timestamp                |
-| users_helper        | Use to get when users connect to MISP | ```LOGIN_TIMESTAMP:date```         | set      | timestamp                |
+| users_helper        | Num. of time an org connected on a date  | ```LOGIN_ORG:date```               | zset     | org                      |
-| users_helper        | When an org connects to MISP          | ```LOGIN_ORG:date```               | zset     | org                      |
+| users_helper        | All ORG that has logged                  | ```LOGIN_ALL_ORG```                | set      | org                      |
 
 ## Trendings
 | Module              | Feature                               | Key name                           | Key type | Key content              |

helpers/contributor_helper.py

@@ -12,6 +12,7 @@ import redis
 import util
 from . import users_helper
 KEYDAY = "CONTRIB_DAY" # To be used by other module
+KEYALLORG = "CONTRIB_ALL_ORG" # To be used by other module
 
 class Contributor_helper:
     def __init__(self, serv_redis_db, cfg):
@@ -91,7 +92,7 @@ class Contributor_helper:
         self.keyDay         = KEYDAY
         self.keyCateg       = "CONTRIB_CATEG"
         self.keyLastContrib = "CONTRIB_LAST"
-        self.keyAllOrg      = "CONTRIB_ALL_ORG"
+        self.keyAllOrg      = KEYALLORG
         self.keyContribReq  = "CONTRIB_ORG"
         self.keyTrophy      = "CONTRIB_TROPHY"
         self.keyLastAward   = "CONTRIB_LAST_AWARDS"
@@ -123,8 +124,8 @@ class Contributor_helper:
         nowSec = int(time.time())
         pnts_to_add = self.default_pnts_per_contribution
     
-        # if there is a contribution, there is a login (even if it comes from the API)
+        # Do not consider contribution as login anymore
-        self.users_helper.add_user_login(nowSec, org)
+        #self.users_helper.add_user_login(nowSec, org)
     
         # is a valid contribution
         if categ is not None:

helpers/users_helper.py

@@ -14,9 +14,9 @@ class Users_helper:
         self.cfg = cfg
         # REDIS keys
         self.keyTimestamp     = "LOGIN_TIMESTAMP"
-        self.keyTimestampSet = "LOGIN_TIMESTAMPSET"
         self.keyOrgLog        = "LOGIN_ORG"
         self.keyContribDay    = contributor_helper.KEYDAY # Key to get monthly contribution
+        self.keyAllOrgLog     = "LOGIN_ALL_ORG" # Key to get all organisation that logged in
 
         #logger
         logDir = cfg.get('Log', 'directory')
@@ -27,58 +27,64 @@ class Users_helper:
         logging.basicConfig(filename=logPath, filemode='a', level=logging.INFO)
         self.logger = logging.getLogger(__name__)
 
-    def addTemporary(self, org, timestamp):
-        timestampDate = datetime.datetime.fromtimestamp(float(timestamp))
-        timestampDate_str = util.getDateHoursStrFormat(timestampDate)
-        keyname_timestamp = "{}:{}".format(self.keyTimestampSet, timestampDate_str)
-        self.serv_redis_db.sadd(keyname_timestamp, org)
-        self.logger.debug('Added to redis: keyname={}, org={}'.format(keyname_timestamp, org))
-        self.serv_redis_db.expire(keyname_timestamp, 60*60)
-
-    def hasAlreadyBeenAdded(self, org, timestamp):
-        timestampDate = datetime.datetime.fromtimestamp(float(timestamp))
-        timestampDate_str = util.getDateHoursStrFormat(timestampDate)
-        keyname_timestamp = "{}:{}".format(self.keyTimestampSet, timestampDate_str)
-        orgs = [ org.decode('utf8') for org in self.serv_redis_db.smembers(keyname_timestamp) ]
-        if orgs is None:
-            return False
-        return (org in orgs)
-
     def add_user_login(self, timestamp, org):
         timestampDate = datetime.datetime.fromtimestamp(float(timestamp))
         timestampDate_str = util.getDateStrFormat(timestampDate)
 
-        if not self.hasAlreadyBeenAdded(org, timestamp):
+        keyname_timestamp = "{}:{}".format(self.keyTimestamp, org)
-            keyname_timestamp = "{}:{}".format(self.keyTimestamp, timestampDate_str)
+        self.serv_redis_db.zadd(keyname_timestamp, timestamp, timestamp)
-            self.serv_redis_db.sadd(keyname_timestamp, timestamp)
         self.logger.debug('Added to redis: keyname={}, org={}'.format(keyname_timestamp, timestamp))
-            self.addTemporary(org, timestamp)
 
         keyname_org = "{}:{}".format(self.keyOrgLog, timestampDate_str)
         self.serv_redis_db.zincrby(keyname_org, org, 1)
         self.logger.debug('Added to redis: keyname={}, org={}'.format(keyname_org, org))
 
-    def getUserLogins(self, date):
+        self.serv_redis_db.sadd(self.keyAllOrgLog, org)
-        keyname = "{}:{}".format(self.keyTimestamp, util.getDateStrFormat(date))
+        self.logger.debug('Added to redis: keyname={}, org={}'.format(self.keyAllOrgLog, org))
-        timestamps = self.serv_redis_db.smembers(keyname)
-        timestamps = [int(timestamp.decode('utf8')) for timestamp in timestamps]
-        return timestamps
 
-    def getOrgslogin(self, date, topNum=12):
+    def getAllOrg(self):
-        keyname = "{}:{}".format(self.keyOrgLog, util.getDateStrFormat(date))
+        temp = self.serv_redis_db.smembers(self.keyAllOrgLog)
-        data = self.serv_redis_db.zrange(keyname, 0, topNum-1, desc=True, withscores=True)
+        return [ org.decode('utf8') for org in temp ]
-        data = [ [record[0].decode('utf8'), record[1]] for record in data ]
-        return data
 
+    # return: All timestamps for one org for the spanned time or not
+    def getDates(self, org, date=None):
+        keyname = "{}:{}".format(self.keyTimestamp, org)
+        timestamps = self.serv_redis_db.zrange(keyname, 0, -1, desc=True, withscores=True)
+        if date is None:
+            to_return = [ t[1] for t in timestamps ]
+        else:
+            to_return = []
+            for t in timestamps:
+                t = datetime.datetime.fromtimestamp(float(t[1]))
+                if util.getDateStrFormat(t) == util.getDateStrFormat(date): #same day
+                    to_return.append(t)
+                elif util.getDateStrFormat(t) > util.getDateStrFormat(date):
+                    continue # timestamps should be sorted, skipping to reach wanted date
+                else:
+                    break # timestamps should be sorted, no need to process anymore
+        return to_return
+                    
+
+    # return: All dates for all orgs, if date is not supplied, return for all dates
+    def getUserLogins(self, date=None):
+        # get all orgs and retreive their timestamps
+        dates = []
+        for org in self.getAllOrg():
+            keyname = "{}:{}".format(self.keyOrgLog, org)
+            dates += self.getDates(org, date)
+        return dates
+
+    # return: All orgs that logged in for the time spanned
     def getAllLoggedInOrgs(self, date, prev_days=31):
         orgs = set()
         for curDate in util.getXPrevDaysSpan(date, prev_days):
             keyname = "{}:{}".format(self.keyOrgLog, util.getDateStrFormat(curDate))
-            data = self.serv_redis_db.zrange(keyname, 0, -1, desc=True, withscores=True)
+            data = self.serv_redis_db.zrange(keyname, 0, -1, desc=True)
             for org in data:
-                orgs.add(org[0].decode('utf8'))
+                orgs.add(org.decode('utf8'))
         return list(orgs)
 
+    # return: list composed of the number of [log, contrib] for one org for the time spanned
     def getOrgContribAndLogin(self, date, org, prev_days=31):
         keyname_log = "{}:{}"
         keyname_contrib = "{}:{}"
@@ -87,10 +93,11 @@ class Users_helper:
             log = self.serv_redis_db.zscore(keyname_log.format(self.keyOrgLog, util.getDateStrFormat(curDate)), org)
             log = 0 if log is None else 1
             contrib = self.serv_redis_db.zscore(keyname_contrib.format(self.keyContribDay, util.getDateStrFormat(curDate)), org)
-            contrib = 0 if contrib is None else 1
+            contrib = 0 if contrib is None else contrib
             data.append([log, contrib])
         return data
 
+    # return: the computed ratio of contribution/login for a given array
     def getContribOverLoginScore(self, array):
         totLog = 0
         totContrib = 0
@@ -101,6 +108,7 @@ class Users_helper:
             totLog = 1
         return totContrib/totLog
 
+    # return: list of org having the greatest ContribOverLoginScore for the time spanned
     def getTopOrglogin(self, date, maxNum=12, prev_days=7):
         all_logged_in_orgs = self.getAllLoggedInOrgs(date, prev_days)
         data = []
@@ -112,11 +120,13 @@ class Users_helper:
         return data[:maxNum]
 
 
+    # return: array composed of [number of org that contributed, number of org that logged in without contribution]
+    #         for the spanned time
     def getLoginVSCOntribution(self, date):
         keyname = "{}:{}".format(self.keyContribDay, util.getDateStrFormat(date))
         orgs_contri = self.serv_redis_db.zrange(keyname, 0, -1, desc=True, withscores=False)
         orgs_contri = [ org.decode('utf8') for org in orgs_contri ]
-        orgs_login = [ org[0] for org in self.getOrgslogin(date, topNum=0) ]
+        orgs_login = [ org for org in self.getAllLoggedInOrgs(date, prev_days=0) ]
         contributed_num = 0
         non_contributed_num = 0
         for org in orgs_login:
@@ -127,13 +137,16 @@ class Users_helper:
         return [contributed_num, non_contributed_num]
 
 
-    def getUserLoginsForPunchCard(self, date, prev_days=6):
+    # return: list of day where day is a list of the number of time users logged in during an hour
+    def getUserLoginsForPunchCard(self, date, org=None, prev_days=6):
         week = {}
         for curDate in util.getXPrevDaysSpan(date, prev_days):
-            timestamps = self.getUserLogins(curDate)
+            if org is None:
+                dates = self.getUserLogins(curDate)
+            else:
+                dates = self.getDates(org, date=curDate)
             day = {}
-            for timestamp in timestamps:
+            for date in dates:
-                date = datetime.datetime.fromtimestamp(float(timestamp))
                 if date.hour not in day:
                     day[date.hour] = 0
                 day[date.hour] += 1
@@ -156,7 +169,9 @@ class Users_helper:
         data = [data[6]]+data[:6]
         return data
 
-    def getUserLoginsAndContribOvertime(self, date, prev_days=6):
+    # return: a dico of the form {login: [[timestamp, count], ...], contrib: [[timestamp, 1/0], ...]} 
+    #         either for all orgs or the supplied one
+    def getUserLoginsAndContribOvertime(self, date, org=None, prev_days=6):
         dico_hours_contrib = {}
         dico_hours = {}
         for curDate in util.getXPrevHoursSpan(date, prev_days*24):
@@ -164,17 +179,26 @@ class Users_helper:
             dico_hours_contrib[util.getTimestamp(curDate)] = 0 # populate with empty data
 
         for curDate in util.getXPrevDaysSpan(date, prev_days):
-            timestamps = self.getUserLogins(curDate)
+            if org is None:
+                dates = self.getUserLogins(curDate)
+            else:
+                dates = self.getDates(org, date=curDate)
             keyname = "{}:{}".format(self.keyContribDay, util.getDateStrFormat(curDate))
 
-            orgs_contri = self.serv_redis_db.zrange(keyname, 0, -1, desc=True, withscores=False)
+            if org is None:
-            orgs_contri_num = len(orgs_contri)
+                orgs_contri = self.serv_redis_db.zrange(keyname, 0, -1, desc=True, withscores=True)
+                orgs_contri_num = 0
+                for _, count in orgs_contri:
+                    orgs_contri_num += count
+            else:
+                orgs_contri_num = self.serv_redis_db.zscore(keyname, org)
+                orgs_contri_num = orgs_contri_num if orgs_contri_num is not None else 0
+
             for curDate in util.getHoursSpanOfDate(curDate, adaptToFitCurrentTime=True): #fill hole day
                 dico_hours_contrib[util.getTimestamp(curDate)] = orgs_contri_num
 
-            for timestamp in timestamps: # sum occurence during the current hour
+            for d in dates: # sum occurence during the current hour
-                dateTimestamp = datetime.datetime.fromtimestamp(float(timestamp))
+                dateTimestamp = d.replace(minute=0, second=0, microsecond=0)
-                dateTimestamp = dateTimestamp.replace(minute=0, second=0, microsecond=0)
                 try:
                     dico_hours[util.getTimestamp(dateTimestamp)] += 1
                 except KeyError: # timestamp out of bound (greater than 1 week)

server.py

@@ -427,17 +427,8 @@ def getUserLogins():
     except:
         date = datetime.datetime.now()
 
-    data = users_helper.getUserLoginsForPunchCard(date)
+    org = request.args.get('org', None)
-    return jsonify(data)
+    data = users_helper.getUserLoginsForPunchCard(date, org)
-
-@app.route("/_getUserLoginsOvertime")
-def getUserLoginsOvertime():
-    try:
-        date = datetime.datetime.fromtimestamp(float(request.args.get('date')))
-    except:
-        date = datetime.datetime.now()
-
-    data = users_helper.getUserLoginsOvertime(date)
     return jsonify(data)
 
 @app.route("/_getTopOrglogin")
@@ -467,7 +458,8 @@ def getUserLoginsAndContribOvertime():
     except:
         date = datetime.datetime.now()
 
-    data = users_helper.getUserLoginsAndContribOvertime(date)
+    org = request.args.get('org', None)
+    data = users_helper.getUserLoginsAndContribOvertime(date, org)
     return jsonify(data)
 
 ''' TRENDINGS '''

static/js/users.js

@@ -3,6 +3,38 @@ var pieOrgWidget;
 var pieApiWidget;
 var overtimeWidget;
 var div_day;
+var allOrg;
+
+var typeaheadOption_punch = {
+    source: function (query, process) {
+        if (allOrg === undefined) { // caching
+            return $.getJSON(url_getTypeaheadData, function (orgs) {
+                    allOrg = orgs;
+                    return process(orgs);
+            });
+        } else {
+            return process(allOrg);
+        }
+    },
+    updater: function(org) {
+        updateDatePunch(undefined, undefined, org);
+    }
+}
+var typeaheadOption_overtime = {
+    source: function (query, process) {
+        if (allOrg === undefined) { // caching
+            return $.getJSON(url_getTypeaheadData, function (orgs) {
+                    allOrg = orgs;
+                    return process(orgs);
+            });
+        } else {
+            return process(allOrg);
+        }
+    },
+    updater: function(org) {
+        updateDateOvertime(undefined, undefined, org);
+    }
+}
 
 function legendFormatter(label, series) {
     // removing unwanted "
@@ -32,9 +64,16 @@ function highlight_punchDay() {
     div_day.addClass('highlightDay')
 }
 
-function updateDatePunch() {
+function updateDatePunch(ignore1, igonre2, org) { //date picker sets ( String dateText, Object inst )
     var date = datePickerWidgetPunch.datepicker( "getDate" );
-    $.getJSON( url_getUserLogins+"?date="+date.getTime()/1000, function( data ) {
+    if (org === undefined){
+        $('#typeaheadPunch').attr('placeholder', "Enter an organization");
+        var url = url_getUserLogins+"?date="+date.getTime()/1000;
+    } else {
+        $('#typeaheadPunch').attr('placeholder', org);
+        var url = url_getUserLogins+"?date="+date.getTime()/1000+"&org="+org;
+    }
+    $.getJSON(url, function( data ) {
         if (!(punchcardWidget === undefined)) {
             punchcardWidget.settings.data = data;
             punchcardWidget.refresh();
@@ -115,7 +154,7 @@ function updateDatePieApi() {
         }
     });
 }
-function updateDateOvertime() {
+function updateDateOvertime(ignore1, igonre2, org) { //date picker sets ( String dateText, Object inst )
     var date = datePickerWidgetOvertime.datepicker( "getDate" );
     var now = new Date();
     if (date.toDateString() == now.toDateString()) {
@@ -123,8 +162,14 @@ function updateDateOvertime() {
     } else {
         date.setTime(date.getTime() + (24*60*60*1000-1)); // include data of selected date
     }
-    $.getJSON( url_getUserLoginsAndContribOvertime+"?date="+parseInt(date.getTime()/1000), function( data ) {
+    if (org === undefined){
-        console.log(data);
+        var url = url_getUserLoginsAndContribOvertime+"?date="+parseInt(date.getTime()/1000)
+        $('#typeaheadOvertime').attr('placeholder', "Enter an organization");
+    } else {
+        var url = url_getUserLoginsAndContribOvertime+"?date="+parseInt(date.getTime()/1000)+"&org="+org;
+        $('#typeaheadOvertime').attr('placeholder', org);
+    }
+    $.getJSON( url, function( data ) {
         data_log = data['login'];
         data_contrib = data['contrib'];
         temp_log = [];
@@ -203,6 +248,9 @@ $(document).ready(function () {
     updateDatePieApi();
     updateDateOvertime();
 
+    $('#typeaheadPunch').typeahead(typeaheadOption_punch);
+    $('#typeaheadOvertime').typeahead(typeaheadOption_overtime);
+
     $("<div id='tooltip'></div>").css({
         position: "absolute",
         display: "none",

templates/users.html

@@ -27,6 +27,7 @@
     <script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
     <!-- Bootstrap Core JavaScript -->
     <script src="{{ url_for('static', filename='js/bootstrap.js') }}"></script>
+    <script src="{{ url_for('static', filename='js/bootstrap3-typeahead.min.js') }}"></script>
     <link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="text/css">
 
     <link rel="stylesheet" href="{{ url_for('static', filename='css/jquery-jvectormap-2.0.3.css') }}" type="text/css" media="screen"/>
@@ -130,6 +131,8 @@ small {
                                     <strong  class='leftSepa textTopHeader' style="float: none; padding: 11px;">Dates:
                                         <input type="text" id="datepickerPunch" size="10" style="">
                                     </strong>
+                                        <a href="#" style="margin-top: 5px; float: right;" onclick="updateDatePunch();"><span class="glyphicon glyphicon-trash"></span></a>
+                                        <input type="text" id="typeaheadPunch" data-provide="typeahead" size="20" style="margin-bottom: 5px; float:right;" placeholder="Enter an organization">
                                 </div>
                                 <div id="panelbody" class="panel-body" style="">
                                     <div id="punchcard" style="width:100%; height: 100%;"></div>
@@ -171,6 +174,8 @@ small {
                                 <strong  class='leftSepa textTopHeader' style="float: none; padding: 11px;">Dates:
                                     <input type="text" id="datepickerOvertimeLogin" size="10" style="">
                                 </strong>
+                                <a href="#" style="margin-top: 5px; float: right;" onclick="updateDateOvertime();"><span class="glyphicon glyphicon-trash"></span></a>
+                                <input type="text" id="typeaheadOvertime" data-provide="typeahead" size="20" style="margin-bottom: 5px; float:right;" placeholder="Enter an organization">
                             </div>
                             <div id="panelbody" class="panel-body" style="">
                                 <div id="lineChart" style="width:100%; height: 20vh;"></div>
@@ -196,6 +201,7 @@ small {
         var url_getTopOrglogin = "{{ url_for('getTopOrglogin') }}";
         var url_getLoginVSCOntribution = "{{ url_for('getLoginVSCOntribution') }}";
         var url_getUserLoginsAndContribOvertime = "{{ url_for('getUserLoginsAndContribOvertime') }}";
+        var url_getTypeaheadData = "{{ url_for('getAllOrg') }}";
 
         /* DATA FROM CONF */