From b913f86e7b85f0fb740197d3c99b841b15729fce Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Wed, 15 Nov 2017 13:45:01 +0100
Subject: [PATCH] Linked MISP - Users with server (nto done for API usage yet)

---
 server.py            |  53 +++++++----
 templates/users.html | 220 ++++++++++++++++++++++++++++++++-----------
 users_helper.py      |  33 ++++---
 zmq_subscriber.py    |   3 +-
 4 files changed, 223 insertions(+), 86 deletions(-)

diff --git a/server.py b/server.py
index a45f663..a20d077 100755
--- a/server.py
+++ b/server.py
@@ -201,26 +201,6 @@ def users():
             )
 
 
-@app.route("/_getUserLogins")
-def getUserLogins():
-    try:
-        date = datetime.datetime.fromtimestamp(float(request.args.get('date')))
-    except:
-        date = datetime.datetime.now()
-
-    data = users_helper.getUserLoginsForPunchCard(date)
-    return jsonify(data)
-
-@app.route("/_getUserLoginsOvertime")
-def getUserLoginsOvertime():
-    try:
-        date = datetime.datetime.fromtimestamp(float(request.args.get('date')))
-    except:
-        date = datetime.datetime.now()
-
-    data = users_helper.getUserLoginsOvertime(date)
-    return jsonify(data)
-
 ''' INDEX '''
 
 @app.route("/_logs")
@@ -457,5 +437,38 @@ def getTrophies():
         org = ''
     return jsonify(contributor_helper.getOrgTrophies(org))
 
+
+''' USERS '''
+
+@app.route("/_getUserLogins")
+def getUserLogins():
+    try:
+        date = datetime.datetime.fromtimestamp(float(request.args.get('date')))
+    except:
+        date = datetime.datetime.now()
+
+    data = users_helper.getUserLoginsForPunchCard(date)
+    return jsonify(data)
+
+@app.route("/_getUserLoginsOvertime")
+def getUserLoginsOvertime():
+    try:
+        date = datetime.datetime.fromtimestamp(float(request.args.get('date')))
+    except:
+        date = datetime.datetime.now()
+
+    data = users_helper.getUserLoginsOvertime(date)
+    return jsonify(data)
+
+@app.route("/_getTopOrglogin")
+def getTopOrglogin():
+    try:
+        date = datetime.datetime.fromtimestamp(float(request.args.get('date')))
+    except:
+        date = datetime.datetime.now()
+
+    data = users_helper.getTopOrglogin(date)
+    return jsonify(data)
+
 if __name__ == '__main__':
     app.run(host='localhost', port=8001, threaded=True)
diff --git a/templates/users.html b/templates/users.html
index bd6345f..382270b 100644
--- a/templates/users.html
+++ b/templates/users.html
@@ -43,6 +43,11 @@
 
 <style>
 
+ .highlightDay {
+     background-color: #edc240;
+     box-shadow: black 0px 0px 2px;
+ }
+
  .panel {
      margin-bottom: 10px;
  }
@@ -110,6 +115,9 @@ small {
                             <div class="panel panel-default" style="">
                                 <div class="panel-heading bg-info" style="font-weight: bold;">
                                     <b>Punch card - login</b>
+                                    <strong  class='leftSepa textTopHeader' style="float: none; padding: 11px;">Dates:
+                                        <input type="text" id="datepickerPunch" size="10" style="">
+                                    </strong>
                                 </div>
                                 <div id="panelbody" class="panel-body" style="">
                                     <div id="punchcard" style="width:100%; height: 100%;"></div>
@@ -120,19 +128,25 @@ small {
                         <div class="col-lg-4">
                             <div class="panel panel-default" style="">
                                 <div class="panel-heading bg-info" style="font-weight: bold;">
-                                    <b>Top login per organisation</b>
+                                    <b>Top organisation login</b>
+                                    <strong  class='leftSepa textTopHeader' style="float: none; padding: 11px;">Dates:
+                                        <input type="text" id="datepickerOrgLogin" size="10" style="">
+                                    </strong>
                                 </div>
                                 <div id="panelbody" class="panel-body" style="">
-                                    <div id="barChart" style="width:100%; height: 20vh;"></div>
+                                    <div id="pieOrg" style="width:100%; height: 20vh;"></div>
                                 </div>
                             </div><!-- /.panel-body -->
 
                             <div class="panel panel-default" style="">
                                 <div class="panel-heading bg-info" style="font-weight: bold;">
                                     <b>Contribution after login (API vs non-API)</b>
+                                    <strong  class='leftSepa textTopHeader' style="float: none; padding: 11px;">Dates:
+                                        <input type="text" id="datepickerApi" size="10" style="">
+                                    </strong>
                                 </div>
                                 <div id="panelbody" class="panel-body" style="">
-                                    <div id="pieChart" style="width:100%; height: 20vh;"></div>
+                                    <div id="pieApi" style="width:100%; height: 20vh;"></div>
                                 </div>
                             </div><!-- /.panel-body -->
                         </div>
@@ -142,6 +156,9 @@ small {
                         <div class="panel panel-default" style="">
                             <div class="panel-heading bg-info" style="font-weight: bold;">
                                 <b>Login overtime</b>
+                                <strong  class='leftSepa textTopHeader' style="float: none; padding: 11px;">Dates:
+                                    <input type="text" id="datepickerOvertimeLogin" size="10" style="">
+                                </strong>
                             </div>
                             <div id="panelbody" class="panel-body" style="">
                                 <div id="lineChart" style="width:100%; height: 20vh;"></div>
@@ -162,79 +179,174 @@ small {
     <!-- Index -->
     <script>
         /* URL */
+        var urlIconCalendar = "{{ url_for('static', filename='pics/calendar.gif') }}";
 
         /* DATA FROM CONF */
 
     </script>
 
     <script>
-        $(document).ready(function () {
+        var punchcardWidget;
+        var pieOrgWidget;
+        var pieApiWidget;
+        var overtimeWidget;
 
-            $.getJSON( "{{ url_for('getUserLogins') }}", function( data ) {
-                $('#punchcard').punchcard({
-                    data: data,
-                    singular: 'login',
-                    plural: 'logins',
-                    timezones: ['local'],
-                    timezoneIndex:0
-                });
+        var div_day;
+        function highlight_punchDay() {
+            if (!(div_day === undefined)) {
+                div_day.removeClass('highlightDay');
+            }
+
+            var curDay = datePickerWidgetPunch.datepicker( "getDate" ).getDay();
+            if (curDay == 7)
+                curDay = 0;
+            div_day = $('[data-daynum="'+curDay+'"]');
+            div_day.addClass('highlightDay')
+        }
+
+        function updateDatePunch() {
+            var date = datePickerWidgetPunch.datepicker( "getDate" );
+            $.getJSON( "{{ url_for('getUserLogins') }}"+"?date="+date.getTime()/1000, function( data ) {
+                if (!(punchcardWidget === undefined)) {
+                    punchcardWidget.settings.data = data;
+                    punchcardWidget.refresh();
+                    highlight_punchDay();
+                } else {
+                    punchcardWidget = $('#punchcard').punchcard({
+                        data: data,
+                        singular: 'login',
+                        plural: 'logins',
+                        timezones: ['local'],
+                        timezoneIndex:0
+                    });
+                    punchcardWidget = punchcardWidget.data("plugin_" + "punchcard");
+                    highlight_punchDay();
+                }
             });
-
-            $.getJSON( "{{ url_for('getUserLogins') }}", function( data ) {
-                data = [{
-                    label: "Contribution with login",
-                    data: Math.floor(Math.random() * 100) + 1
-                },
-                {
-                    label: "Contribution without login",
-                    data: Math.floor(Math.random() * 100) + 1
-                    }
-                ];
-                $.plot('#pieChart', data, {
-                    series: {
-                        pie: {
-                            innerRadius: 0.5,
-                            show: true
+        }
+        function updateDatePieOrg() {
+            var date = datePickerWidgetOrg.datepicker( "getDate" );
+            $.getJSON( "{{ url_for('getTopOrglogin') }}"+"?date="+date.getTime()/1000, function( data ) {
+                toPlot = [];
+                for (item of data) { toPlot.push({ label: item[0], data: item[1] }); }
+                if (toPlot.length == 0) {
+                    toPlot = [{ label: 'No data', data: 100 }];
+                }
+                if (!pieOrgWidget === undefined) {
+                    pieOrgWidget.setData(toPlot);
+                    pieOrgWidget.draw();
+                } else {
+                    pieOrgWidget = $.plot('#pieOrg', toPlot, {
+                        series: {
+                            pie: {
+                                innerRadius: 0.5,
+                                show: true
+                            }
                         }
-                    }
-                });
+                    });
+                }
             });
-
-            $.getJSON( "{{ url_for('getUserLogins') }}", function( data ) {
+        }
+        function updateDatePieApi() {
+            var date = datePickerWidgetApi.datepicker( "getDate" );
+            $.getJSON( "{{ url_for('getUserLogins') }}"+"?date="+date.getTime()/1000, function( data ) {
                 data = [{label: 'org1', data:[ Math.floor(Math.random() * 100) + 1]},
                 {label: 'org2', data:[ Math.floor(Math.random() * 100) + 1]},
                 {label: 'org3', data:[ Math.floor(Math.random() * 100) + 1]}]
-                $.plot('#barChart', data, {
-                    series: {
-                        pie: {
-                            innerRadius: 0.5,
-                            show: true
-                        }
-                    }
-                });
-            });
 
-            $.getJSON( "{{ url_for('getUserLoginsOvertime') }}", function( data ) {
+                if (!pieApiWidget === undefined) {
+                    pieApiWidget.setData(toPlot);
+                    pieApiWidget.draw();
+                } else {
+                    pieApiWidget = $.plot('#pieApi', data, {
+                        series: {
+                            pie: {
+                                innerRadius: 0.5,
+                                show: true
+                            }
+                        }
+                    });
+                }
+            });
+        }
+        function updateDateOvertime() {
+            var date = datePickerWidgetOvertime.datepicker( "getDate" );
+            var now = new Date();
+            if (date.toDateString() == now.toDateString()) {
+                date = now;
+            } else {
+                date.setTime(date.getTime() + (24*60*60*1000-1)); // include data of selected date
+            }
+            $.getJSON( "{{ url_for('getUserLoginsOvertime') }}"+"?date="+parseInt(date.getTime()/1000), function( data ) {
                 temp = [];
                 var i=0;
                 for (item of data) {
                     temp.push([new Date(item[0]*1000), item[1]]);
                 }
-                console.log(temp);
                 data = {label: 'Login overtime', data: temp}
-                $.plot('#lineChart', [data], {
-                    lines: {
-                        show: true,
-                        steps: true,
-                        fill: true
-                    },
-                    xaxis: {
-                        mode: "time",
-                        minTickSize: [1, "hour"],
-                    }
-                });
+                if (!overtimeWidget === undefined) {
+                    overtimeWidget.setData(toPlot);
+                    overtimeWidget.draw();
+                } else {
+                    overtimeWidget = $.plot('#lineChart', [data], {
+                        lines: {
+                            show: true,
+                            steps: true,
+                            fill: true
+                        },
+                        xaxis: {
+                            mode: "time",
+                            minTickSize: [1, "day"],
+                        }
+                    });
+                }
             });
 
+        }
+    </script>
+
+    <script>
+        $(document).ready(function () {
+
+            var datePickerOptions = {
+                showOn: "button",
+                // minDate: -31,
+                maxDate: 0,
+                buttonImage: urlIconCalendar,
+                buttonImageOnly: true,
+                buttonText: "Select date",
+                showAnim: "slideDown"
+            };
+            // punch
+            var datePickerOptionsPunch = jQuery.extend({}, datePickerOptions);
+            datePickerOptionsPunch['onSelect'] = updateDatePunch;
+            datePickerWidgetPunch = $( "#datepickerPunch" );
+            datePickerWidgetPunch.datepicker(datePickerOptionsPunch);
+            datePickerWidgetPunch.datepicker("setDate", new Date());
+            // org login
+            var datePickerOptionsOrg = jQuery.extend({}, datePickerOptions);
+            datePickerOptionsOrg['onSelect'] = updateDatePieOrg;
+            datePickerWidgetOrg = $( "#datepickerOrgLogin" );
+            datePickerWidgetOrg.datepicker(datePickerOptionsOrg);
+            datePickerWidgetOrg.datepicker("setDate", new Date());
+            // api
+            var datePickerOptionsApi = jQuery.extend({}, datePickerOptions);
+            datePickerOptionsApi['onSelect'] = updateDatePieApi;
+            datePickerWidgetApi = $( "#datepickerApi" );
+            datePickerWidgetApi.datepicker(datePickerOptionsApi);
+            datePickerWidgetApi.datepicker("setDate", new Date());
+            // overtime
+            var datePickerOptionsOvertime = jQuery.extend({}, datePickerOptions);
+            datePickerOptionsOvertime['onSelect'] = updateDateOvertime;
+            datePickerWidgetOvertime = $( "#datepickerOvertimeLogin" );
+            datePickerWidgetOvertime.datepicker(datePickerOptionsOvertime);
+            datePickerWidgetOvertime.datepicker("setDate", new Date());
+
+            updateDatePunch();
+            updateDatePieOrg();
+            updateDatePieApi();
+            updateDateOvertime();
+
         });
     </script>
 
diff --git a/users_helper.py b/users_helper.py
index caf4914..84eca6f 100644
--- a/users_helper.py
+++ b/users_helper.py
@@ -10,18 +10,28 @@ class Users_helper:
         self.serv_redis_db = serv_redis_db
         self.cfg = cfg
 
-    def add_user_login(self, timestamp):
+    def add_user_login(self, timestamp, org):
         timestampDate = datetime.datetime.fromtimestamp(float(timestamp))
         timestampDate_str = util.getDateStrFormat(timestampDate)
-        keyname = "{}:{}".format('USER_LOGIN', timestampDate_str)
-        self.serv_redis_db.sadd(keyname, timestamp)
+
+        keyname_timestamp = "{}:{}".format('LOGIN_TIMESTAMP', timestampDate_str)
+        self.serv_redis_db.sadd(keyname_timestamp, timestamp)
+
+        keyname_org = "{}:{}".format('LOGIN_ORG', timestampDate_str)
+        self.serv_redis_db.zincrby(keyname_org, org, 1)
 
     def getUserLogins(self, date):
-        keyname = "USER_LOGIN:{}"
-        timestamps = self.serv_redis_db.smembers(keyname.format(util.getDateStrFormat(date)))
+        keyname = "LOGIN_TIMESTAMP:{}".format(util.getDateStrFormat(date))
+        timestamps = self.serv_redis_db.smembers(keyname)
         timestamps = [int(timestamp.decode('utf8')) for timestamp in timestamps]
         return timestamps
 
+    def getTopOrglogin(self, date, topNum=0):
+        keyname = "LOGIN_ORG:{}".format(util.getDateStrFormat(date))
+        data = self.serv_redis_db.zrange(keyname, 0, topNum-1, desc=True, withscores=True)
+        data = [ [record[0].decode('utf8'), record[1]] for record in data ]
+        return data
+
     def getUserLoginsForPunchCard(self, date, prev_days=6):
         week = {}
         for curDate in util.getXPrevDaysSpan(date, prev_days):
@@ -49,7 +59,8 @@ class Users_helper:
                 data.append(temp)
             except KeyError: # no data
                 data.append([0 for x in range(24)])
-
+        # swap: punchcard day starts on monday
+        data = [data[6]]+data[:6]
         return data
 
     def getUserLoginsOvertime(self, date, prev_days=6):
@@ -60,14 +71,14 @@ class Users_helper:
         for curDate in util.getXPrevDaysSpan(date, prev_days):
             timestamps = self.getUserLogins(curDate)
             for timestamp in timestamps: # sum occurence during the current hour
-                date = datetime.datetime.fromtimestamp(float(timestamp))
-                date = date.replace(minute=0, second=0, microsecond=0)
-                dico_hours[util.getTimestamp(date)] += 1
+                dateTimestamp = datetime.datetime.fromtimestamp(float(timestamp))
+                dateTimestamp = dateTimestamp.replace(minute=0, second=0, microsecond=0)
+                dico_hours[util.getTimestamp(dateTimestamp)] += 1
 
         # Format data
         data = []
-        for date, occ in dico_hours.items():
-            data.append([date, occ])
+        for curDate, occ in dico_hours.items():
+            data.append([curDate, occ])
 
         data.sort(key=lambda x: x[0])
         return data
diff --git a/zmq_subscriber.py b/zmq_subscriber.py
index 4369814..8e532e7 100755
--- a/zmq_subscriber.py
+++ b/zmq_subscriber.py
@@ -192,12 +192,13 @@ def handler_keepalive(zmq_name, jsonevent):
 def handler_user(zmq_name, jsondata):
     json_user = jsondata['User']
     userID = json_user['id']
+    org = userID
     try: #only consider user login
         timestamp = json_user['current_login']
     except KeyError:
         return
     if timestamp != 0: # "invited_by": "xxxx" ???
-        users_helper.add_user_login(timestamp)
+        users_helper.add_user_login(timestamp, org)
 
 def handler_conversation(zmq_name, jsonevent):
     try: #only consider POST, not THREAD