Merge pull request #9 from mokaddem/trendings

Trendings

README.md

@@ -1,19 +1,38 @@
 # MISP-Dashboard
 An experimental dashboard showing live data and statistics from the ZMQ of one or more MISP instances.
 
+
+# Installation
+- Launch ```./install_dependencies.sh``` from the MISP-Dashboard directory
+- Update the configuration file ```config.cfg``` so that it matches your system
+  - Fields that you may change:
+    - RedisGlobal -> host
+    - RedisGlobal -> port
+    - RedisGlobal -> zmq_url
+    - RedisGlobal -> misp_web_url
+    - RedisMap    -> pathMaxMindDB
+
+# Starting the System
+- Activate your virtualenv ```. ./DASHENV/bin/activate```
+- Listen to the MISP feed by starting the zmq_subscriber ```./zmq_subscriber.py```
+- Start the Flask server ```./server.py```
+- Access the interface at ```http://localhost:8001/```
+
+# Features
+
 ## Live Dashboard
 - Possibility to subscribe to multiple ZMQ feeds
 - Shows direct contribution made by organisations
 - Shows live resolvable posted locations
 
-![MISP event view](./screenshots/dashboard-live.png)
+![Dashboard live](./screenshots/dashboard-live.png)
 
 ## Geolocalisation Dashboard
 
 - Provides historical geolocalised information to support security teams, CSIRTs or SOC finding threats in their constituency
 - Possibility to get geospatial information from specific regions
 
-![MISP event view](./screenshots/dashboard-geo.png)
+![Dashbaord geo](./screenshots/dashboard-geo.png)
 
 ## Contributors Dashboard
 
@@ -29,25 +48,25 @@ __Includes__:
 - Gamification of the platform:
   - Two different levels of ranking with unique icons
   - Exclusive obtainable badges for source code contributors and donator
-  
-![Dashboard-contributor2](./screenshots/dashboard-contributors2.png)
-![Dashboard-contributor3](./screenshots/dashboard-contributors3.png)
 
+![Dashboard contributor](./screenshots/dashboard-contributors2.png)
+![Dashboard contributor2](./screenshots/dashboard-contributors3.png)
 
-# Installation
+## Users Dashboard
-- Launch ```./install_dependencies.sh``` from the MISP-Dashboard directory
+
-- Update the configuration file ```config.cfg``` so that it matches your system
+- Shows when and how the platform is used:
-  - Fields that you may change:
+    - Login punchcard and overtime
-    - RedisGlobal -> host
+    - Contribution vs login
-    - RedisGlobal -> port
+
-    - RedisGlobal -> zmq_url
+![Dashboard users](./screenshots/dashboard-users.png)
-    - RedisGlobal -> misp_web_url
+
-    
+## Trendings Dashboard
-# Starting the System
+
-- Activate your virtualenv ```. ./DASHENV/bin/activate```
+- Provides real time information to support security teams, CSIRTs or SOC showing current threats and activity
-- Listen to the MISP feed by starting the zmq_subscriber ```./zmq_subscriber.py```
+    - Shows most active events, categories and tags
-- Start the Flask server ```./server.py```
+    - Shows sightings and discussion overtime
-- Access the interface at ```http://localhost:8001/```
+
+![Dashboard users](./screenshots/dashboard-trendings.png)
 
 # zmq_subscriber options
 ```usage: zmq_subscriber.py [-h] [-n ZMQNAME] [-u ZMQURL]
@@ -62,7 +81,7 @@ optional arguments:
                         The URL to connect to
 ```
 
-## License
+# License
 Images and logos are handmade for:
 - rankingMISPOrg/
 - rankingMISPMonthly/

config/config.cfg.default

@@ -27,7 +27,7 @@ rankMultiplier = 2
 categories_in_datatable = ["internal_reference", "targeting_data", "antivirus_detection", "payload_delivery", "artifacts_dropped", "payload_installation", "persistence_mechanism", "network_activity", "payload_type", "attribution", "external_analysis", "financial_fraud", "support_Tool", "social_network", "person", "other" ]
 default_pnts_per_contribution = 1
 # array of the form [[category, pntsRcv], ...]
-pnts_per_contribution = [["payload_delivery", 1], ["artifact_dropped", 1], ["network_activity", 1]]
+pnts_per_contribution = [["payload_delivery", 1], ["artifacts_dropped", 1], ["network_activity", 1]]
 additional_help_text = ["Sightings multiplies earned points by 2", "Editing an attribute earns you the same as creating one"]
 
 [Log]

install_dependencies.sh

@@ -3,7 +3,7 @@
 set -e
 set -x
 
-sudo apt-get install python3-virtualenv -y
+sudo apt-get install python3-virtualenv virtualenv screen redis-server -y
 
 if [ -z "$VIRTUAL_ENV" ]; then
     virtualenv -p python3 DASHENV
@@ -102,4 +102,10 @@ wget https://cdn.datatables.net/${DATATABLE_VERSION}/js/dataTables.bootstrap.js
 git clone https://github.com/bassjobsen/Bootstrap-3-Typeahead.git temp/Bootstrap-3-Typeahead
 mv temp/Bootstrap-3-Typeahead/bootstrap3-typeahead.min.js ./static/js
 
+#punchcard
+git clone https://github.com/melenaos/jquery-punchcard.git temp/jquery-punchcard
+mv temp/jquery-punchcard/src/punchcard.js ./static/js
+mv temp/jquery-punchcard/src/punchcard.css ./static/css
+wget https://momentjs.com/downloads/moment.js -O ./static/js/moment.js
+
 rm -rf ./temp

server.py

@@ -12,6 +12,7 @@ import os
 import util
 import contributor_helper
 import users_helper
+import trendings_helper
 
 configfile = os.path.join(os.environ['DASH_CONFIG'], 'config.cfg')
 cfg = configparser.ConfigParser()
@@ -34,6 +35,7 @@ serv_redis_db = redis.StrictRedis(
 
 contributor_helper = contributor_helper.Contributor_helper(serv_redis_db, cfg)
 users_helper = users_helper.Users_helper(serv_redis_db, cfg)
+trendings_helper = trendings_helper.Trendings_helper(serv_redis_db, cfg)
 
 subscriber_log = redis_server_log.pubsub(ignore_subscribe_messages=True)
 subscriber_log.psubscribe(cfg.get('RedisLog', 'channel'))
@@ -208,6 +210,18 @@ def users():
             )
 
 
+@app.route("/trendings")
+def trendings():
+    maxNum = request.args.get('maxNum')
+    try:
+        maxNum = int(maxNum)
+    except:
+        maxNum = 15
+
+    return render_template('trendings.html',
+            maxNum=maxNum
+            )
+
 ''' INDEX '''
 
 @app.route("/_logs")
@@ -497,6 +511,82 @@ def getUserLoginsAndContribOvertime():
     data = users_helper.getUserLoginsAndContribOvertime(date)
     return jsonify(data)
 
+''' TRENDINGS '''
+@app.route("/_getTrendingEvents")
+def getTrendingEvents():
+    try:
+        dateS = datetime.datetime.fromtimestamp(float(request.args.get('dateS')))
+        dateE = datetime.datetime.fromtimestamp(float(request.args.get('dateE')))
+    except:
+        dateS = datetime.datetime.now() - datetime.timedelta(days=7)
+        dateE = datetime.datetime.now()
+
+    specificLabel = request.args.get('specificLabel')
+    data = trendings_helper.getTrendingEvents(dateS, dateE, specificLabel)
+    return jsonify(data)
+
+@app.route("/_getTrendingCategs")
+def getTrendingCategs():
+    try:
+        dateS = datetime.datetime.fromtimestamp(float(request.args.get('dateS')))
+        dateE = datetime.datetime.fromtimestamp(float(request.args.get('dateE')))
+    except:
+        dateS = datetime.datetime.now() - datetime.timedelta(days=7)
+        dateE = datetime.datetime.now()
+
+
+    data = trendings_helper.getTrendingCategs(dateS, dateE)
+    return jsonify(data)
+
+@app.route("/_getTrendingTags")
+def getTrendingTags():
+    try:
+        dateS = datetime.datetime.fromtimestamp(float(request.args.get('dateS')))
+        dateE = datetime.datetime.fromtimestamp(float(request.args.get('dateE')))
+    except:
+        dateS = datetime.datetime.now() - datetime.timedelta(days=7)
+        dateE = datetime.datetime.now()
+
+
+    data = trendings_helper.getTrendingTags(dateS, dateE)
+    return jsonify(data)
+
+@app.route("/_getTrendingSightings")
+def getTrendingSightings():
+    try:
+        dateS = datetime.datetime.fromtimestamp(float(request.args.get('dateS')))
+        dateE = datetime.datetime.fromtimestamp(float(request.args.get('dateE')))
+    except:
+        dateS = datetime.datetime.now() - datetime.timedelta(days=7)
+        dateE = datetime.datetime.now()
+
+    data = trendings_helper.getTrendingSightings(dateS, dateE)
+    return jsonify(data)
+
+@app.route("/_getTrendingDisc")
+def getTrendingDisc():
+    try:
+        dateS = datetime.datetime.fromtimestamp(float(request.args.get('dateS')))
+        dateE = datetime.datetime.fromtimestamp(float(request.args.get('dateE')))
+    except:
+        dateS = datetime.datetime.now() - datetime.timedelta(days=7)
+        dateE = datetime.datetime.now()
+
+
+    data = trendings_helper.getTrendingDisc(dateS, dateE)
+    return jsonify(data)
+
+@app.route("/_getTypeaheadData")
+def getTypeaheadData():
+    try:
+        dateS = datetime.datetime.fromtimestamp(float(request.args.get('dateS')))
+        dateE = datetime.datetime.fromtimestamp(float(request.args.get('dateE')))
+    except:
+        dateS = datetime.datetime.now() - datetime.timedelta(days=7)
+        dateE = datetime.datetime.now()
+
+    data = trendings_helper.getTypeaheadData(dateS, dateE)
+    return jsonify(data)
 
 if __name__ == '__main__':
     app.run(host='localhost', port=8001, threaded=True)

static/css/ranking.css

@@ -90,6 +90,11 @@
     display: none;
 }
 
+.table > thead > tr > th.centerCell {
+    text-align: center;
+    min-width: 45px;
+}
+
 .table > tbody > tr > td.centerCell {
     text-align: center;
     min-width: 45px;

static/js/trendings.js

@@ -0,0 +1,441 @@
+/* VARS */
+var dateStart;
+var dateEnd;
+var eventPie = ["#eventPie"];
+var eventLine = ["#eventLine"];
+var categPie = ["#categPie"];
+var categLine = ["#categLine"];
+var tagPie = ["#tagPie"];
+var tagLine = ["#tagLine"];
+var sightingLineWidget;
+var discLine = ["#discussionLine"];
+var allData;
+var globalColorMapping = {};
+
+/* OPTIONS */
+var datePickerOptions = {
+    showOn: "button",
+    maxDate: 0,
+    buttonImage: urlIconCalendar,
+    buttonImageOnly: true,
+    buttonText: "Select date",
+    showAnim: "slideDown",
+    onSelect: dateChanged
+};
+var lineChartOption = {
+    lines: {
+        show: true,
+    },
+    points: { show: true },
+    xaxis: {
+        mode: "time",
+        minTickSize: [1, "day"],
+    },
+    legend: { show: false },
+    grid: {
+        hoverable: true
+    }
+};
+var pieChartOption = {
+    series: {
+        pie: {
+            innerRadius: 0.2,
+            show: true,
+            radius: 100,
+            label: {
+                show: true,
+                radius: 6/10,
+                formatter: innerPieLabelFormatter,
+            }
+        }
+    },
+    legend: {
+        show: true,
+        labelFormatter: legendFormatter
+    },
+    grid: {
+        hoverable: true,
+        clickable: true
+    }
+};
+var typeaheadOption_event = {
+    source: function (query, process) {
+        if (allData === undefined) { // caching
+            return $.getJSON(url_getTypeaheadData, function (data) {
+                    allData = data;
+                    return process(data.TRENDINGS_EVENTS);
+            });
+        } else {
+            return process(allData.TRENDINGS_EVENTS);
+        }
+    },
+    updater: function(theevent) {
+        updateLineForLabel(eventLine, theevent, undefined, url_getTrendingEvent);
+    }
+}
+var typeaheadOption_categ = {
+    source: function (query, process) {
+        if (allData  === undefined) { // caching
+            return $.getJSON(url_getTypeaheadData, function (data) {
+                    allData = data;
+                    return process(data.TRENDINGS_CATEGS);
+            });
+        } else {
+            return process(allData.TRENDINGS_CATEGS);
+        }
+    },
+    updater: function(categ) {
+        updateLineForLabel(categLine, categ, undefined, url_getTrendingCateg);
+    }
+}
+var typeaheadOption_tag = {
+    source: function (query, process) {
+        if (allData === undefined) { // caching
+            return $.getJSON(url_getTypeaheadData, function (data) {
+                    allData = data;
+                    return process(data.TRENDINGS_TAGS);
+            });
+        } else {
+            return process(allData.TRENDINGS_TAGS);
+        }
+    },
+    updater: function(tag) {
+        updateLineForLabel(tagLine, tag, undefined, url_getTrendingTag);
+    }
+}
+
+/* FUNCTIONS */
+function getColor(label) {
+    try {
+        return globalColorMapping[label];
+    } catch(err) {
+        return undefined;
+    }
+
+}
+
+function innerPieLabelFormatter(label, series) {
+    var count = series.data[0][1];
+    return '<div '
+           + 'style="font-size:8pt;text-align:inherit;padding:2px;">'
+               + '<a class="tagElem" style="background-color: '+ 'white' + ';'
+               + 'color: ' + 'black' + ';"> ' + count + '</a>'
+           + '</div>';
+}
+
+function getTextColour(rgb) {
+    var r = parseInt('0x'+rgb.substring(0,2));
+    var g = parseInt('0x'+rgb.substring(2,4));
+    var b = parseInt('0x'+rgb.substring(4,6));
+    var avg = ((2 * r) + b + (3 * g))/6;
+    if (avg < 128) {
+        return 'white';
+    } else {
+        return 'black';
+    }
+}
+function legendFormatter(label, series) {
+    try {
+        jsonLabel = JSON.parse(label);
+        var backgroundColor = jsonLabel.colour;
+        var color = getTextColour(backgroundColor.substring(1,6));;
+        var labelText = jsonLabel.name;
+        return '<div '
+               + 'style="font-size:8pt;text-align:inherit;padding:2px;">'
+                   + '<a class="tagElem" style="background-color: '+ backgroundColor + ';'
+                   + 'color: ' + color + ';"> ' + labelText + '</a>'
+               + '</div>';
+    } catch(err) {
+        // removing unwanted "
+        var label = label.replace(/\\"/g, "").replace(/\"/g, "");
+        // limiting size
+        if (label.length >= 40){
+            labelLimited = label.substring(0, 40) + '[...]';
+        }   else {
+            labelLimited = label;
+        }
+        return '<div '
+                + 'style="font-size:8pt;text-align:inherit;padding:2px;">'
+                    + '<a class="tagElem" style="background-color: white; color: black;"> ' + labelLimited
+                    + '</a>';
+                + '</div>';
+    }
+}
+
+function generateEmptyAndFillData(data, specificLabel, colorMapping) {
+    // formating - Generate empty data
+    var toPlot_obj = {};
+    var allDates = [];
+    for (var arr of data) {
+        var date = new Date(arr[0]*1000);
+        date = new Date(date.valueOf() - date.getTimezoneOffset() * 60000); // center the data around the day
+        allDates.push(date);
+        var items = arr[1];
+        if (items.length > 0) {
+            for(var item_arr of items) {
+                var count = item_arr[1];
+                var itemStr = JSON.stringify(item_arr[0]);
+                if (specificLabel === undefined || specificLabel == item_arr[0]) { // no tag
+                    if(toPlot_obj[itemStr] === undefined)
+                        toPlot_obj[itemStr] = {};
+                    toPlot_obj[itemStr][date] = count;
+                } else if (specificLabel == item_arr[0].name) { // tag
+                    if(toPlot_obj[itemStr] === undefined)
+                        toPlot_obj[itemStr] = {};
+                    toPlot_obj[itemStr][date] = count;
+                } else if (specificLabel == itemStr.substring(1, itemStr.length-1)) { // tag from click (countain { and }, need to supress it)
+                    if(toPlot_obj[itemStr] === undefined)
+                        toPlot_obj[itemStr] = {};
+                    toPlot_obj[itemStr][date] = count;
+                }
+            }
+        }
+    }
+    toPlot = []
+    for (var itemStr in toPlot_obj) {
+        if (toPlot_obj.hasOwnProperty(itemStr)) {
+            data_toPlot = []
+            for (var curDate of allDates) {
+                if (toPlot_obj[itemStr].hasOwnProperty(curDate)) {
+                    data_toPlot.push([curDate, toPlot_obj[itemStr][curDate]])
+                } else {
+                    data_toPlot.push([curDate, 0])
+                }
+            }
+            if (colorMapping === undefined) {
+                //try to get color, else no color
+                var colorCode = getColor(itemStr);
+                if (!( colorCode === undefined)) {
+                    toPlot.push({label: itemStr, data: data_toPlot, color: colorCode})
+                } else {
+                    toPlot.push({label: itemStr, data: data_toPlot})
+                }
+            } else {
+                try {
+                    var color = colorMapping[itemStr].colour;
+                    toPlot.push({label: itemStr, data: data_toPlot, color: color})
+                } catch(err) {
+                    // ignore, only shows data displayed in the pie chart
+                }
+            }
+        }
+    }
+    return toPlot;
+}
+
+function compareObj(a,b) {
+  if (a.data < b.data)
+    return -1;
+  if (a.data > b.data)
+    return 1;
+  return 0;
+}
+/* UPDATES */
+
+// return the color maping: label->color
+function updatePie(pie, line, data, url) {
+    var pieID = pie[0];
+    var pieWidget = pie[1];
+    var itemMapping = {};
+    var colorMapping = {};
+    if (data === undefined || data.length == 0 || (data[0] == 0 && data[1] == 0)) {
+        toPlot = [{ label: 'No data', data: 100 }];
+    } else {
+        toPlot_obj = {}
+        for (var arr of data) {
+            var date = arr[0];
+            var items = arr[1]
+            for(var item_arr of items) {
+                var itemStr = JSON.stringify(item_arr[0]);
+                itemMapping[itemStr] = item_arr[0];
+                var count = item_arr[1];
+                if(toPlot_obj[itemStr] === undefined)
+                    toPlot_obj[itemStr] = 0;
+                toPlot_obj[itemStr] += count;
+            }
+        }
+        if (Object.keys(toPlot_obj).length == 0) { // no data
+            toPlot = [{ label: 'No data', data: 100 }];
+        } else {
+            toPlot = [];
+            for (var itemStr in toPlot_obj) {
+                if (toPlot_obj.hasOwnProperty(itemStr)) {
+                    var itemColor = itemMapping[itemStr].colour
+                    colorMapping[itemStr] = itemColor;
+                    toPlot.push({label: itemStr, data: toPlot_obj[itemStr], color: itemColor})
+                }
+            }
+        }
+        toPlot.sort(compareObj).reverse();
+        var maxNum = $('#num_selector').val();
+        toPlot = toPlot.slice(0,maxNum); // take at max 12 elements
+    }
+    if (!(pieWidget === undefined)) {
+        pieWidget.setData(toPlot);
+        pieWidget.setupGrid();
+        pieWidget.draw();
+        // fill colorMapping
+        for (item of pieWidget.getData()) {
+            colorMapping[item.label] = {colour: item.color};
+        }
+    } else {
+        pieWidget = $.plot(pieID, toPlot, pieChartOption);
+        pie.push(pieWidget);
+        // Hover
+        $(pieID).bind("plothover", function (event, pos, item) {
+            if (item) {
+                $("#tooltip").html(legendFormatter(item.series.label))
+                    .css({top: pos.pageY+5, left: pos.pageX+5})
+                    .fadeIn(200);
+            } else {
+                $("#tooltip").hide();
+            }
+        });
+        // Click
+        $(pieID).bind("plotclick", function(event, pos, obj) {
+            if (!obj) { return; }
+            var specificLabel = obj.series.label;
+            colorMapping[specificLabel] = {};
+            colorMapping[specificLabel] =  { colour: obj.series.color };
+            updateLineForLabel(line, specificLabel.substring(1, specificLabel.length-1), colorMapping, url);
+        });
+        for (item of pieWidget.getData()) {
+            colorMapping[item.label] = {colour: item.color};
+        }
+    }
+    return colorMapping;
+}
+
+function updateLine(line, data, chartOptions, specificLabel, colorMapping) {
+    lineID = line[0];
+    lineWidget = line[1];
+    toPlot = generateEmptyAndFillData(data, specificLabel, colorMapping);
+    // plot
+    if (!(lineWidget === undefined)) {
+        lineWidget.setData(toPlot);
+        lineWidget.setupGrid();
+        lineWidget.draw();
+    } else {
+        if (chartOptions === undefined) {
+            chartOptions = lineChartOption;
+        }
+        lineWidget = $.plot(lineID, toPlot, chartOptions);
+        line.push(lineWidget);
+        $(lineID).bind("plothover", function (event, pos, item) {
+            if (item) {
+                $("#tooltip").html(legendFormatter(item.series.label))
+                    .css({top: item.pageY+5, left: item.pageX+5})
+                    .fadeIn(200);
+                } else {
+                    $("#tooltip").hide();
+                }
+        });
+    }
+}
+
+function updateSignthingsChart() {
+        $.getJSON( url_getTrendingSightings+"?dateS="+parseInt(dateStart.getTime()/1000)+"&dateE="+parseInt(dateEnd.getTime()/1000), function( data ) {
+            var toPlot_obj = {};
+            toPlot_obj['Sightings'] = [];
+            toPlot_obj['False positive'] = [];
+            var allDates = [];
+            for (var arr of data) {
+                var date = new Date(arr[0]*1000);
+                date = new Date(date.valueOf() - date.getTimezoneOffset() * 60000); // center the data around the day
+                allDates.push(date);
+                var items = arr[1];
+                var sight = items.sightings;
+                var fp = items.false_positive;
+                toPlot_obj['Sightings'].push([date, sight]);
+                toPlot_obj['False positive'].push([date, -fp]);
+            }
+            toPlot = []
+            toPlot.push({label: 'Sightings', data: toPlot_obj['Sightings'], color: '#4da74d'})
+            toPlot.push({label: 'False positive', data: toPlot_obj['False positive'], color: '#cb4b4b'})
+
+            if (!(sightingLineWidget === undefined)) {
+                sightingLineWidget.setData(toPlot);
+                sightingLineWidget.setupGrid();
+                sightingLineWidget.draw();
+            } else {
+                var lineChartOptionSight = jQuery.extend(true, {}, lineChartOption);
+                lineChartOptionSight['legend']['show'] = true;
+                lineChartOptionSight['legend']['position'] = 'nw';
+                lineChartOptionSight['grid'] = {};
+                lineChartOptionSight['lines']['fill'] = true;
+                sightingLineWidget = $.plot("#sightingLine", toPlot, lineChartOptionSight);
+            }
+        });
+}
+
+function updateLineForLabel(line, specificLabel, colorMapping, url) {
+    $.getJSON( url+"?dateS="+parseInt(dateStart.getTime()/1000)+"&dateE="+parseInt(dateEnd.getTime()/1000)+"&specificLabel="+specificLabel, function( data ) {
+        updateLine(line, data, undefined, specificLabel, colorMapping);
+    });
+}
+
+function updatePieLine(pie, line, url) {
+    $.getJSON( url+"?dateS="+parseInt(dateStart.getTime()/1000)+"&dateE="+parseInt(dateEnd.getTime()/1000), function( data ) {
+        var colorMapping = updatePie(pie, line, data, url);
+        for (var item in colorMapping) {
+            if (colorMapping.hasOwnProperty(item) && colorMapping[item] != undefined) {
+                globalColorMapping[item] = colorMapping[item].colour;
+            }
+        }
+        updateLine(line, data, undefined, undefined, colorMapping);
+    });
+}
+
+function updateDisc() {
+    var lineChartOptionDisc = jQuery.extend(true, {}, lineChartOption);
+    lineChartOptionDisc['legend']['show'] = true;
+    lineChartOptionDisc['legend']['position'] = 'nw';
+    lineChartOptionDisc['lines']['fill'] = true;
+    $.getJSON( url_getTrendingDisc+"?dateS="+parseInt(dateStart.getTime()/1000)+"&dateE="+parseInt(dateEnd.getTime()/1000), function( data ) {
+        updateLine(discLine, data, lineChartOptionDisc);
+    });
+}
+
+function dateChanged() {
+    dateStart = datePickerWidgetStart.datepicker( "getDate" );
+    dateEnd = datePickerWidgetEnd.datepicker( "getDate" );
+    updatePieLine(eventPie, eventLine, url_getTrendingEvent);
+    updatePieLine(categPie, categLine, url_getTrendingCateg);
+    updatePieLine(tagPie, tagLine, url_getTrendingTag);
+    updateSignthingsChart();
+    updateDisc();
+}
+
+$(document).ready(function () {
+
+    datePickerWidgetStart = $( "#datepickerStart" ).datepicker(datePickerOptions);
+    var lastWeekDate = new Date();
+    lastWeekDate.setDate(lastWeekDate.getDate()-7);
+    datePickerWidgetStart.datepicker("setDate", lastWeekDate);
+    dateStart = datePickerWidgetStart.datepicker( "getDate" );
+    datePickerWidgetEnd = $( "#datepickerEnd" ).datepicker(datePickerOptions);
+    datePickerWidgetEnd.datepicker("setDate", new Date());
+    dateEnd = datePickerWidgetEnd.datepicker( "getDate" );
+
+    $('#typeaheadEvent').typeahead(typeaheadOption_event);
+    $('#typeaheadCateg').typeahead(typeaheadOption_categ);
+    $('#typeaheadTag').typeahead(typeaheadOption_tag);
+
+    updatePieLine(eventPie, eventLine, url_getTrendingEvent)
+    updatePieLine(categPie, categLine, url_getTrendingCateg)
+    updatePieLine(tagPie, tagLine, url_getTrendingTag)
+    updateSignthingsChart();
+    updateDisc();
+
+    $( "#num_selector" ).change(function() {
+        var sel = parseInt($( this ).val());
+        var maxNum = sel;
+        window.location.href = url_currentPage+'?maxNum='+maxNum;
+    });
+
+    $("<div id='tooltip'></div>").css({
+        position: "absolute",
+        display: "none",
+    }).appendTo("body");
+
+});

templates/contrib.html

@@ -8,7 +8,7 @@
     <meta name="description" content="">
     <meta name="author" content="">
 
-    <title>MISP live dashboard</title>
+    <title>MISP Contributors</title>
 
     <!-- Bootstrap Core CSS -->
     <link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
@@ -188,14 +188,14 @@
                             <thead>
                                 <tr>
                                     {% for title in trophy_title_str %}
-                                            <th>{{ title }}</th>
+                                            <th class="centerCell">{{ title }}</th>
                                     {% endfor %}
                                 </tr>
                             </thead>
                             <tbody id='bodyTableTrophyModal'>
                                 <tr>
                                     {% for perc in trophy_mapping %}
-                                            <td>{{ perc }}</td>
+                                            <td class="centerCell">{{ perc }}</td>
                                     {% endfor %}
                                 </tr>
                                 <tr>
@@ -249,6 +249,8 @@
                       <li><a href="{{ url_for('index') }}">MISP Live Dashboard</a></li>
                       <li><a href="{{ url_for('geo') }}">MISP Geolocalisation</a></li>
                       <li><a href="{{ url_for('contrib') }}">MISP Contributors</a></li>
+                      <li><a href="{{ url_for('users') }}">MISP Users</a></li>
+                      <li><a href="{{ url_for('trendings') }}">MISP Trendings</a></li>
                     </ul>
                 <div id="ledsHolder" style="float: left; height: 50px;">
                     <div class='leftSepa textTopHeader'>

templates/geo.html

@@ -8,7 +8,7 @@
     <meta name="description" content="">
     <meta name="author" content="">
 
-    <title>MISP live dashboard</title>
+    <title>MISP Geolocalisation</title>
 
     <!-- Bootstrap Core CSS -->
     <link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
@@ -91,6 +91,8 @@ small {
                       <li><a href="{{ url_for('index') }}">MISP Live Dashboard</a></li>
                       <li><a href="{{ url_for('geo') }}">MISP Geolocalisation</a></li>
                       <li><a href="{{ url_for('contrib') }}">MISP Contributors</a></li>
+                      <li><a href="{{ url_for('users') }}">MISP Users</a></li>
+                      <li><a href="{{ url_for('trendings') }}">MISP Trendings</a></li>
                     </ul>
 
                 <div id="ledsHolder" style="float: right; height: 50px;">

templates/index.html

@@ -133,6 +133,8 @@ small {
                       <li><a href="{{ url_for('index') }}">MISP Live Dashboard</a></li>
                       <li><a href="{{ url_for('geo') }}">MISP Geolocalisation</a></li>
                       <li><a href="{{ url_for('contrib') }}">MISP Contributors</a></li>
+                      <li><a href="{{ url_for('users') }}">MISP Users</a></li>
+                      <li><a href="{{ url_for('trendings') }}">MISP Trendings</a></li>
                     </ul>
 
                 <div id="ledsHolder" style="float: right; height: 50px;"></div>

templates/trendings.html

@@ -0,0 +1,240 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <meta name="description" content="">
+    <meta name="author" content="">
+
+    <title>MISP Trendings</title>
+
+    <!-- Bootstrap Core CSS -->
+    <link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
+    <!-- Custom CSS -->
+    <link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
+    <!-- LeafLet -->
+    <link rel="stylesheet" href="{{ url_for('static', filename='css/leaflet.css') }}">
+    <script src="{{ url_for('static', filename='js/leaflet.js') }}"></script>
+
+    <!-- jQuery -->
+    <script src="{{ url_for('static', filename='js/jquery.min.js') }}"></script>
+    <!-- jQuery flot -->
+    <script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
+    <script src="{{ url_for('static', filename='js/jquery.flot.pie.min.js') }}"></script>
+    <script src="{{ url_for('static', filename='js/jquery.flot.resize.js') }}"></script>
+    <script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
+    <!-- Bootstrap Core JavaScript -->
+    <script src="{{ url_for('static', filename='js/bootstrap.js') }}"></script>
+    <script src="{{ url_for('static', filename='js/bootstrap3-typeahead.min.js') }}"></script>
+    <link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="text/css">
+
+    <link rel="stylesheet" href="{{ url_for('static', filename='css/jquery-jvectormap-2.0.3.css') }}" type="text/css" media="screen"/>
+    <script src="{{ url_for('static', filename='js/jquery-jvectormap-2.0.3.min.js') }}"></script>
+    <script src="{{ url_for('static', filename='js/jquery-jvectormap-world-mill.js') }}"></script>
+    <link href="{{ url_for('static', filename='css/jquery-ui.min.css') }}" rel="stylesheet" type="text/css" />
+    <script src="{{ url_for('static', filename='js/jquery-ui.min.js') }}"></script>
+
+</head>
+
+<style>
+
+ .highlightDay {
+     background-color: #edc240;
+     box-shadow: black 0px 0px 2px;
+ }
+
+ .tagElem {
+     display: inline-block;
+     padding: 2px 4px;
+     font-size: 12px;
+     font-weight: bold;
+     line-height: 14px;
+     border-radius: 3px;
+     box-shadow: 3px 3px 3px #888888;
+ }
+
+ .panel {
+     margin-bottom: 10px;
+ }
+
+ .panel-body {
+     padding: 0px;
+ }
+
+.col-lg-6 {
+    padding-right: 5px;
+    padding-left: 5px;
+}
+
+.leftSepa {
+    border-left-color: rgb(221, 221, 221);
+    border-left-style: solid;
+    border-left-width: 2px;
+    padding-left: 8px;
+}
+
+.textTopHeader {
+    height: 50px;
+    text-align: center;
+    margin-left: 8px;
+    float: left;
+    padding-top: 9px;
+}
+
+.center {
+  display:block;
+  margin-left:auto;
+  margin-right:auto;
+}
+
+small {
+    font-size: 100%;
+    font-weight: bold;
+}
+
+</style>
+
+<body>
+
+    <div id="wrapper">
+
+        <!-- Navigation -->
+        <nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0; padding-left: 15px;">
+            <div class="navbar-header">
+                <img src="{{ url_for('static', filename='pics/MISP.png') }}" alt="MISP" style="float: left; height: 40px; margin-top: 5px;"><a class="navbar-brand dropdown-toggle" data-toggle="dropdown" href="#">MISP Trendings
+                    <span class="caret"></span></a>
+                    <ul class="dropdown-menu">
+                      <li><a href="{{ url_for('index') }}">MISP Live Dashboard</a></li>
+                      <li><a href="{{ url_for('geo') }}">MISP Geolocalisation</a></li>
+                      <li><a href="{{ url_for('contrib') }}">MISP Contributors</a></li>
+                      <li><a href="{{ url_for('users') }}">MISP Users</a></li>
+                      <li><a href="{{ url_for('trendings') }}">MISP Trendings</a></li>
+                    </ul>
+                <div class='leftSepa textTopHeader'>
+                    <strong>Max display:
+                        <select id="num_selector">
+                            <option value="1" {% if maxNum == 1 %}selected=""{% endif %}>1</option>
+                            <option value="3" {% if maxNum == 3 %}selected=""{% endif %}>3</option>
+                            <option value="5" {% if maxNum == 5 %}selected=""{% endif %}>5</option>
+                            <option value="10" {% if maxNum == 10 %}selected=""{% endif %}>10</option>
+                            <option value="15" {% if maxNum == 15 %}selected=""{% endif %}>15</option>
+                            <option value="20" {% if maxNum == 20 %}selected=""{% endif %}>20</option>
+                        </select>
+                    </strong>
+                </div>
+                <div class='leftSepa textTopHeader'>
+                    <strong>Date:
+                        <input type="text" id="datepickerStart" size="30" style="margin-bottom: 5px;">
+                        <strong> to </strong>
+                        <input type="text" id="datepickerEnd" size="30" style="margin-bottom: 5px;">
+                    </strong>
+                </div>
+            </div>
+            <!-- /.navbar-header -->
+
+        </nav>
+
+        <!-- Page Content -->
+        <div id="page-wrapper" style="margin: 0px; padding: 0px;">
+            <div class="container-fluid">
+                <div class="row">
+                    <div style="height: 10px;"></div>
+
+                    <div class="col-lg-12">
+                        <div class="col-lg-4">
+                            <div class="panel panel-default" style="">
+                                <div class="panel-heading bg-info" style="font-weight: bold;">
+                                    <b>Most active events</b>
+                                    <input type="text" id="typeaheadEvent" data-provide="typeahead" size="20" style="margin-bottom: 5px; float:right;" placeholder="Enter an event">
+                                </div>
+                                <div id="panelbody" class="panel-body" style="">
+                                    <div id="eventPie" style="width:100%; height: 34vh;"></div>
+                                    <div id="eventLine" style="width:100%; height: 27vh;"></div>
+                                </div>
+                            </div><!-- /.panel-body -->
+                        </div>
+
+                        <div class="col-lg-3">
+                            <div class="panel panel-default" style="">
+                                <div class="panel-heading bg-info" style="font-weight: bold;">
+                                    <b>Most active categories</b>
+                                    <input type="text" id="typeaheadCateg" data-provide="typeahead" size="20" style="margin-bottom: 5px; float:right;" placeholder="Enter a category">
+                                </div>
+                                <div id="panelbody" class="panel-body" style="">
+                                    <div id="categPie" style="width:100%; height: 34vh;"></div>
+                                    <div id="categLine" style="width:100%; height: 27vh;"></div>
+                                </div>
+                            </div><!-- /.panel-body -->
+                        </div>
+
+                        <div class="col-lg-5">
+                            <div class="panel panel-default" style="">
+                                <div class="panel-heading bg-info" style="font-weight: bold;">
+                                    <b>Most popular tags</b>
+                                    <input type="text" id="typeaheadTag" data-provide="typeahead" size="20" style="margin-bottom: 5px; float:right;" placeholder="Enter a tag">
+                                </div>
+                                <div id="panelbody" class="panel-body" style="">
+                                    <div id="tagPie" style="width:100%; height: 34vh;"></div>
+                                    <div id="tagLine" style="width:100%; height: 27vh;"></div>
+                                </div>
+                            </div><!-- /.panel-body -->
+                        </div>
+
+                    </div><!-- /.col-lg-12 -->
+                    <div class="col-lg-12">
+                        <div class="col-lg-6">
+                            <div class="panel panel-default" style="">
+                                <div class="panel-heading bg-info" style="font-weight: bold;">
+                                    <b>Sightings</b>
+                                </div>
+                                <div id="panelbody" class="panel-body" style="">
+                                    <div id="sightingLine" style="width:100%; height: 20vh;"></div>
+                                </div>
+                            </div><!-- /.panel-body -->
+                        </div>
+                        <div class="col-lg-6">
+                            <div class="panel panel-default" style="">
+                                <div class="panel-heading bg-info" style="font-weight: bold;">
+                                    <b>Discussion</b>
+                                </div>
+                                <div id="panelbody" class="panel-body" style="">
+                                    <div id="discussionLine" style="width:100%; height: 20vh;"></div>
+                                </div>
+                            </div><!-- /.panel-body -->
+                        </div>
+                    </div><!-- /.col-lg-12 -->
+
+                </div><!-- /.row -->
+
+            </div> <!-- /.container-fluid -->
+
+        </div>
+        <!-- /#page-wrapper -->
+
+    </div>
+<!-- /#wrapper -->
+
+    <!-- Index -->
+    <script>
+        /* URL */
+        var urlIconCalendar = "{{ url_for('static', filename='pics/calendar.gif') }}";
+        var url_currentPage = "{{ url_for('trendings') }}";
+        var url_getTrendingEvent = "{{ url_for('getTrendingEvents') }}";
+        var url_getTrendingCateg = "{{ url_for('getTrendingCategs') }}";
+        var url_getTrendingTag = "{{ url_for('getTrendingTags') }}";
+        var url_getTrendingSightings = "{{ url_for('getTrendingSightings') }}";
+        var url_getTrendingDisc = "{{ url_for('getTrendingDisc') }}";
+
+        var url_getTypeaheadData = "{{ url_for('getTypeaheadData') }}";
+
+        /* DATA FROM CONF */
+
+    </script>
+
+    <script src="{{ url_for('static', filename='js/trendings.js') }}"></script>
+
+</body>
+
+</html>

templates/users.html

@@ -8,7 +8,7 @@
     <meta name="description" content="">
     <meta name="author" content="">
 
-    <title>MISP live dashboard</title>
+    <title>MISP Users</title>
 
     <!-- Bootstrap Core CSS -->
     <link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
@@ -109,7 +109,8 @@ small {
                       <li><a href="{{ url_for('geo') }}">MISP Geolocalisation</a></li>
                       <li><a href="{{ url_for('contrib') }}">MISP Contributors</a></li>
                       <li><a href="{{ url_for('users') }}">MISP Users</a></li>
-                    </ul>
+                      <li><a href="{{ url_for('trendings') }}">MISP Trendings</a></li>
+                   </ul>
             </div>
             <!-- /.navbar-header -->
 
@@ -139,7 +140,7 @@ small {
                         <div class="col-lg-5">
                             <div class="panel panel-default" style="">
                                 <div class="panel-heading bg-info" style="font-weight: bold;">
-                                    <b>Contribution/login (last 31 days)</b>
+                                    <b>Contribution/login (last 7 days)</b>
                                     <strong  class='leftSepa textTopHeader' style="float: none; padding: 11px;">Dates:
                                         <input type="text" id="datepickerOrgLogin" size="10" style="">
                                     </strong>

trendings_helper.py

@@ -0,0 +1,139 @@
+import math, random
+import os
+import json
+import datetime, time
+from collections import OrderedDict
+
+import util
+
+class Trendings_helper:
+    def __init__(self, serv_redis_db, cfg):
+        self.serv_redis_db = serv_redis_db
+        self.cfg = cfg
+
+    ''' SETTER '''
+
+    def addGenericTrending(self, trendingType, data, timestamp):
+        timestampDate = datetime.datetime.fromtimestamp(float(timestamp))
+        timestampDate_str = util.getDateStrFormat(timestampDate)
+        keyname = "{}:{}".format(trendingType, timestampDate_str)
+        if isinstance(data, OrderedDict):
+            to_save = json.dumps(data)
+        else:
+            to_save = data
+        self.serv_redis_db.zincrby(keyname, to_save, 1)
+
+    def addTrendingEvent(self, eventName, timestamp):
+        self.addGenericTrending('TRENDINGS_EVENTS', eventName, timestamp)
+
+    def addTrendingCateg(self, categName, timestamp):
+        self.addGenericTrending('TRENDINGS_CATEGS', categName, timestamp)
+
+    def addTrendingDisc(self, eventName, timestamp):
+        self.addGenericTrending('TRENDINGS_DISC', eventName, timestamp)
+
+    def addTrendingTags(self, tags, timestamp):
+        for tag in tags:
+            ordDic = OrderedDict() #keep fields with the same layout in redis
+            ordDic['id'] = tag['id']
+            ordDic['name'] = tag['name']
+            ordDic['colour'] = tag['colour']
+            self.addGenericTrending('TRENDINGS_TAGS', ordDic, timestamp)
+
+    def addSightings(self, timestamp):
+        timestampDate = datetime.datetime.fromtimestamp(float(timestamp))
+        timestampDate_str = util.getDateStrFormat(timestampDate)
+        keyname = "{}:{}".format("TRENDINGS_SIGHT_sightings", timestampDate_str)
+        self.serv_redis_db.incrby(keyname, 1)
+
+    def addFalsePositive(self, timestamp):
+        timestampDate = datetime.datetime.fromtimestamp(float(timestamp))
+        timestampDate_str = util.getDateStrFormat(timestampDate)
+        keyname = "{}:{}".format("TRENDINGS_SIGHT_false_positive", timestampDate_str)
+        self.serv_redis_db.incrby(keyname, 1)
+
+    ''' GETTER '''
+
+    def getGenericTrending(self, trendingType, dateS, dateE, topNum=0):
+        to_ret = []
+        prev_days = (dateE - dateS).days
+        for curDate in util.getXPrevDaysSpan(dateE, prev_days):
+            keyname = "{}:{}".format(trendingType, util.getDateStrFormat(curDate))
+            data = self.serv_redis_db.zrange(keyname, 0, topNum-1, desc=True, withscores=True)
+            data = [ [record[0].decode('utf8'), record[1]] for record in data ]
+            data = data if data is not None else []
+            to_ret.append([util.getTimestamp(curDate), data])
+        return to_ret
+
+    def getSpecificTrending(self, trendingType, dateS, dateE, specificLabel=''):
+        to_ret = []
+        prev_days = (dateE - dateS).days
+        for curDate in util.getXPrevDaysSpan(dateE, prev_days):
+            keyname = "{}:{}".format(trendingType, util.getDateStrFormat(curDate))
+            data = self.serv_redis_db.zscore(keyname, specificLabel)
+            data = [[specificLabel, data]] if data is not None else []
+            to_ret.append([util.getTimestamp(curDate), data])
+        return to_ret
+
+    def getTrendingEvents(self, dateS, dateE, specificLabel=None):
+        if specificLabel is None:
+            return self.getGenericTrending('TRENDINGS_EVENTS', dateS, dateE)
+        else:
+            specificLabel = specificLabel.replace('\\n', '\n'); # reset correctly label with their \n (CR) instead of their char value
+            return self.getSpecificTrending('TRENDINGS_EVENTS', dateS, dateE, specificLabel)
+
+    def getTrendingCategs(self, dateS, dateE):
+        return self.getGenericTrending('TRENDINGS_CATEGS', dateS, dateE)
+
+    # FIXME: Construct this when getting data
+    def getTrendingTags(self, dateS, dateE, topNum=12):
+        to_ret = []
+        prev_days = (dateE - dateS).days
+        for curDate in util.getXPrevDaysSpan(dateE, prev_days):
+            keyname = "{}:{}".format('TRENDINGS_TAGS', util.getDateStrFormat(curDate))
+            data = self.serv_redis_db.zrange(keyname, 0, topNum-1, desc=True, withscores=True)
+            data = [ [record[0].decode('utf8'), record[1]] for record in data ]
+            data = data if data is not None else []
+            temp = []
+            for jText, score in data:
+                temp.append([json.loads(jText), score])
+            data = temp
+            to_ret.append([util.getTimestamp(curDate), data])
+        return to_ret
+
+    def getTrendingSightings(self, dateS, dateE):
+        to_ret = []
+        prev_days = (dateE - dateS).days
+        for curDate in util.getXPrevDaysSpan(dateE, prev_days):
+            keyname = "{}:{}".format("TRENDINGS_SIGHT_sightings", util.getDateStrFormat(curDate))
+            sight = self.serv_redis_db.get(keyname)
+            sight = 0 if sight is None else int(sight.decode('utf8'))
+            keyname = "{}:{}".format("TRENDINGS_SIGHT_false_positive", util.getDateStrFormat(curDate))
+            fp = self.serv_redis_db.get(keyname)
+            fp = 0 if fp is None else int(fp.decode('utf8'))
+            to_ret.append([util.getTimestamp(curDate), { 'sightings': sight, 'false_positive': fp}])
+        return to_ret
+
+    def getTrendingDisc(self, dateS, dateE):
+        return self.getGenericTrending('TRENDINGS_DISC', dateS, dateE)
+
+    def getTypeaheadData(self, dateS, dateE):
+        to_ret = {}
+        for trendingType in ['TRENDINGS_EVENTS', 'TRENDINGS_CATEGS']:
+            allSet = set()
+            prev_days = (dateE - dateS).days
+            for curDate in util.getXPrevDaysSpan(dateE, prev_days):
+                keyname = "{}:{}".format(trendingType, util.getDateStrFormat(curDate))
+                data = self.serv_redis_db.zrange(keyname, 0, -1, desc=True)
+                for elem in data:
+                    allSet.add(elem.decode('utf8'))
+            to_ret[trendingType] = list(allSet)
+        tags = self.getTrendingTags(dateS, dateE)
+        tagSet = set()
+        for item in tags:
+            theDate, tagList = item
+            for tag in tagList:
+                tag = tag[0]
+                tagSet.add(tag['name'])
+        to_ret['TRENDINGS_TAGS'] = list(tagSet)
+        return to_ret

users_helper.py

@@ -81,7 +81,7 @@ class Users_helper:
             totLog = 1
         return totContrib/totLog
 
-    def getTopOrglogin(self, date, maxNum=12, prev_days=31):
+    def getTopOrglogin(self, date, maxNum=12, prev_days=7):
         all_logged_in_orgs = self.getAllLoggedInOrgs(date, prev_days)
         data = []
         for org in all_logged_in_orgs:

zmq_subscriber.py

@@ -17,6 +17,7 @@ import geoip2.database
 import util
 import contributor_helper
 import users_helper
+import trendings_helper
 
 configfile = os.path.join(os.environ['DASH_CONFIG'], 'config.cfg')
 cfg = configparser.ConfigParser()
@@ -55,6 +56,7 @@ serv_redis_db = redis.StrictRedis(
 
 contributor_helper = contributor_helper.Contributor_helper(serv_redis_db, cfg)
 users_helper = users_helper.Users_helper(serv_redis_db, cfg)
+trendings_helper = trendings_helper.Trendings_helper(serv_redis_db, cfg)
 
 reader = geoip2.database.Reader(PATH_TO_DB)
 
@@ -193,15 +195,15 @@ def handler_keepalive(zmq_name, jsonevent):
     publish_log(zmq_name, 'Keepalive', to_push)
 
 def handler_user(zmq_name, jsondata):
+    action = jsondata['action']
     json_user = jsondata['User']
-    userID = json_user['id']
+    json_org = jsondata['Organisation']
-    org = userID
+    org = json_org['name']
-    try: #only consider user login
+    if action == 'login': #only consider user login
-        timestamp = json_user['current_login']
+        timestamp = int(time.time())
-    except KeyError:
-        return
-    if timestamp != 0: # "invited_by": "xxxx" ???
         users_helper.add_user_login(timestamp, org)
+    else:
+        pass
 
 def handler_conversation(zmq_name, jsonevent):
     try: #only consider POST, not THREAD
@@ -212,11 +214,15 @@ def handler_conversation(zmq_name, jsonevent):
     org = jsonpost['org_name']
     categ = None
     action = 'add'
+    eventName = 'no name or id yet...'
     handleContribution(zmq_name, org,
                     'Discussion',
                     None,
                     action,
                     isLabeled=False)
+    # add Discussion
+    nowSec = int(time.time())
+    trendings_helper.addTrendingDisc(eventName, nowSec)
 
 def handler_object(zmq_name, jsondata):
     print('obj')
@@ -234,9 +240,33 @@ def handler_sighting(zmq_name, jsondata):
     handleContribution(zmq_name, org, 'Sighting', categ, action, pntMultiplier=2)
     handler_attribute(zmq_name, jsonsight, hasAlreadyBeenContributed=True)
 
+    try:
+        timestamp = jsonsight['date_sighting']
+    except KeyError:
+        pass
+
+    if jsonsight['type'] == "0": # sightings
+        trendings_helper.addSightings(timestamp)
+    elif jsonsight['type'] == "1": # false positive
+        trendings_helper.addFalsePositive(timestamp)
+
 def handler_event(zmq_name, jsonobj):
     #fields: threat_level_id, id, info
     jsonevent = jsonobj['Event']
+
+    #Add trending
+    eventName = jsonevent['info']
+    timestamp = jsonevent['timestamp']
+    trendings_helper.addTrendingEvent(eventName, timestamp)
+    try:
+        temp = jsonobj['EventTag']
+        tags = []
+        for tag in temp:
+            tags.append(tag['Tag'])
+    except KeyError:
+        tags = []
+    trendings_helper.addTrendingTags(tags, timestamp)
+
     #redirect to handler_attribute
     if 'Attribute' in jsonevent:
         attributes = jsonevent['Attribute']
@@ -273,6 +303,22 @@ def handler_attribute(zmq_name, jsonobj, hasAlreadyBeenContributed=False):
     if 'Attribute' in jsonobj:
         jsonattr = jsonobj['Attribute']
 
+    #Add trending
+    categName = jsonattr['category']
+    try:
+        timestamp = jsonattr['timestamp']
+    except KeyError:
+        timestamp = int(time.time())
+    trendings_helper.addTrendingCateg(categName, timestamp)
+    try:
+        temp = jsonattr['Tag']
+        tags = []
+        for tag in temp:
+            tags.append(tag['Tag'])
+    except KeyError:
+        tags = []
+    trendings_helper.addTrendingTags(tags, timestamp)
+
     to_push = []
     for field in json.loads(cfg.get('Log', 'fieldname_order')):
         if type(field) is list:
@@ -315,7 +361,10 @@ def process_log(zmq_name, event):
     topic, eventdata = event.split(' ', maxsplit=1)
     jsonevent = json.loads(eventdata)
     print(event)
-    dico_action[topic](zmq_name, jsonevent)
+    try:
+        dico_action[topic](zmq_name, jsonevent)
+    except KeyError as e:
+        print(e)
 
 
 def main(zmqName):
@@ -340,7 +389,8 @@ dico_action = {
         "misp_json_sighting":       handler_sighting,
         "misp_json_organisation":   handler_log,
         "misp_json_user":           handler_user,
-        "misp_json_conversation":   handler_conversation
+        "misp_json_conversation":   handler_conversation,
+        "misp_json_object_reference": handler_log,
         }