From 71837d33087932e12890912cbd30ec0f7bdb4414 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Thu, 16 Nov 2017 12:23:02 +0100 Subject: [PATCH 01/35] Added trendings template --- server.py | 5 + static/js/trendings.js | 22 +++++ templates/trendings.html | 203 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 230 insertions(+) create mode 100644 static/js/trendings.js create mode 100644 templates/trendings.html diff --git a/server.py b/server.py index e17302b..5ad54cd 100755 --- a/server.py +++ b/server.py @@ -201,6 +201,11 @@ def users(): ) +@app.route("/trendings") +def trendings(): + return render_template('trendings.html', + ) + ''' INDEX ''' @app.route("/_logs") diff --git a/static/js/trendings.js b/static/js/trendings.js new file mode 100644 index 0000000..375a976 --- /dev/null +++ b/static/js/trendings.js @@ -0,0 +1,22 @@ +function dateChanged() { + var date = datePickerWidget.datepicker( "getDate" ); + console.log(date); +} + +$(document).ready(function () { + + var datePickerOptions = { + showOn: "button", + maxDate: 0, + buttonImage: urlIconCalendar, + buttonImageOnly: true, + buttonText: "Select date", + showAnim: "slideDown", + onSelect: dateChanged + }; + var datePickerOptions = jQuery.extend({}, datePickerOptions); + datePickerWidget = $( "#datepicker" ); + datePickerWidget.datepicker(datePickerOptions); + datePickerWidget.datepicker("setDate", new Date()); + +}); diff --git a/templates/trendings.html b/templates/trendings.html new file mode 100644 index 0000000..7cdb26a --- /dev/null +++ b/templates/trendings.html @@ -0,0 +1,203 @@ + + + + + + + + + + + MISP live dashboard + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + +
+
+
+
+ +
+
+
+
+ Most active events +
+
+
+
+
+
+
+ +
+
+
+ Most active categories +
+
+
+
+
+
+
+ +
+
+
+ Popular tags +
+
+
+
+
+
+
+ +
+
+
+
+
+ Sightings +
+
+
+
+
+
+
+
+
+
+ Empty +
+
+
+
+
+
+
+ +
+ +
+ +
+ + +
+ + + + + + + + + + From 8942a43c18a4d3e8da9d43ae4d2cecb9abddbdc8 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Thu, 16 Nov 2017 15:08:14 +0100 Subject: [PATCH 02/35] Added linked pie plot with server --- server.py | 33 +++++++++++ static/js/trendings.js | 121 ++++++++++++++++++++++++++++++++++----- templates/trendings.html | 26 +++++---- zmq_subscriber.py | 43 +++++++++++--- 4 files changed, 192 insertions(+), 31 deletions(-) diff --git a/server.py b/server.py index 5ad54cd..467293a 100755 --- a/server.py +++ b/server.py @@ -12,6 +12,7 @@ import os import util import contributor_helper import users_helper +import trendings_helper configfile = os.path.join(os.environ['DASH_CONFIG'], 'config.cfg') cfg = configparser.ConfigParser() @@ -34,6 +35,7 @@ serv_redis_db = redis.StrictRedis( contributor_helper = contributor_helper.Contributor_helper(serv_redis_db, cfg) users_helper = users_helper.Users_helper(serv_redis_db, cfg) +trendings_helper = trendings_helper.Trendings_helper(serv_redis_db, cfg) subscriber_log = redis_server_log.pubsub(ignore_subscribe_messages=True) subscriber_log.psubscribe(cfg.get('RedisLog', 'channel')) @@ -485,6 +487,37 @@ def getLoginVSCOntribution(): data = users_helper.getLoginVSCOntribution(date) return jsonify(data) +''' TRENDINGS ''' +@app.route("/_getTrendingEvents") +def getTrendingEvents(): + try: + date = datetime.datetime.fromtimestamp(float(request.args.get('date'))) + except: + date = datetime.datetime.now() + + data = trendings_helper.getTrendingEvents(date) + return jsonify(data) + +@app.route("/_getTrendingCategs") +def getTrendingCategs(): + try: + date = datetime.datetime.fromtimestamp(float(request.args.get('date'))) + except: + date = datetime.datetime.now() + + data = trendings_helper.getTrendingCategs(date) + return jsonify(data) + +@app.route("/_getTrendingTags") +def getTrendingTags(): + try: + date = datetime.datetime.fromtimestamp(float(request.args.get('date'))) + except: + date = datetime.datetime.now() + + data = trendings_helper.getTrendingTags(date) + return jsonify(data) + if __name__ == '__main__': app.run(host='localhost', port=8001, threaded=True) diff --git a/static/js/trendings.js b/static/js/trendings.js index 375a976..a1e339e 100644 --- a/static/js/trendings.js +++ b/static/js/trendings.js @@ -1,22 +1,115 @@ +/* VARS */ +var date; +var eventPie = ["#eventPie"]; +var eventLine = ["#eventLine"]; +var categPie = ["#categPie"]; +var categLine = ["#categLine"]; +var tagPie = ["#tagPie"]; +var tagLine = ["#tagLine"]; +var sightingEventPieWidget; +var sightingCategLineWidget; + +/* OPTIONS */ +var datePickerOptions = { + showOn: "button", + maxDate: 0, + buttonImage: urlIconCalendar, + buttonImageOnly: true, + buttonText: "Select date", + showAnim: "slideDown", + onSelect: dateChanged +}; +var lineChartOption = { + lines: { + show: true, + steps: true, + fill: true + }, + xaxis: { + mode: "time", + minTickSize: [1, "day"], + } +}; +var pieChartOption = { + series: { + pie: { + innerRadius: 0.5, + show: true + } + } +}; + +/* FUNCTIONS */ + +function updatePie(pie, data) { + pieID = pie[0]; + pieWidget = pie[1]; + if (data === undefined || data.length == 0 || (data[0] == 0 && data[1] == 0)) { + toPlot = [{ label: 'No data', data: 100 }]; + } else { + toPlot = []; + for (item of data) { + toPlot.push({label: item[0], data: item[1]}); + } + } + console.log(toPlot); + if (!(pieWidget === undefined)) { + pieWidget.setData(toPlot); + pieWidget.setupGrid(); + pieWidget.draw(); + } else { + pieWidget = $.plot(pieID, toPlot, pieChartOption); + pie.push(pieWidget); + } +} + +function updateLine(line, data) { + lineID = line[0]; + lineWidget = line[1]; + temp = []; + var i=0; + for (item of data) { + temp.push([new Date(item[0]*1000), item[1]]); + } + data = {label: 'Overtime', data: temp} + if (!(lineWidget === undefined)) { + lineWidget.setData(toPlot); + lineWidget.draw(); + } else { + lineWidget = $.plot(lineID, [data], lineChartOption); + line.push(lineWidget); + } +} + +function updatePieLine(pie, line, url) { + // format date + // var now = new Date(); + // if (date.toDateString() == now.toDateString()) { + // date = now; + // } else { + // date.setTime(date.getTime() + (24*60*60*1000-1)); // include data of selected date + // } + $.getJSON( url+"?date="+parseInt(date.getTime()/1000), function( data ) { + updatePie(pie, data[0][1]); + updateLine(line, data); + }); +} + function dateChanged() { - var date = datePickerWidget.datepicker( "getDate" ); - console.log(date); + date = datePickerWidget.datepicker( "getDate" ); + updatePieLine(eventPie, eventLine, url_getTrendingEvent) + updatePieLine(categPie, categLine, url_getTrendingCateg) + updatePieLine(tagPie, tagLine, url_getTrendingTag) } $(document).ready(function () { - var datePickerOptions = { - showOn: "button", - maxDate: 0, - buttonImage: urlIconCalendar, - buttonImageOnly: true, - buttonText: "Select date", - showAnim: "slideDown", - onSelect: dateChanged - }; - var datePickerOptions = jQuery.extend({}, datePickerOptions); - datePickerWidget = $( "#datepicker" ); - datePickerWidget.datepicker(datePickerOptions); + datePickerWidget = $( "#datepicker" ).datepicker(datePickerOptions); datePickerWidget.datepicker("setDate", new Date()); + date = datePickerWidget.datepicker( "getDate" ); + + updatePieLine(eventPie, eventLine, url_getTrendingEvent) + updatePieLine(categPie, categLine, url_getTrendingCateg) + updatePieLine(tagPie, tagLine, url_getTrendingTag) }); diff --git a/templates/trendings.html b/templates/trendings.html index 7cdb26a..36fcb95 100644 --- a/templates/trendings.html +++ b/templates/trendings.html @@ -122,8 +122,9 @@ small { Most active events
-
-
+
+
+
@@ -134,8 +135,9 @@ small { Most active categories
-
-
+
+
+
@@ -143,11 +145,12 @@ small {
- Popular tags + Most popular tags
-
-
+
+
+
@@ -160,8 +163,8 @@ small { Sightings
-
-
+
+
@@ -171,7 +174,7 @@ small { Empty
-
+
@@ -191,6 +194,9 @@ small { + @@ -138,6 +139,7 @@ small {
Most active events +
@@ -150,6 +152,7 @@ small {
Most active categories +
@@ -162,6 +165,7 @@ small {
Most popular tags +
@@ -214,6 +218,8 @@ small { var url_getTrendingSightings = "{{ url_for('getTrendingSightings') }}"; var url_getTrendingDisc = "{{ url_for('getTrendingDisc') }}"; + var url_getTypeaheadData = "{{ url_for('getTypeaheadData') }}"; + /* DATA FROM CONF */ diff --git a/trendings_helper.py b/trendings_helper.py index 2fe4fdf..8d32b14 100644 --- a/trendings_helper.py +++ b/trendings_helper.py @@ -71,6 +71,7 @@ class Trendings_helper: def getTrendingCategs(self, dateS, dateE): return self.getGenericTrending('TRENDINGS_CATEGS', dateS, dateE) + # FIXME: Construct this when getting data def getTrendingTags(self, dateS, dateE, topNum=12): to_ret = [] prev_days = (dateE - dateS).days @@ -101,3 +102,24 @@ class Trendings_helper: def getTrendingDisc(self, dateS, dateE): return self.getGenericTrending('TRENDINGS_DISC', dateS, dateE) + + def getTypeaheadData(self, dateS, dateE): + to_ret = {} + for trendingType in ['TRENDINGS_EVENTS', 'TRENDINGS_CATEGS']: + allSet = set() + prev_days = (dateE - dateS).days + for curDate in util.getXPrevDaysSpan(dateE, prev_days): + keyname = "{}:{}".format(trendingType, util.getDateStrFormat(curDate)) + data = self.serv_redis_db.zrange(keyname, 0, -1, desc=True) + for elem in data: + allSet.add(elem.decode('utf8')) + to_ret[trendingType] = list(allSet) + tags = self.getTrendingTags(dateS, dateE) + tagSet = set() + for item in tags: + theDate, tagList = item + for tag in tagList: + tag = tag[0] + tagSet.add(tag['name']) + to_ret['TRENDINGS_TAGS'] = list(tagSet) + return to_ret From 914514023bb81bd7be6031334f13bcf3e1e47a94 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Tue, 21 Nov 2017 08:42:54 +0100 Subject: [PATCH 23/35] Added typeahead plot for events --- server.py | 3 ++- static/js/trendings.js | 8 ++++---- trendings_helper.py | 17 +++++++++++++++-- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/server.py b/server.py index f2efbad..f8c5692 100755 --- a/server.py +++ b/server.py @@ -497,7 +497,8 @@ def getTrendingEvents(): dateS = datetime.datetime.now() - datetime.timedelta(days=7) dateE = datetime.datetime.now() - data = trendings_helper.getTrendingEvents(dateS, dateE) + specificLabel = request.args.get('specificLabel') + data = trendings_helper.getTrendingEvents(dateS, dateE, specificLabel) return jsonify(data) @app.route("/_getTrendingCategs") diff --git a/static/js/trendings.js b/static/js/trendings.js index ae7b75c..27cfe7d 100644 --- a/static/js/trendings.js +++ b/static/js/trendings.js @@ -70,7 +70,7 @@ var typeaheadOption_event = { } }, updater: function(theevent) { - console.log(theevent); + updateLineForLabel(eventLine, theevent, undefined, url_getTrendingEvent); } } var typeaheadOption_categ = { @@ -166,7 +166,7 @@ function generateEmptyAndFillData(data, specificLabel, colorMapping) { for(var item_arr of items) { var count = item_arr[1]; var itemStr = JSON.stringify(item_arr[0]); - if (specificLabel === undefined || specificLabel == itemStr) { + if (specificLabel === undefined || specificLabel == item_arr[0]) { if(toPlot_obj[itemStr] === undefined) toPlot_obj[itemStr] = {}; toPlot_obj[itemStr][date] = count; @@ -273,7 +273,7 @@ function updatePie(pie, line, data, url) { var specificLabel = obj.series.label; colorMapping[specificLabel] = {}; colorMapping[specificLabel] = { colour: obj.series.color }; - updateLineForLabel(line, specificLabel, colorMapping, url); + updateLineForLabel(line, specificLabel.substring(1, specificLabel.length-1), colorMapping, url); }); for (item of pieWidget.getData()) { colorMapping[item.label] = {colour: item.color}; @@ -344,7 +344,7 @@ function updateSignthingsChart() { } function updateLineForLabel(line, specificLabel, colorMapping, url) { - $.getJSON( url+"?dateS="+parseInt(dateStart.getTime()/1000)+"&dateE="+parseInt(dateEnd.getTime()/1000), function( data ) { + $.getJSON( url+"?dateS="+parseInt(dateStart.getTime()/1000)+"&dateE="+parseInt(dateEnd.getTime()/1000)+"&specificLabel="+specificLabel, function( data ) { updateLine(line, data, undefined, specificLabel, colorMapping); }); } diff --git a/trendings_helper.py b/trendings_helper.py index 8d32b14..e178230 100644 --- a/trendings_helper.py +++ b/trendings_helper.py @@ -65,8 +65,21 @@ class Trendings_helper: to_ret.append([util.getTimestamp(curDate), data]) return to_ret - def getTrendingEvents(self, dateS, dateE): - return self.getGenericTrending('TRENDINGS_EVENTS', dateS, dateE) + def getSpecificTrending(self, trendingType, dateS, dateE, specificLabel=''): + to_ret = [] + prev_days = (dateE - dateS).days + for curDate in util.getXPrevDaysSpan(dateE, prev_days): + keyname = "{}:{}".format(trendingType, util.getDateStrFormat(curDate)) + data = self.serv_redis_db.zscore(keyname, specificLabel) + data = [[specificLabel, data]] if data is not None else [] + to_ret.append([util.getTimestamp(curDate), data]) + return to_ret + + def getTrendingEvents(self, dateS, dateE, specificLabel=None): + if specificLabel is None: + return self.getGenericTrending('TRENDINGS_EVENTS', dateS, dateE) + else: + return self.getSpecificTrending('TRENDINGS_EVENTS', dateS, dateE, specificLabel) def getTrendingCategs(self, dateS, dateE): return self.getGenericTrending('TRENDINGS_CATEGS', dateS, dateE) From 7d5603ab54f03760f10ae73b2e4a4e801c591a5a Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Tue, 21 Nov 2017 09:17:02 +0100 Subject: [PATCH 24/35] Added typeahead plot for categ and tag --- static/js/trendings.js | 39 ++++++++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/static/js/trendings.js b/static/js/trendings.js index 27cfe7d..2654383 100644 --- a/static/js/trendings.js +++ b/static/js/trendings.js @@ -10,6 +10,7 @@ var tagLine = ["#tagLine"]; var sightingLineWidget; var discLine = ["#discussionLine"]; var allData; +var globalColorMapping = {}; /* OPTIONS */ var datePickerOptions = { @@ -85,7 +86,7 @@ var typeaheadOption_categ = { } }, updater: function(categ) { - console.log(categ); + updateLineForLabel(categLine, categ, undefined, url_getTrendingCateg); } } var typeaheadOption_tag = { @@ -100,11 +101,20 @@ var typeaheadOption_tag = { } }, updater: function(tag) { - console.log(tag); + updateLineForLabel(tagLine, tag, undefined, url_getTrendingTag); } } /* FUNCTIONS */ +function getColor(label) { + try { + return globalColorMapping[label]; + } catch(err) { + return undefined; + } + +} + function innerPieLabelFormatter(label, series) { var count = series.data[0][1]; return '
= 50){ - labelLimited = label.substring(0, 50) + '[...]'; + if (label.length >= 40){ + labelLimited = label.substring(0, 40) + '[...]'; } else { labelLimited = label; } diff --git a/templates/trendings.html b/templates/trendings.html index 2647d0f..d4d9c47 100644 --- a/templates/trendings.html +++ b/templates/trendings.html @@ -135,7 +135,7 @@ small {
-
+
Most active events @@ -161,7 +161,7 @@ small {
-
+
Most popular tags From 60470a1ade89e6fcd118ae351f3558cddd3b97ba Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Tue, 21 Nov 2017 14:49:41 +0100 Subject: [PATCH 26/35] fix: bug when label contain '\n' --- trendings_helper.py | 1 + 1 file changed, 1 insertion(+) diff --git a/trendings_helper.py b/trendings_helper.py index e178230..cd04010 100644 --- a/trendings_helper.py +++ b/trendings_helper.py @@ -79,6 +79,7 @@ class Trendings_helper: if specificLabel is None: return self.getGenericTrending('TRENDINGS_EVENTS', dateS, dateE) else: + specificLabel = specificLabel.replace('\\n', '\n'); # reset correctly label with their \n (CR) instead of their char value return self.getSpecificTrending('TRENDINGS_EVENTS', dateS, dateE, specificLabel) def getTrendingCategs(self, dateS, dateE): From 127c1cef5e530fc856b9ef15a3718acd323b8f5e Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Tue, 21 Nov 2017 15:20:07 +0100 Subject: [PATCH 27/35] feature: Added a selector for the maximum number of showed items --- server.py | 7 +++++++ static/js/trendings.js | 9 ++++++++- templates/trendings.html | 13 +++++++++++++ 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/server.py b/server.py index d88622c..dc2e2ec 100755 --- a/server.py +++ b/server.py @@ -205,7 +205,14 @@ def users(): @app.route("/trendings") def trendings(): + maxNum = request.args.get('maxNum') + try: + maxNum = int(maxNum) + except: + maxNum = 15 + return render_template('trendings.html', + maxNum=maxNum ) ''' INDEX ''' diff --git a/static/js/trendings.js b/static/js/trendings.js index 8040985..90319a6 100644 --- a/static/js/trendings.js +++ b/static/js/trendings.js @@ -268,7 +268,8 @@ function updatePie(pie, line, data, url) { } } toPlot.sort(compareObj).reverse(); - toPlot = toPlot.slice(0,15); // take at max 12 elements + var maxNum = $('#num_selector').val(); + toPlot = toPlot.slice(0,maxNum); // take at max 12 elements } if (!(pieWidget === undefined)) { pieWidget.setData(toPlot); @@ -425,6 +426,12 @@ $(document).ready(function () { updateSignthingsChart(); updateDisc(); + $( "#num_selector" ).change(function() { + var sel = parseInt($( this ).val()); + var maxNum = sel; + window.location.href = url_currentPage+'?maxNum='+maxNum; + }); + $("
").css({ position: "absolute", display: "none", diff --git a/templates/trendings.html b/templates/trendings.html index d4d9c47..2fa90c1 100644 --- a/templates/trendings.html +++ b/templates/trendings.html @@ -116,6 +116,18 @@ small {
  • MISP Users
  • MISP Trendings
    • +
    + Max display: + + +
    Date: @@ -212,6 +224,7 @@ small { - - - -