diff --git a/server.py b/server.py index ef4233a..97ad405 100755 --- a/server.py +++ b/server.py @@ -526,6 +526,18 @@ def getTrendingTags(): data = trendings_helper.getTrendingTags(dateS, dateE) return jsonify(data) +@app.route("/_getTrendingSightings") +def getTrendingSightings(): + try: + dateS = datetime.datetime.fromtimestamp(float(request.args.get('dateS'))) + dateE = datetime.datetime.fromtimestamp(float(request.args.get('dateE'))) + except: + dateS = datetime.datetime.now() - datetime.timedelta(days=7) + dateE = datetime.datetime.now() + + data = trendings_helper.getTrendingSightings(dateS, dateE) + return jsonify(data) + if __name__ == '__main__': app.run(host='localhost', port=8001, threaded=True) diff --git a/static/js/trendings.js b/static/js/trendings.js index 7be04b8..a5a66fc 100644 --- a/static/js/trendings.js +++ b/static/js/trendings.js @@ -7,8 +7,7 @@ var categPie = ["#categPie"]; var categLine = ["#categLine"]; var tagPie = ["#tagPie"]; var tagLine = ["#tagLine"]; -var sightingEventPieWidget; -var sightingCategLineWidget; +var sightingLineWidget; /* OPTIONS */ var datePickerOptions = { @@ -40,9 +39,10 @@ var pieChartOption = { pie: { innerRadius: 0.2, show: true, + // radius: 50, label: { show: true, - radius: 4/5, + radius: 7/10, formatter: innerPieLabelFormatter, } } @@ -79,9 +79,7 @@ function getTextColour(rgb) { } function legendFormatter(label, series) { try { - // transforming true into "true", removing unwanted " - var jsonLabel = label.replace(/\"/g, "").replace(/True/g, "\"True\"").replace(/False/g, "\"False\"").replace(/\'/g, "\"") - jsonLabel = JSON.parse(jsonLabel); + jsonLabel = JSON.parse(label); var backgroundColor = jsonLabel.colour; var color = getTextColour(backgroundColor.substring(1,6));; var labelText = jsonLabel.name; @@ -91,17 +89,66 @@ function legendFormatter(label, series) { + 'color: ' + color + ';"> ' + labelText + '' + ''; } catch(err) { + // removing unwanted " + var label = label.replace(/\\"/g, "").replace(/\"/g, ""); + // limiting size + if (label.length >= 50){ + labelLimited = label.substring(0, 50) + '[...]'; + } else { + labelLimited = label; + } return '
' - + ' ' + label + + ' ' + labelLimited + ''; + '
'; } } +function generateEmptyAndFillData(data) { + // formating - Generate empty data + var toPlot_obj = {}; + var allDates = []; + var itemMapping = {}; + for (var arr of data) { + var date = new Date(arr[0]*1000); + date = new Date(date.valueOf() - date.getTimezoneOffset() * 60000); // center the data around the day + allDates.push(date); + var items = arr[1]; + if (items.length > 0) { + for(var item_arr of items) { + var count = item_arr[1]; + var itemStr = JSON.stringify(item_arr[0]); + itemMapping[itemStr] = item_arr[0]; + if(toPlot_obj[itemStr] === undefined) + toPlot_obj[itemStr] = {}; + toPlot_obj[itemStr][date] = count; + } + } + } + toPlot = [] + for (var itemStr in toPlot_obj) { + if (toPlot_obj.hasOwnProperty(itemStr)) { + data_toPlot = [] + for (var curDate of allDates) { + if (toPlot_obj[itemStr].hasOwnProperty(curDate)) { + data_toPlot.push([curDate, toPlot_obj[itemStr][curDate]]) + } else { + data_toPlot.push([curDate, 0]) + } + } + toPlot.push({label: itemStr, data: data_toPlot, color: itemMapping[itemStr].colour}) + } + } + return toPlot; +} + +/* UPDATES */ + function updatePie(pie, data) { - pieID = pie[0]; - pieWidget = pie[1]; + var pieID = pie[0]; + var pieWidget = pie[1]; + var itemMapping = {}; if (data === undefined || data.length == 0 || (data[0] == 0 && data[1] == 0)) { toPlot = [{ label: 'No data', data: 100 }]; } else { @@ -110,17 +157,18 @@ function updatePie(pie, data) { var date = arr[0]; var items = arr[1] for(var item_arr of items) { - var item = item_arr[0]; + var itemStr = JSON.stringify(item_arr[0]); + itemMapping[itemStr] = item_arr[0]; var count = item_arr[1]; - if(toPlot_obj[item] === undefined) - toPlot_obj[item] = 0; - toPlot_obj[item] += count; + if(toPlot_obj[itemStr] === undefined) + toPlot_obj[itemStr] = 0; + toPlot_obj[itemStr] += count; } } toPlot = []; - for (var item in toPlot_obj) { - if (toPlot_obj.hasOwnProperty(item)) { - toPlot.push({label: item, data: toPlot_obj[item]}) + for (var itemStr in toPlot_obj) { + if (toPlot_obj.hasOwnProperty(itemStr)) { + toPlot.push({label: itemStr, data: toPlot_obj[itemStr], color: itemMapping[itemStr].colour}) } } } @@ -148,38 +196,7 @@ function updateLine(line, data) { lineID = line[0]; lineWidget = line[1]; - // formating - Generate empty data - toPlot_obj = {}; - allDates = []; - for (var arr of data) { - var date = new Date(arr[0]*1000); - date = new Date(date.valueOf() - date.getTimezoneOffset() * 60000); // center the data around the day - allDates.push(date); - var items = arr[1]; - if (items.length > 0) { - for(var item_arr of items) { - var count = item_arr[1]; - var item = item_arr[0] - if(toPlot_obj[item] === undefined) - toPlot_obj[item] = {}; - toPlot_obj[item][date] = count; - } - } - } - toPlot = [] - for (var item in toPlot_obj) { - if (toPlot_obj.hasOwnProperty(item)) { - data_toPlot = [] - for (var curDate of allDates) { - if (toPlot_obj[item].hasOwnProperty(curDate)) { - data_toPlot.push([curDate, toPlot_obj[item][curDate]]) - } else { - data_toPlot.push([curDate, 0]) - } - } - toPlot.push({label: item, data: data_toPlot}) - } - } + toPlot = generateEmptyAndFillData(data); // plot if (!(lineWidget === undefined)) { lineWidget.setData(toPlot); @@ -200,6 +217,40 @@ function updateLine(line, data) { } } +function updateSignthingsChart() { + $.getJSON( url_getTrendingSightings+"?dateS="+parseInt(dateStart.getTime()/1000)+"&dateE="+parseInt(dateEnd.getTime()/1000), function( data ) { + var toPlot_obj = {}; + toPlot_obj['Sightings'] = []; + toPlot_obj['False positive'] = []; + var allDates = []; + for (var arr of data) { + var date = new Date(arr[0]*1000); + date = new Date(date.valueOf() - date.getTimezoneOffset() * 60000); // center the data around the day + allDates.push(date); + var items = arr[1]; + var sight = items.sightings; + var fp = items.false_positive; + toPlot_obj['Sightings'].push([date, sight]); + toPlot_obj['False positive'].push([date, -fp]); + } + toPlot = [] + toPlot.push({label: 'Sightings', data: toPlot_obj['Sightings'], color: '#4da74d'}) + toPlot.push({label: 'False positive', data: toPlot_obj['False positive'], color: '#cb4b4b'}) + + if (!(sightingLineWidget === undefined)) { + sightingLineWidget.setData(toPlot); + sightingLineWidget.setupGrid(); + sightingLineWidget.draw(); + } else { + var lineChartOptionSight = jQuery.extend(true, {}, lineChartOption); + lineChartOptionSight['legend']['show'] = true; + lineChartOptionSight['legend']['position'] = 'nw'; + lineChartOptionSight['grid'] = {}; + sightingLineWidget = $.plot("#sightingLine", toPlot, lineChartOptionSight); + } + }); +} + function updatePieLine(pie, line, url) { $.getJSON( url+"?dateS="+parseInt(dateStart.getTime()/1000)+"&dateE="+parseInt(dateEnd.getTime()/1000), function( data ) { updatePie(pie, data); @@ -210,9 +261,10 @@ function updatePieLine(pie, line, url) { function dateChanged() { dateStart = datePickerWidgetStart.datepicker( "getDate" ); dateEnd = datePickerWidgetEnd.datepicker( "getDate" ); - updatePieLine(eventPie, eventLine, url_getTrendingEvent) - updatePieLine(categPie, categLine, url_getTrendingCateg) - updatePieLine(tagPie, tagLine, url_getTrendingTag) + updatePieLine(eventPie, eventLine, url_getTrendingEvent); + updatePieLine(categPie, categLine, url_getTrendingCateg); + updatePieLine(tagPie, tagLine, url_getTrendingTag); + updateSignthingsChart(); } $(document).ready(function () { @@ -229,10 +281,12 @@ $(document).ready(function () { updatePieLine(eventPie, eventLine, url_getTrendingEvent) updatePieLine(categPie, categLine, url_getTrendingCateg) updatePieLine(tagPie, tagLine, url_getTrendingTag) + updateSignthingsChart(); $("
").css({ position: "absolute", display: "none", }).appendTo("body"); + }); diff --git a/templates/trendings.html b/templates/trendings.html index cba6a8e..cf34bd3 100644 --- a/templates/trendings.html +++ b/templates/trendings.html @@ -140,8 +140,8 @@ small { Most active events
-
-
+
+
@@ -152,8 +152,8 @@ small { Most active categories
-
-
+
+
@@ -164,8 +164,8 @@ small { Most popular tags
-
-
+
+
@@ -178,8 +178,7 @@ small { Sightings
-
-
+
@@ -189,7 +188,7 @@ small { Empty
-
+
@@ -212,6 +211,7 @@ small { var url_getTrendingEvent = "{{ url_for('getTrendingEvents') }}"; var url_getTrendingCateg = "{{ url_for('getTrendingCategs') }}"; var url_getTrendingTag = "{{ url_for('getTrendingTags') }}"; + var url_getTrendingSightings = "{{ url_for('getTrendingSightings') }}"; /* DATA FROM CONF */ diff --git a/trendings_helper.py b/trendings_helper.py index dbeb684..eba6f52 100644 --- a/trendings_helper.py +++ b/trendings_helper.py @@ -2,6 +2,7 @@ import math, random import os import json import datetime, time +from collections import OrderedDict import util @@ -16,7 +17,7 @@ class Trendings_helper: timestampDate = datetime.datetime.fromtimestamp(float(timestamp)) timestampDate_str = util.getDateStrFormat(timestampDate) keyname = "{}:{}".format(trendingType, timestampDate_str) - self.serv_redis_db.zincrby(keyname, data, 1) + self.serv_redis_db.zincrby(keyname, json.dumps(data), 1) def addTrendingEvent(self, eventName, timestamp): self.addGenericTrending('TRENDINGS_EVENTS', eventName, timestamp) @@ -26,7 +27,11 @@ class Trendings_helper: def addTrendingTags(self, tags, timestamp): for tag in tags: - self.addGenericTrending('TRENDINGS_TAGS', tag, timestamp) + ordDic = OrderedDict() #keep fields with the same layout in redis + ordDic['id'] = tag['id'] + ordDic['name'] = tag['name'] + ordDic['colour'] = tag['colour'] + self.addGenericTrending('TRENDINGS_TAGS', ordDic, timestamp) def addSightings(self, timestamp): timestampDate = datetime.datetime.fromtimestamp(float(timestamp)) @@ -59,5 +64,30 @@ class Trendings_helper: def getTrendingCategs(self, dateS, dateE): return self.getGenericTrending('TRENDINGS_CATEGS', dateS, dateE) - def getTrendingTags(self, dateS, dateE): - return self.getGenericTrending('TRENDINGS_TAGS', dateS, dateE) + def getTrendingTags(self, dateS, dateE, topNum=12): + to_ret = [] + prev_days = (dateE - dateS).days + for curDate in util.getXPrevDaysSpan(dateE, prev_days): + keyname = "{}:{}".format('TRENDINGS_TAGS', util.getDateStrFormat(curDate)) + data = self.serv_redis_db.zrange(keyname, 0, topNum-1, desc=True, withscores=True) + data = [ [record[0].decode('utf8'), record[1]] for record in data ] + data = data if data is not None else [] + temp = [] + for jText, score in data: + temp.append([json.loads(jText), score]) + data = temp + to_ret.append([util.getTimestamp(curDate), data]) + return to_ret + + def getTrendingSightings(self, dateS, dateE): + to_ret = [] + prev_days = (dateE - dateS).days + for curDate in util.getXPrevDaysSpan(dateE, prev_days): + keyname = "{}:{}".format("TRENDINGS_SIGHT_sightings", util.getDateStrFormat(curDate)) + sight = self.serv_redis_db.get(keyname) + sight = 0 if sight is None else int(sight.decode('utf8')) + keyname = "{}:{}".format("TRENDINGS_SIGHT_false_positive", util.getDateStrFormat(curDate)) + fp = self.serv_redis_db.get(keyname) + fp = 0 if fp is None else int(fp.decode('utf8')) + to_ret.append([util.getTimestamp(curDate), { 'sightings': sight, 'false_positive': fp}]) + return to_ret diff --git a/zmq_subscriber.py b/zmq_subscriber.py index 2f97965..1a08ef5 100755 --- a/zmq_subscriber.py +++ b/zmq_subscriber.py @@ -232,9 +232,16 @@ def handler_sighting(zmq_name, jsondata): action = None handleContribution(zmq_name, org, 'Sighting', categ, action, pntMultiplier=2) handler_attribute(zmq_name, jsonsight, hasAlreadyBeenContributed=True) - - trendings_helper.addSightings() - trendings_helper.addFalsePositive() + + try: + timestamp = jsonsight['date_sighting'] + except KeyError: + pass + + if jsonsight['type'] == "0": # sightings + trendings_helper.addSightings(timestamp) + elif jsonsight['type'] == "1": # false positive + trendings_helper.addFalsePositive(timestamp) def handler_event(zmq_name, jsonobj): #fields: threat_level_id, id, info @@ -291,7 +298,10 @@ def handler_attribute(zmq_name, jsonobj, hasAlreadyBeenContributed=False): #Add trending categName = jsonattr['category'] - timestamp = jsonattr['timestamp'] + try: + timestamp = jsonattr['timestamp'] + except KeyError: + timestamp = int(time.time()) trendings_helper.addTrendingCateg(categName, timestamp) try: temp = jsonattr['Tag']