diff --git a/server.py b/server.py
index ef4233a..97ad405 100755
--- a/server.py
+++ b/server.py
@@ -526,6 +526,18 @@ def getTrendingTags():
data = trendings_helper.getTrendingTags(dateS, dateE)
return jsonify(data)
+@app.route("/_getTrendingSightings")
+def getTrendingSightings():
+ try:
+ dateS = datetime.datetime.fromtimestamp(float(request.args.get('dateS')))
+ dateE = datetime.datetime.fromtimestamp(float(request.args.get('dateE')))
+ except:
+ dateS = datetime.datetime.now() - datetime.timedelta(days=7)
+ dateE = datetime.datetime.now()
+
+ data = trendings_helper.getTrendingSightings(dateS, dateE)
+ return jsonify(data)
+
if __name__ == '__main__':
app.run(host='localhost', port=8001, threaded=True)
diff --git a/static/js/trendings.js b/static/js/trendings.js
index 7be04b8..a5a66fc 100644
--- a/static/js/trendings.js
+++ b/static/js/trendings.js
@@ -7,8 +7,7 @@ var categPie = ["#categPie"];
var categLine = ["#categLine"];
var tagPie = ["#tagPie"];
var tagLine = ["#tagLine"];
-var sightingEventPieWidget;
-var sightingCategLineWidget;
+var sightingLineWidget;
/* OPTIONS */
var datePickerOptions = {
@@ -40,9 +39,10 @@ var pieChartOption = {
pie: {
innerRadius: 0.2,
show: true,
+ // radius: 50,
label: {
show: true,
- radius: 4/5,
+ radius: 7/10,
formatter: innerPieLabelFormatter,
}
}
@@ -79,9 +79,7 @@ function getTextColour(rgb) {
}
function legendFormatter(label, series) {
try {
- // transforming true into "true", removing unwanted "
- var jsonLabel = label.replace(/\"/g, "").replace(/True/g, "\"True\"").replace(/False/g, "\"False\"").replace(/\'/g, "\"")
- jsonLabel = JSON.parse(jsonLabel);
+ jsonLabel = JSON.parse(label);
var backgroundColor = jsonLabel.colour;
var color = getTextColour(backgroundColor.substring(1,6));;
var labelText = jsonLabel.name;
@@ -91,17 +89,66 @@ function legendFormatter(label, series) {
+ 'color: ' + color + ';"> ' + labelText + ''
+ '';
} catch(err) {
+ // removing unwanted "
+ var label = label.replace(/\\"/g, "").replace(/\"/g, "");
+ // limiting size
+ if (label.length >= 50){
+ labelLimited = label.substring(0, 50) + '[...]';
+ } else {
+ labelLimited = label;
+ }
return '
';
}
}
+function generateEmptyAndFillData(data) {
+ // formating - Generate empty data
+ var toPlot_obj = {};
+ var allDates = [];
+ var itemMapping = {};
+ for (var arr of data) {
+ var date = new Date(arr[0]*1000);
+ date = new Date(date.valueOf() - date.getTimezoneOffset() * 60000); // center the data around the day
+ allDates.push(date);
+ var items = arr[1];
+ if (items.length > 0) {
+ for(var item_arr of items) {
+ var count = item_arr[1];
+ var itemStr = JSON.stringify(item_arr[0]);
+ itemMapping[itemStr] = item_arr[0];
+ if(toPlot_obj[itemStr] === undefined)
+ toPlot_obj[itemStr] = {};
+ toPlot_obj[itemStr][date] = count;
+ }
+ }
+ }
+ toPlot = []
+ for (var itemStr in toPlot_obj) {
+ if (toPlot_obj.hasOwnProperty(itemStr)) {
+ data_toPlot = []
+ for (var curDate of allDates) {
+ if (toPlot_obj[itemStr].hasOwnProperty(curDate)) {
+ data_toPlot.push([curDate, toPlot_obj[itemStr][curDate]])
+ } else {
+ data_toPlot.push([curDate, 0])
+ }
+ }
+ toPlot.push({label: itemStr, data: data_toPlot, color: itemMapping[itemStr].colour})
+ }
+ }
+ return toPlot;
+}
+
+/* UPDATES */
+
function updatePie(pie, data) {
- pieID = pie[0];
- pieWidget = pie[1];
+ var pieID = pie[0];
+ var pieWidget = pie[1];
+ var itemMapping = {};
if (data === undefined || data.length == 0 || (data[0] == 0 && data[1] == 0)) {
toPlot = [{ label: 'No data', data: 100 }];
} else {
@@ -110,17 +157,18 @@ function updatePie(pie, data) {
var date = arr[0];
var items = arr[1]
for(var item_arr of items) {
- var item = item_arr[0];
+ var itemStr = JSON.stringify(item_arr[0]);
+ itemMapping[itemStr] = item_arr[0];
var count = item_arr[1];
- if(toPlot_obj[item] === undefined)
- toPlot_obj[item] = 0;
- toPlot_obj[item] += count;
+ if(toPlot_obj[itemStr] === undefined)
+ toPlot_obj[itemStr] = 0;
+ toPlot_obj[itemStr] += count;
}
}
toPlot = [];
- for (var item in toPlot_obj) {
- if (toPlot_obj.hasOwnProperty(item)) {
- toPlot.push({label: item, data: toPlot_obj[item]})
+ for (var itemStr in toPlot_obj) {
+ if (toPlot_obj.hasOwnProperty(itemStr)) {
+ toPlot.push({label: itemStr, data: toPlot_obj[itemStr], color: itemMapping[itemStr].colour})
}
}
}
@@ -148,38 +196,7 @@ function updateLine(line, data) {
lineID = line[0];
lineWidget = line[1];
- // formating - Generate empty data
- toPlot_obj = {};
- allDates = [];
- for (var arr of data) {
- var date = new Date(arr[0]*1000);
- date = new Date(date.valueOf() - date.getTimezoneOffset() * 60000); // center the data around the day
- allDates.push(date);
- var items = arr[1];
- if (items.length > 0) {
- for(var item_arr of items) {
- var count = item_arr[1];
- var item = item_arr[0]
- if(toPlot_obj[item] === undefined)
- toPlot_obj[item] = {};
- toPlot_obj[item][date] = count;
- }
- }
- }
- toPlot = []
- for (var item in toPlot_obj) {
- if (toPlot_obj.hasOwnProperty(item)) {
- data_toPlot = []
- for (var curDate of allDates) {
- if (toPlot_obj[item].hasOwnProperty(curDate)) {
- data_toPlot.push([curDate, toPlot_obj[item][curDate]])
- } else {
- data_toPlot.push([curDate, 0])
- }
- }
- toPlot.push({label: item, data: data_toPlot})
- }
- }
+ toPlot = generateEmptyAndFillData(data);
// plot
if (!(lineWidget === undefined)) {
lineWidget.setData(toPlot);
@@ -200,6 +217,40 @@ function updateLine(line, data) {
}
}
+function updateSignthingsChart() {
+ $.getJSON( url_getTrendingSightings+"?dateS="+parseInt(dateStart.getTime()/1000)+"&dateE="+parseInt(dateEnd.getTime()/1000), function( data ) {
+ var toPlot_obj = {};
+ toPlot_obj['Sightings'] = [];
+ toPlot_obj['False positive'] = [];
+ var allDates = [];
+ for (var arr of data) {
+ var date = new Date(arr[0]*1000);
+ date = new Date(date.valueOf() - date.getTimezoneOffset() * 60000); // center the data around the day
+ allDates.push(date);
+ var items = arr[1];
+ var sight = items.sightings;
+ var fp = items.false_positive;
+ toPlot_obj['Sightings'].push([date, sight]);
+ toPlot_obj['False positive'].push([date, -fp]);
+ }
+ toPlot = []
+ toPlot.push({label: 'Sightings', data: toPlot_obj['Sightings'], color: '#4da74d'})
+ toPlot.push({label: 'False positive', data: toPlot_obj['False positive'], color: '#cb4b4b'})
+
+ if (!(sightingLineWidget === undefined)) {
+ sightingLineWidget.setData(toPlot);
+ sightingLineWidget.setupGrid();
+ sightingLineWidget.draw();
+ } else {
+ var lineChartOptionSight = jQuery.extend(true, {}, lineChartOption);
+ lineChartOptionSight['legend']['show'] = true;
+ lineChartOptionSight['legend']['position'] = 'nw';
+ lineChartOptionSight['grid'] = {};
+ sightingLineWidget = $.plot("#sightingLine", toPlot, lineChartOptionSight);
+ }
+ });
+}
+
function updatePieLine(pie, line, url) {
$.getJSON( url+"?dateS="+parseInt(dateStart.getTime()/1000)+"&dateE="+parseInt(dateEnd.getTime()/1000), function( data ) {
updatePie(pie, data);
@@ -210,9 +261,10 @@ function updatePieLine(pie, line, url) {
function dateChanged() {
dateStart = datePickerWidgetStart.datepicker( "getDate" );
dateEnd = datePickerWidgetEnd.datepicker( "getDate" );
- updatePieLine(eventPie, eventLine, url_getTrendingEvent)
- updatePieLine(categPie, categLine, url_getTrendingCateg)
- updatePieLine(tagPie, tagLine, url_getTrendingTag)
+ updatePieLine(eventPie, eventLine, url_getTrendingEvent);
+ updatePieLine(categPie, categLine, url_getTrendingCateg);
+ updatePieLine(tagPie, tagLine, url_getTrendingTag);
+ updateSignthingsChart();
}
$(document).ready(function () {
@@ -229,10 +281,12 @@ $(document).ready(function () {
updatePieLine(eventPie, eventLine, url_getTrendingEvent)
updatePieLine(categPie, categLine, url_getTrendingCateg)
updatePieLine(tagPie, tagLine, url_getTrendingTag)
+ updateSignthingsChart();
$("").css({
position: "absolute",
display: "none",
}).appendTo("body");
+
});
diff --git a/templates/trendings.html b/templates/trendings.html
index cba6a8e..cf34bd3 100644
--- a/templates/trendings.html
+++ b/templates/trendings.html
@@ -140,8 +140,8 @@ small {
Most active events
@@ -152,8 +152,8 @@ small {
Most active categories
@@ -164,8 +164,8 @@ small {
Most popular tags
@@ -178,8 +178,7 @@ small {
Sightings
@@ -189,7 +188,7 @@ small {
Empty
@@ -212,6 +211,7 @@ small {
var url_getTrendingEvent = "{{ url_for('getTrendingEvents') }}";
var url_getTrendingCateg = "{{ url_for('getTrendingCategs') }}";
var url_getTrendingTag = "{{ url_for('getTrendingTags') }}";
+ var url_getTrendingSightings = "{{ url_for('getTrendingSightings') }}";
/* DATA FROM CONF */
diff --git a/trendings_helper.py b/trendings_helper.py
index dbeb684..eba6f52 100644
--- a/trendings_helper.py
+++ b/trendings_helper.py
@@ -2,6 +2,7 @@ import math, random
import os
import json
import datetime, time
+from collections import OrderedDict
import util
@@ -16,7 +17,7 @@ class Trendings_helper:
timestampDate = datetime.datetime.fromtimestamp(float(timestamp))
timestampDate_str = util.getDateStrFormat(timestampDate)
keyname = "{}:{}".format(trendingType, timestampDate_str)
- self.serv_redis_db.zincrby(keyname, data, 1)
+ self.serv_redis_db.zincrby(keyname, json.dumps(data), 1)
def addTrendingEvent(self, eventName, timestamp):
self.addGenericTrending('TRENDINGS_EVENTS', eventName, timestamp)
@@ -26,7 +27,11 @@ class Trendings_helper:
def addTrendingTags(self, tags, timestamp):
for tag in tags:
- self.addGenericTrending('TRENDINGS_TAGS', tag, timestamp)
+ ordDic = OrderedDict() #keep fields with the same layout in redis
+ ordDic['id'] = tag['id']
+ ordDic['name'] = tag['name']
+ ordDic['colour'] = tag['colour']
+ self.addGenericTrending('TRENDINGS_TAGS', ordDic, timestamp)
def addSightings(self, timestamp):
timestampDate = datetime.datetime.fromtimestamp(float(timestamp))
@@ -59,5 +64,30 @@ class Trendings_helper:
def getTrendingCategs(self, dateS, dateE):
return self.getGenericTrending('TRENDINGS_CATEGS', dateS, dateE)
- def getTrendingTags(self, dateS, dateE):
- return self.getGenericTrending('TRENDINGS_TAGS', dateS, dateE)
+ def getTrendingTags(self, dateS, dateE, topNum=12):
+ to_ret = []
+ prev_days = (dateE - dateS).days
+ for curDate in util.getXPrevDaysSpan(dateE, prev_days):
+ keyname = "{}:{}".format('TRENDINGS_TAGS', util.getDateStrFormat(curDate))
+ data = self.serv_redis_db.zrange(keyname, 0, topNum-1, desc=True, withscores=True)
+ data = [ [record[0].decode('utf8'), record[1]] for record in data ]
+ data = data if data is not None else []
+ temp = []
+ for jText, score in data:
+ temp.append([json.loads(jText), score])
+ data = temp
+ to_ret.append([util.getTimestamp(curDate), data])
+ return to_ret
+
+ def getTrendingSightings(self, dateS, dateE):
+ to_ret = []
+ prev_days = (dateE - dateS).days
+ for curDate in util.getXPrevDaysSpan(dateE, prev_days):
+ keyname = "{}:{}".format("TRENDINGS_SIGHT_sightings", util.getDateStrFormat(curDate))
+ sight = self.serv_redis_db.get(keyname)
+ sight = 0 if sight is None else int(sight.decode('utf8'))
+ keyname = "{}:{}".format("TRENDINGS_SIGHT_false_positive", util.getDateStrFormat(curDate))
+ fp = self.serv_redis_db.get(keyname)
+ fp = 0 if fp is None else int(fp.decode('utf8'))
+ to_ret.append([util.getTimestamp(curDate), { 'sightings': sight, 'false_positive': fp}])
+ return to_ret
diff --git a/zmq_subscriber.py b/zmq_subscriber.py
index 2f97965..1a08ef5 100755
--- a/zmq_subscriber.py
+++ b/zmq_subscriber.py
@@ -232,9 +232,16 @@ def handler_sighting(zmq_name, jsondata):
action = None
handleContribution(zmq_name, org, 'Sighting', categ, action, pntMultiplier=2)
handler_attribute(zmq_name, jsonsight, hasAlreadyBeenContributed=True)
-
- trendings_helper.addSightings()
- trendings_helper.addFalsePositive()
+
+ try:
+ timestamp = jsonsight['date_sighting']
+ except KeyError:
+ pass
+
+ if jsonsight['type'] == "0": # sightings
+ trendings_helper.addSightings(timestamp)
+ elif jsonsight['type'] == "1": # false positive
+ trendings_helper.addFalsePositive(timestamp)
def handler_event(zmq_name, jsonobj):
#fields: threat_level_id, id, info
@@ -291,7 +298,10 @@ def handler_attribute(zmq_name, jsonobj, hasAlreadyBeenContributed=False):
#Add trending
categName = jsonattr['category']
- timestamp = jsonattr['timestamp']
+ try:
+ timestamp = jsonattr['timestamp']
+ except KeyError:
+ timestamp = int(time.time())
trendings_helper.addTrendingCateg(categName, timestamp)
try:
temp = jsonattr['Tag']