mirror of https://github.com/MISP/misp-dashboard
fix and feature: Added data ordering in reids (useing orderedDico) +
Added sightings in UI + colored charts woth tag color"pull/9/head
Sami Mokaddem
parent
074998c227
commit
f55ad8255c
12
server.py
12
server.py
|
|
@ -526,6 +526,18 @@ def getTrendingTags():
|
|||
data = trendings_helper.getTrendingTags(dateS, dateE)
|
||||
return jsonify(data)
|
||||
|
||||
@app.route("/_getTrendingSightings")
|
||||
def getTrendingSightings():
|
||||
try:
|
||||
dateS = datetime.datetime.fromtimestamp(float(request.args.get('dateS')))
|
||||
dateE = datetime.datetime.fromtimestamp(float(request.args.get('dateE')))
|
||||
except:
|
||||
dateS = datetime.datetime.now() - datetime.timedelta(days=7)
|
||||
dateE = datetime.datetime.now()
|
||||
|
||||
data = trendings_helper.getTrendingSightings(dateS, dateE)
|
||||
return jsonify(data)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
app.run(host='localhost', port=8001, threaded=True)
|
||||
|
|
|
|||
|
|
@ -7,8 +7,7 @@ var categPie = ["#categPie"];
|
|||
var categLine = ["#categLine"];
|
||||
var tagPie = ["#tagPie"];
|
||||
var tagLine = ["#tagLine"];
|
||||
var sightingEventPieWidget;
|
||||
var sightingCategLineWidget;
|
||||
var sightingLineWidget;
|
||||
|
||||
/* OPTIONS */
|
||||
var datePickerOptions = {
|
||||
|
|
@ -40,9 +39,10 @@ var pieChartOption = {
|
|||
pie: {
|
||||
innerRadius: 0.2,
|
||||
show: true,
|
||||
// radius: 50,
|
||||
label: {
|
||||
show: true,
|
||||
radius: 4/5,
|
||||
radius: 7/10,
|
||||
formatter: innerPieLabelFormatter,
|
||||
}
|
||||
}
|
||||
|
|
@ -79,9 +79,7 @@ function getTextColour(rgb) {
|
|||
}
|
||||
function legendFormatter(label, series) {
|
||||
try {
|
||||
// transforming true into "true", removing unwanted "
|
||||
var jsonLabel = label.replace(/\"/g, "").replace(/True/g, "\"True\"").replace(/False/g, "\"False\"").replace(/\'/g, "\"")
|
||||
jsonLabel = JSON.parse(jsonLabel);
|
||||
jsonLabel = JSON.parse(label);
|
||||
var backgroundColor = jsonLabel.colour;
|
||||
var color = getTextColour(backgroundColor.substring(1,6));;
|
||||
var labelText = jsonLabel.name;
|
||||
|
|
@ -91,17 +89,66 @@ function legendFormatter(label, series) {
|
|||
+ 'color: ' + color + ';"> ' + labelText + '</a>'
|
||||
+ '</div>';
|
||||
} catch(err) {
|
||||
// removing unwanted "
|
||||
var label = label.replace(/\\"/g, "").replace(/\"/g, "");
|
||||
// limiting size
|
||||
if (label.length >= 50){
|
||||
labelLimited = label.substring(0, 50) + '[...]';
|
||||
} else {
|
||||
labelLimited = label;
|
||||
}
|
||||
return '<div '
|
||||
+ 'style="font-size:8pt;text-align:inherit;padding:2px;">'
|
||||
+ '<a class="tagElem" style="background-color: white; color: black;"> ' + label
|
||||
+ '<a class="tagElem" style="background-color: white; color: black;"> ' + labelLimited
|
||||
+ '</a>';
|
||||
+ '</div>';
|
||||
}
|
||||
}
|
||||
|
||||
function generateEmptyAndFillData(data) {
|
||||
// formating - Generate empty data
|
||||
var toPlot_obj = {};
|
||||
var allDates = [];
|
||||
var itemMapping = {};
|
||||
for (var arr of data) {
|
||||
var date = new Date(arr[0]*1000);
|
||||
date = new Date(date.valueOf() - date.getTimezoneOffset() * 60000); // center the data around the day
|
||||
allDates.push(date);
|
||||
var items = arr[1];
|
||||
if (items.length > 0) {
|
||||
for(var item_arr of items) {
|
||||
var count = item_arr[1];
|
||||
var itemStr = JSON.stringify(item_arr[0]);
|
||||
itemMapping[itemStr] = item_arr[0];
|
||||
if(toPlot_obj[itemStr] === undefined)
|
||||
toPlot_obj[itemStr] = {};
|
||||
toPlot_obj[itemStr][date] = count;
|
||||
}
|
||||
}
|
||||
}
|
||||
toPlot = []
|
||||
for (var itemStr in toPlot_obj) {
|
||||
if (toPlot_obj.hasOwnProperty(itemStr)) {
|
||||
data_toPlot = []
|
||||
for (var curDate of allDates) {
|
||||
if (toPlot_obj[itemStr].hasOwnProperty(curDate)) {
|
||||
data_toPlot.push([curDate, toPlot_obj[itemStr][curDate]])
|
||||
} else {
|
||||
data_toPlot.push([curDate, 0])
|
||||
}
|
||||
}
|
||||
toPlot.push({label: itemStr, data: data_toPlot, color: itemMapping[itemStr].colour})
|
||||
}
|
||||
}
|
||||
return toPlot;
|
||||
}
|
||||
|
||||
/* UPDATES */
|
||||
|
||||
function updatePie(pie, data) {
|
||||
pieID = pie[0];
|
||||
pieWidget = pie[1];
|
||||
var pieID = pie[0];
|
||||
var pieWidget = pie[1];
|
||||
var itemMapping = {};
|
||||
if (data === undefined || data.length == 0 || (data[0] == 0 && data[1] == 0)) {
|
||||
toPlot = [{ label: 'No data', data: 100 }];
|
||||
} else {
|
||||
|
|
@ -110,17 +157,18 @@ function updatePie(pie, data) {
|
|||
var date = arr[0];
|
||||
var items = arr[1]
|
||||
for(var item_arr of items) {
|
||||
var item = item_arr[0];
|
||||
var itemStr = JSON.stringify(item_arr[0]);
|
||||
itemMapping[itemStr] = item_arr[0];
|
||||
var count = item_arr[1];
|
||||
if(toPlot_obj[item] === undefined)
|
||||
toPlot_obj[item] = 0;
|
||||
toPlot_obj[item] += count;
|
||||
if(toPlot_obj[itemStr] === undefined)
|
||||
toPlot_obj[itemStr] = 0;
|
||||
toPlot_obj[itemStr] += count;
|
||||
}
|
||||
}
|
||||
toPlot = [];
|
||||
for (var item in toPlot_obj) {
|
||||
if (toPlot_obj.hasOwnProperty(item)) {
|
||||
toPlot.push({label: item, data: toPlot_obj[item]})
|
||||
for (var itemStr in toPlot_obj) {
|
||||
if (toPlot_obj.hasOwnProperty(itemStr)) {
|
||||
toPlot.push({label: itemStr, data: toPlot_obj[itemStr], color: itemMapping[itemStr].colour})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -148,38 +196,7 @@ function updateLine(line, data) {
|
|||
lineID = line[0];
|
||||
lineWidget = line[1];
|
||||
|
||||
// formating - Generate empty data
|
||||
toPlot_obj = {};
|
||||
allDates = [];
|
||||
for (var arr of data) {
|
||||
var date = new Date(arr[0]*1000);
|
||||
date = new Date(date.valueOf() - date.getTimezoneOffset() * 60000); // center the data around the day
|
||||
allDates.push(date);
|
||||
var items = arr[1];
|
||||
if (items.length > 0) {
|
||||
for(var item_arr of items) {
|
||||
var count = item_arr[1];
|
||||
var item = item_arr[0]
|
||||
if(toPlot_obj[item] === undefined)
|
||||
toPlot_obj[item] = {};
|
||||
toPlot_obj[item][date] = count;
|
||||
}
|
||||
}
|
||||
}
|
||||
toPlot = []
|
||||
for (var item in toPlot_obj) {
|
||||
if (toPlot_obj.hasOwnProperty(item)) {
|
||||
data_toPlot = []
|
||||
for (var curDate of allDates) {
|
||||
if (toPlot_obj[item].hasOwnProperty(curDate)) {
|
||||
data_toPlot.push([curDate, toPlot_obj[item][curDate]])
|
||||
} else {
|
||||
data_toPlot.push([curDate, 0])
|
||||
}
|
||||
}
|
||||
toPlot.push({label: item, data: data_toPlot})
|
||||
}
|
||||
}
|
||||
toPlot = generateEmptyAndFillData(data);
|
||||
// plot
|
||||
if (!(lineWidget === undefined)) {
|
||||
lineWidget.setData(toPlot);
|
||||
|
|
@ -200,6 +217,40 @@ function updateLine(line, data) {
|
|||
}
|
||||
}
|
||||
|
||||
function updateSignthingsChart() {
|
||||
$.getJSON( url_getTrendingSightings+"?dateS="+parseInt(dateStart.getTime()/1000)+"&dateE="+parseInt(dateEnd.getTime()/1000), function( data ) {
|
||||
var toPlot_obj = {};
|
||||
toPlot_obj['Sightings'] = [];
|
||||
toPlot_obj['False positive'] = [];
|
||||
var allDates = [];
|
||||
for (var arr of data) {
|
||||
var date = new Date(arr[0]*1000);
|
||||
date = new Date(date.valueOf() - date.getTimezoneOffset() * 60000); // center the data around the day
|
||||
allDates.push(date);
|
||||
var items = arr[1];
|
||||
var sight = items.sightings;
|
||||
var fp = items.false_positive;
|
||||
toPlot_obj['Sightings'].push([date, sight]);
|
||||
toPlot_obj['False positive'].push([date, -fp]);
|
||||
}
|
||||
toPlot = []
|
||||
toPlot.push({label: 'Sightings', data: toPlot_obj['Sightings'], color: '#4da74d'})
|
||||
toPlot.push({label: 'False positive', data: toPlot_obj['False positive'], color: '#cb4b4b'})
|
||||
|
||||
if (!(sightingLineWidget === undefined)) {
|
||||
sightingLineWidget.setData(toPlot);
|
||||
sightingLineWidget.setupGrid();
|
||||
sightingLineWidget.draw();
|
||||
} else {
|
||||
var lineChartOptionSight = jQuery.extend(true, {}, lineChartOption);
|
||||
lineChartOptionSight['legend']['show'] = true;
|
||||
lineChartOptionSight['legend']['position'] = 'nw';
|
||||
lineChartOptionSight['grid'] = {};
|
||||
sightingLineWidget = $.plot("#sightingLine", toPlot, lineChartOptionSight);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function updatePieLine(pie, line, url) {
|
||||
$.getJSON( url+"?dateS="+parseInt(dateStart.getTime()/1000)+"&dateE="+parseInt(dateEnd.getTime()/1000), function( data ) {
|
||||
updatePie(pie, data);
|
||||
|
|
@ -210,9 +261,10 @@ function updatePieLine(pie, line, url) {
|
|||
function dateChanged() {
|
||||
dateStart = datePickerWidgetStart.datepicker( "getDate" );
|
||||
dateEnd = datePickerWidgetEnd.datepicker( "getDate" );
|
||||
updatePieLine(eventPie, eventLine, url_getTrendingEvent)
|
||||
updatePieLine(categPie, categLine, url_getTrendingCateg)
|
||||
updatePieLine(tagPie, tagLine, url_getTrendingTag)
|
||||
updatePieLine(eventPie, eventLine, url_getTrendingEvent);
|
||||
updatePieLine(categPie, categLine, url_getTrendingCateg);
|
||||
updatePieLine(tagPie, tagLine, url_getTrendingTag);
|
||||
updateSignthingsChart();
|
||||
}
|
||||
|
||||
$(document).ready(function () {
|
||||
|
|
@ -229,10 +281,12 @@ $(document).ready(function () {
|
|||
updatePieLine(eventPie, eventLine, url_getTrendingEvent)
|
||||
updatePieLine(categPie, categLine, url_getTrendingCateg)
|
||||
updatePieLine(tagPie, tagLine, url_getTrendingTag)
|
||||
updateSignthingsChart();
|
||||
|
||||
|
||||
$("<div id='tooltip'></div>").css({
|
||||
position: "absolute",
|
||||
display: "none",
|
||||
}).appendTo("body");
|
||||
|
||||
});
|
||||
|
|
|
|||
|
|
@ -140,8 +140,8 @@ small {
|
|||
<b>Most active events</b>
|
||||
</div>
|
||||
<div id="panelbody" class="panel-body" style="">
|
||||
<div id="eventPie" style="width:100%; height: 25vh;"></div>
|
||||
<div id="eventLine" style="width:100%; height: 25vh;"></div>
|
||||
<div id="eventPie" style="width:100%; height: 34vh;"></div>
|
||||
<div id="eventLine" style="width:100%; height: 27vh;"></div>
|
||||
</div>
|
||||
</div><!-- /.panel-body -->
|
||||
</div>
|
||||
|
|
@ -152,8 +152,8 @@ small {
|
|||
<b>Most active categories</b>
|
||||
</div>
|
||||
<div id="panelbody" class="panel-body" style="">
|
||||
<div id="categPie" style="width:100%; height: 25vh;"></div>
|
||||
<div id="categLine" style="width:100%; height: 25vh;"></div>
|
||||
<div id="categPie" style="width:100%; height: 34vh;"></div>
|
||||
<div id="categLine" style="width:100%; height: 27vh;"></div>
|
||||
</div>
|
||||
</div><!-- /.panel-body -->
|
||||
</div>
|
||||
|
|
@ -164,8 +164,8 @@ small {
|
|||
<b>Most popular tags</b>
|
||||
</div>
|
||||
<div id="panelbody" class="panel-body" style="">
|
||||
<div id="tagPie" style="width:100%; height: 25vh;"></div>
|
||||
<div id="tagLine" style="width:100%; height: 25vh;"></div>
|
||||
<div id="tagPie" style="width:100%; height: 34vh;"></div>
|
||||
<div id="tagLine" style="width:100%; height: 27vh;"></div>
|
||||
</div>
|
||||
</div><!-- /.panel-body -->
|
||||
</div>
|
||||
|
|
@ -178,8 +178,7 @@ small {
|
|||
<b>Sightings</b>
|
||||
</div>
|
||||
<div id="panelbody" class="panel-body" style="">
|
||||
<div id="sightingEventPie" style="width:100%; height: 10vh;"></div>
|
||||
<div id="sightingCategLine" style="width:100%; height: 10vh;"></div>
|
||||
<div id="sightingLine" style="width:100%; height: 20vh;"></div>
|
||||
</div>
|
||||
</div><!-- /.panel-body -->
|
||||
</div>
|
||||
|
|
@ -189,7 +188,7 @@ small {
|
|||
<b>Empty</b>
|
||||
</div>
|
||||
<div id="panelbody" class="panel-body" style="">
|
||||
<div id="" style="width:100%; height: 10vh;"></div>
|
||||
<div id="" style="width:100%; height: 20vh;"></div>
|
||||
</div>
|
||||
</div><!-- /.panel-body -->
|
||||
</div>
|
||||
|
|
@ -212,6 +211,7 @@ small {
|
|||
var url_getTrendingEvent = "{{ url_for('getTrendingEvents') }}";
|
||||
var url_getTrendingCateg = "{{ url_for('getTrendingCategs') }}";
|
||||
var url_getTrendingTag = "{{ url_for('getTrendingTags') }}";
|
||||
var url_getTrendingSightings = "{{ url_for('getTrendingSightings') }}";
|
||||
|
||||
/* DATA FROM CONF */
|
||||
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import math, random
|
|||
import os
|
||||
import json
|
||||
import datetime, time
|
||||
from collections import OrderedDict
|
||||
|
||||
import util
|
||||
|
||||
|
|
@ -16,7 +17,7 @@ class Trendings_helper:
|
|||
timestampDate = datetime.datetime.fromtimestamp(float(timestamp))
|
||||
timestampDate_str = util.getDateStrFormat(timestampDate)
|
||||
keyname = "{}:{}".format(trendingType, timestampDate_str)
|
||||
self.serv_redis_db.zincrby(keyname, data, 1)
|
||||
self.serv_redis_db.zincrby(keyname, json.dumps(data), 1)
|
||||
|
||||
def addTrendingEvent(self, eventName, timestamp):
|
||||
self.addGenericTrending('TRENDINGS_EVENTS', eventName, timestamp)
|
||||
|
|
@ -26,7 +27,11 @@ class Trendings_helper:
|
|||
|
||||
def addTrendingTags(self, tags, timestamp):
|
||||
for tag in tags:
|
||||
self.addGenericTrending('TRENDINGS_TAGS', tag, timestamp)
|
||||
ordDic = OrderedDict() #keep fields with the same layout in redis
|
||||
ordDic['id'] = tag['id']
|
||||
ordDic['name'] = tag['name']
|
||||
ordDic['colour'] = tag['colour']
|
||||
self.addGenericTrending('TRENDINGS_TAGS', ordDic, timestamp)
|
||||
|
||||
def addSightings(self, timestamp):
|
||||
timestampDate = datetime.datetime.fromtimestamp(float(timestamp))
|
||||
|
|
@ -59,5 +64,30 @@ class Trendings_helper:
|
|||
def getTrendingCategs(self, dateS, dateE):
|
||||
return self.getGenericTrending('TRENDINGS_CATEGS', dateS, dateE)
|
||||
|
||||
def getTrendingTags(self, dateS, dateE):
|
||||
return self.getGenericTrending('TRENDINGS_TAGS', dateS, dateE)
|
||||
def getTrendingTags(self, dateS, dateE, topNum=12):
|
||||
to_ret = []
|
||||
prev_days = (dateE - dateS).days
|
||||
for curDate in util.getXPrevDaysSpan(dateE, prev_days):
|
||||
keyname = "{}:{}".format('TRENDINGS_TAGS', util.getDateStrFormat(curDate))
|
||||
data = self.serv_redis_db.zrange(keyname, 0, topNum-1, desc=True, withscores=True)
|
||||
data = [ [record[0].decode('utf8'), record[1]] for record in data ]
|
||||
data = data if data is not None else []
|
||||
temp = []
|
||||
for jText, score in data:
|
||||
temp.append([json.loads(jText), score])
|
||||
data = temp
|
||||
to_ret.append([util.getTimestamp(curDate), data])
|
||||
return to_ret
|
||||
|
||||
def getTrendingSightings(self, dateS, dateE):
|
||||
to_ret = []
|
||||
prev_days = (dateE - dateS).days
|
||||
for curDate in util.getXPrevDaysSpan(dateE, prev_days):
|
||||
keyname = "{}:{}".format("TRENDINGS_SIGHT_sightings", util.getDateStrFormat(curDate))
|
||||
sight = self.serv_redis_db.get(keyname)
|
||||
sight = 0 if sight is None else int(sight.decode('utf8'))
|
||||
keyname = "{}:{}".format("TRENDINGS_SIGHT_false_positive", util.getDateStrFormat(curDate))
|
||||
fp = self.serv_redis_db.get(keyname)
|
||||
fp = 0 if fp is None else int(fp.decode('utf8'))
|
||||
to_ret.append([util.getTimestamp(curDate), { 'sightings': sight, 'false_positive': fp}])
|
||||
return to_ret
|
||||
|
|
|
|||
|
|
@ -233,8 +233,15 @@ def handler_sighting(zmq_name, jsondata):
|
|||
handleContribution(zmq_name, org, 'Sighting', categ, action, pntMultiplier=2)
|
||||
handler_attribute(zmq_name, jsonsight, hasAlreadyBeenContributed=True)
|
||||
|
||||
trendings_helper.addSightings()
|
||||
trendings_helper.addFalsePositive()
|
||||
try:
|
||||
timestamp = jsonsight['date_sighting']
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
if jsonsight['type'] == "0": # sightings
|
||||
trendings_helper.addSightings(timestamp)
|
||||
elif jsonsight['type'] == "1": # false positive
|
||||
trendings_helper.addFalsePositive(timestamp)
|
||||
|
||||
def handler_event(zmq_name, jsonobj):
|
||||
#fields: threat_level_id, id, info
|
||||
|
|
@ -291,7 +298,10 @@ def handler_attribute(zmq_name, jsonobj, hasAlreadyBeenContributed=False):
|
|||
|
||||
#Add trending
|
||||
categName = jsonattr['category']
|
||||
try:
|
||||
timestamp = jsonattr['timestamp']
|
||||
except KeyError:
|
||||
timestamp = int(time.time())
|
||||
trendings_helper.addTrendingCateg(categName, timestamp)
|
||||
try:
|
||||
temp = jsonattr['Tag']
|
||||
|
|
|
|||
Loading…
Reference in New Issue