mirror of https://github.com/MISP/misp-dashboard
Loads of bug fixes: Added possibility to plot any field in lineChart + Fixed bug GEO where data was not from the correct date + Clear view when no data + Probably solved unsorted dic stored in redis + Support of publish_event_to_zmq
parent
c566cea338
commit
62522f822f
|
@ -6,6 +6,7 @@ rotation_wait_time = 30
|
||||||
max_img_rotation = 10
|
max_img_rotation = 10
|
||||||
hours_spanned = 48
|
hours_spanned = 48
|
||||||
zoomlevel = 15
|
zoomlevel = 15
|
||||||
|
item_to_plot = Attribute.category
|
||||||
# [1->12]
|
# [1->12]
|
||||||
size_dashboard_left_width = 5
|
size_dashboard_left_width = 5
|
||||||
size_openStreet_pannel_perc = 55
|
size_openStreet_pannel_perc = 55
|
||||||
|
@ -19,8 +20,8 @@ zoomlevel = 11
|
||||||
clusteringDistance = 10
|
clusteringDistance = 10
|
||||||
|
|
||||||
[Log]
|
[Log]
|
||||||
|
field_to_plot = Attribute.category
|
||||||
fieldname_order=["Event.id", "Attribute.Tag", "Attribute.category", "Attribute.type", ["Attribute.value", "Attribute.comment"]]
|
fieldname_order=["Event.id", "Attribute.Tag", "Attribute.category", "Attribute.type", ["Attribute.value", "Attribute.comment"]]
|
||||||
#fieldname_order=["id", "category", "type", ["value", "comment"]]
|
|
||||||
char_separator=||
|
char_separator=||
|
||||||
|
|
||||||
[RedisLog]
|
[RedisLog]
|
||||||
|
|
20
server.py
20
server.py
|
@ -91,11 +91,8 @@ class EventMessage():
|
||||||
to_ret = { 'log': self.feed, 'feedName': self.feedName, 'zmqName': self.zmqName }
|
to_ret = { 'log': self.feed, 'feedName': self.feedName, 'zmqName': self.zmqName }
|
||||||
return 'data: {}\n\n'.format(json.dumps(to_ret))
|
return 'data: {}\n\n'.format(json.dumps(to_ret))
|
||||||
|
|
||||||
def getZrange(keyCateg, dayNum, topNum):
|
def getZrange(keyCateg, date, topNum):
|
||||||
aDateTime = datetime.datetime.now()
|
date_str = str(date.year)+str(date.month)+str(date.day)
|
||||||
correctDatetime = aDateTime - datetime.timedelta(days = dayNum)
|
|
||||||
|
|
||||||
date_str = str(correctDatetime.year)+str(correctDatetime.month)+str(correctDatetime.day)
|
|
||||||
keyname = "{}:{}".format(keyCateg, date_str)
|
keyname = "{}:{}".format(keyCateg, date_str)
|
||||||
data = serv_redis_db.zrange(keyname, 0, 5, desc=True, withscores=True)
|
data = serv_redis_db.zrange(keyname, 0, 5, desc=True, withscores=True)
|
||||||
data = [ [record[0].decode('utf8'), record[1]] for record in data ]
|
data = [ [record[0].decode('utf8'), record[1]] for record in data ]
|
||||||
|
@ -114,6 +111,7 @@ def index():
|
||||||
return render_template('index.html',
|
return render_template('index.html',
|
||||||
pannelSize=pannelSize,
|
pannelSize=pannelSize,
|
||||||
size_dashboard_width=[cfg.getint('Dashboard' ,'size_dashboard_left_width'), 12-cfg.getint('Dashboard', 'size_dashboard_left_width')],
|
size_dashboard_width=[cfg.getint('Dashboard' ,'size_dashboard_left_width'), 12-cfg.getint('Dashboard', 'size_dashboard_left_width')],
|
||||||
|
itemToPlot=cfg.get('Dashboard', 'item_to_plot'),
|
||||||
graph_log_refresh_rate=cfg.getint('Dashboard' ,'graph_log_refresh_rate'),
|
graph_log_refresh_rate=cfg.getint('Dashboard' ,'graph_log_refresh_rate'),
|
||||||
char_separator=cfg.get('Log', 'char_separator'),
|
char_separator=cfg.get('Log', 'char_separator'),
|
||||||
rotation_wait_time=cfg.getint('Dashboard' ,'rotation_wait_time'),
|
rotation_wait_time=cfg.getint('Dashboard' ,'rotation_wait_time'),
|
||||||
|
@ -133,23 +131,23 @@ def geo():
|
||||||
@app.route("/_getTopCoord")
|
@app.route("/_getTopCoord")
|
||||||
def getTopCoord():
|
def getTopCoord():
|
||||||
try:
|
try:
|
||||||
dayNum = int(request.args.get('dayNum'))
|
date = datetime.datetime.fromtimestamp(float(request.args.get('date')))
|
||||||
except:
|
except:
|
||||||
dayNum = 0
|
date = datetime.datetime.now()
|
||||||
keyCateg = "GEO_COORD"
|
keyCateg = "GEO_COORD"
|
||||||
topNum = 6 # default Num
|
topNum = 6 # default Num
|
||||||
data = getZrange(keyCateg, dayNum, topNum)
|
data = getZrange(keyCateg, date, topNum)
|
||||||
return jsonify(data)
|
return jsonify(data)
|
||||||
|
|
||||||
@app.route("/_getHitMap")
|
@app.route("/_getHitMap")
|
||||||
def getHitMap():
|
def getHitMap():
|
||||||
try:
|
try:
|
||||||
dayNum = int(request.args.get('dayNum'))
|
date = datetime.datetime.fromtimestamp(float(request.args.get('date')))
|
||||||
except:
|
except:
|
||||||
dayNum = 0
|
date = datetime.datetime.now()
|
||||||
keyCateg = "GEO_COUNTRY"
|
keyCateg = "GEO_COUNTRY"
|
||||||
topNum = -1 # default Num
|
topNum = -1 # default Num
|
||||||
data = getZrange(keyCateg, dayNum, topNum)
|
data = getZrange(keyCateg, date, topNum)
|
||||||
return jsonify(data)
|
return jsonify(data)
|
||||||
|
|
||||||
def isCloseTo(coord1, coord2):
|
def isCloseTo(coord1, coord2):
|
||||||
|
|
|
@ -110,8 +110,15 @@ $(document).ready(function () {
|
||||||
|
|
||||||
/* TOP LOCATION */
|
/* TOP LOCATION */
|
||||||
|
|
||||||
function updateTopMaps(dayNum) {
|
function updateTopMaps(date) {
|
||||||
$.getJSON(urlTopCoord+"?dayNum="+dayNum, function(list){
|
$.getJSON(urlTopCoord+"?date="+date.getTime()/1000, function(list){
|
||||||
|
if (list.length==0) {
|
||||||
|
for(var i=0; i<6; i++) { // clear maps
|
||||||
|
allOpenStreetMap[i+1].setView([0, 0], 1);
|
||||||
|
savedMarker[i+1].remove(); // remove marker
|
||||||
|
}
|
||||||
|
savedMarker = {};
|
||||||
|
}
|
||||||
for(var i=0; i<6 && i<list.length; i++) {
|
for(var i=0; i<6 && i<list.length; i++) {
|
||||||
// create marker + flyToIt
|
// create marker + flyToIt
|
||||||
coordJson = JSON.parse(list[i][0]);
|
coordJson = JSON.parse(list[i][0]);
|
||||||
|
@ -121,7 +128,7 @@ function updateTopMaps(dayNum) {
|
||||||
var markerToUpdate = savedMarker[i+1];
|
var markerToUpdate = savedMarker[i+1];
|
||||||
if (markerToUpdate != undefined) {
|
if (markerToUpdate != undefined) {
|
||||||
markerToUpdate.setLatLng({lat: coordJson.lat, lng: coordJson.lon});
|
markerToUpdate.setLatLng({lat: coordJson.lat, lng: coordJson.lon});
|
||||||
markerToUpdate._popup._content = 'lat: '+coordJson.lat+', lon: '+coordJson.lon+' - '+list[i][1];
|
markerToUpdate._popup.setContent('lat: '+coordJson.lat+', lon: '+coordJson.lon+' (<strong>'+list[i][1]+'</strong>)');
|
||||||
markerToUpdate.update();
|
markerToUpdate.update();
|
||||||
} else { // create new marker
|
} else { // create new marker
|
||||||
var marker = L.marker([coordJson.lat, coordJson.lon]).addTo(allOpenStreetMap[i+1]);
|
var marker = L.marker([coordJson.lat, coordJson.lon]).addTo(allOpenStreetMap[i+1]);
|
||||||
|
@ -135,8 +142,10 @@ function updateTopMaps(dayNum) {
|
||||||
|
|
||||||
/* WORLD MAP */
|
/* WORLD MAP */
|
||||||
|
|
||||||
function updateWorldMap(dayNum) {
|
function updateWorldMap(date) {
|
||||||
$.getJSON(urlHitMap+"?dayNum="+dayNum, function(list){
|
$.getJSON(urlHitMap+"?date="+date.getTime()/1000, function(list){
|
||||||
|
regionhits = {};
|
||||||
|
worldMapObj.series.regions[0].clear();
|
||||||
for(var i=0; i<list.length; i++) {
|
for(var i=0; i<list.length; i++) {
|
||||||
var rCode = list[i][0];
|
var rCode = list[i][0];
|
||||||
var rNum = list[i][1];
|
var rNum = list[i][1];
|
||||||
|
@ -148,8 +157,7 @@ function updateWorldMap(dayNum) {
|
||||||
function update_region(regionCode, num) {
|
function update_region(regionCode, num) {
|
||||||
regionhits[regionCode] = num;
|
regionhits[regionCode] = num;
|
||||||
// Force recomputation of min and max for correct color scaling
|
// Force recomputation of min and max for correct color scaling
|
||||||
regionhitsMax = regionhitsMax >= regionhits[regionCode] ? regionhitsMax : regionhits[regionCode];
|
worldMapObj.series.regions[0].params.max = undefined;
|
||||||
worldMapObj.series.regions[0].params.max = regionhitsMax;
|
|
||||||
worldMapObj.series.regions[0].legend.render();
|
worldMapObj.series.regions[0].legend.render();
|
||||||
// Update data
|
// Update data
|
||||||
worldMapObj.series.regions[0].setValues(regionhits);
|
worldMapObj.series.regions[0].setValues(regionhits);
|
||||||
|
@ -210,6 +218,6 @@ function updateAll() {
|
||||||
var currentDate = datePickerWidget.datepicker( "getDate" );
|
var currentDate = datePickerWidget.datepicker( "getDate" );
|
||||||
var now = new Date();
|
var now = new Date();
|
||||||
var numDay = days_between(now, currentDate);
|
var numDay = days_between(now, currentDate);
|
||||||
updateTopMaps(numDay);
|
updateTopMaps(currentDate);
|
||||||
updateWorldMap(numDay);
|
updateWorldMap(currentDate);
|
||||||
}
|
}
|
||||||
|
|
|
@ -3,6 +3,7 @@ var maxNumPoint = hours_spanned;
|
||||||
var keepaliveTime = 0;
|
var keepaliveTime = 0;
|
||||||
var emptyArray = [];
|
var emptyArray = [];
|
||||||
var _timeoutLed;
|
var _timeoutLed;
|
||||||
|
var toPlotLocationLog;
|
||||||
for(i=0; i<maxNumPoint; i++) {
|
for(i=0; i<maxNumPoint; i++) {
|
||||||
emptyArray.push([i, 0]);
|
emptyArray.push([i, 0]);
|
||||||
}
|
}
|
||||||
|
@ -233,10 +234,11 @@ function updateLogTable(feedName, log, zmqName) {
|
||||||
|
|
||||||
// only add row for attribute
|
// only add row for attribute
|
||||||
if (feedName == "Attribute" ) {
|
if (feedName == "Attribute" ) {
|
||||||
var categName = log[2];
|
var categName = log[toPlotLocationLog];
|
||||||
sources.addIfNotPresent(categName);
|
sources.addIfNotPresent(categName);
|
||||||
sources.incCountOnSource(categName);
|
sources.incCountOnSource(categName);
|
||||||
sources.incCountOnSource('global');
|
sources.incCountOnSource('global');
|
||||||
|
updateChartDirect();
|
||||||
createRow(tableBody, log);
|
createRow(tableBody, log);
|
||||||
|
|
||||||
// Remove old row
|
// Remove old row
|
||||||
|
@ -320,14 +322,15 @@ function createHead(callback) {
|
||||||
return
|
return
|
||||||
$.getJSON( urlForHead, function( data ) {
|
$.getJSON( urlForHead, function( data ) {
|
||||||
var tr = document.createElement('TR');
|
var tr = document.createElement('TR');
|
||||||
for (head of data) {
|
for (i in data) {
|
||||||
|
var head = data[i];
|
||||||
var th = document.createElement('TH');
|
var th = document.createElement('TH');
|
||||||
|
if (head == itemToPlot) {
|
||||||
|
toPlotLocationLog = i;
|
||||||
|
}
|
||||||
th.appendChild(document.createTextNode(head));
|
th.appendChild(document.createTextNode(head));
|
||||||
tr.appendChild(th);
|
tr.appendChild(th);
|
||||||
}
|
}
|
||||||
//var action = document.createElement('TH');
|
|
||||||
//action.appendChild(document.createTextNode("Actions"));
|
|
||||||
//tr.appendChild(action);
|
|
||||||
document.getElementById('table_log_head').appendChild(tr);
|
document.getElementById('table_log_head').appendChild(tr);
|
||||||
callback();
|
callback();
|
||||||
});
|
});
|
||||||
|
|
|
@ -51,7 +51,7 @@ function updateChartDirect() {
|
||||||
plotLineChart.getOptions().yaxes[0].max = sources.getGlobalMax();
|
plotLineChart.getOptions().yaxes[0].max = sources.getGlobalMax();
|
||||||
plotLineChart.setupGrid();
|
plotLineChart.setupGrid();
|
||||||
plotLineChart.draw();
|
plotLineChart.draw();
|
||||||
setTimeout(updateChartDirect, updateIntervalDirect);
|
//setTimeout(updateChartDirect, updateIntervalDirect);
|
||||||
}
|
}
|
||||||
updateChartDirect()
|
updateChartDirect()
|
||||||
|
|
||||||
|
|
|
@ -157,7 +157,7 @@ small {
|
||||||
<!-- /.panel -->
|
<!-- /.panel -->
|
||||||
<div id="panelLineChart" class="panel panel-default" style="margin-top: 15px; height: {{ pannelSize[1] }}vh;">
|
<div id="panelLineChart" class="panel panel-default" style="margin-top: 15px; height: {{ pannelSize[1] }}vh;">
|
||||||
<div class="panel-heading">
|
<div class="panel-heading">
|
||||||
<i class="fa fa-bar-chart-o fa-fw"></i> Categories overtime (hours)
|
<i class="fa fa-bar-chart-o fa-fw"></i> <strong>{{ itemToPlot }}</strong> overtime (hours)
|
||||||
</div>
|
</div>
|
||||||
<div id="panelbody" class="panel-body" style="width:100%; height: 100%;">
|
<div id="panelbody" class="panel-body" style="width:100%; height: 100%;">
|
||||||
<div id="feedDiv3" style="width:100%; height: calc(100% - 46px); position: relative;"></div>
|
<div id="feedDiv3" style="width:100%; height: calc(100% - 46px); position: relative;"></div>
|
||||||
|
@ -241,6 +241,7 @@ small {
|
||||||
var linkForDefaultMap = "{{ url_for('static', filename='maps/default.png') }}";
|
var linkForDefaultMap = "{{ url_for('static', filename='maps/default.png') }}";
|
||||||
|
|
||||||
/* DATA FROM CONF */
|
/* DATA FROM CONF */
|
||||||
|
var itemToPlot = "{{ itemToPlot }}";
|
||||||
var graph_log_refresh_rate = {{ graph_log_refresh_rate }};
|
var graph_log_refresh_rate = {{ graph_log_refresh_rate }};
|
||||||
var char_separator = "{{ char_separator }}";
|
var char_separator = "{{ char_separator }}";
|
||||||
var rotation_wait_time = {{ rotation_wait_time }};
|
var rotation_wait_time = {{ rotation_wait_time }};
|
||||||
|
|
|
@ -1,6 +1,9 @@
|
||||||
#!/usr/bin/env python3.5
|
#!/usr/bin/env python3.5
|
||||||
|
|
||||||
import time, datetime
|
import time, datetime
|
||||||
|
import copy
|
||||||
|
from collections import OrderedDict
|
||||||
|
from pprint import pprint
|
||||||
import zmq
|
import zmq
|
||||||
import redis
|
import redis
|
||||||
import random
|
import random
|
||||||
|
@ -69,8 +72,11 @@ def getCoordAndPublish(zmq_name, supposed_ip, categ):
|
||||||
rep = ip_to_coord(supposed_ip)
|
rep = ip_to_coord(supposed_ip)
|
||||||
coord = rep['coord']
|
coord = rep['coord']
|
||||||
coord_dic = {'lat': coord['lat'], 'lon': coord['lon']}
|
coord_dic = {'lat': coord['lat'], 'lon': coord['lon']}
|
||||||
|
ordDic = OrderedDict()
|
||||||
|
ordDic['lat'] = coord_dic['lat']
|
||||||
|
ordDic['lon'] = coord_dic['lon']
|
||||||
coord_list = [coord['lat'], coord['lon']]
|
coord_list = [coord['lat'], coord['lon']]
|
||||||
push_to_redis_zset('GEO_COORD', json.dumps(coord_dic))
|
push_to_redis_zset('GEO_COORD', json.dumps(ordDic))
|
||||||
push_to_redis_zset('GEO_COUNTRY', rep['full_rep'].country.iso_code)
|
push_to_redis_zset('GEO_COUNTRY', rep['full_rep'].country.iso_code)
|
||||||
push_to_redis_geo('GEO_RAD', coord['lon'], coord['lat'], json.dumps({ 'categ': categ, 'value': supposed_ip }))
|
push_to_redis_geo('GEO_RAD', coord['lon'], coord['lat'], json.dumps({ 'categ': categ, 'value': supposed_ip }))
|
||||||
to_send = {
|
to_send = {
|
||||||
|
@ -85,6 +91,8 @@ def getCoordAndPublish(zmq_name, supposed_ip, categ):
|
||||||
serv_coord.publish(CHANNELDISP, json.dumps(to_send))
|
serv_coord.publish(CHANNELDISP, json.dumps(to_send))
|
||||||
except ValueError:
|
except ValueError:
|
||||||
print("can't resolve ip")
|
print("can't resolve ip")
|
||||||
|
except ip2.errors.AddressNotFoundError:
|
||||||
|
print("Address not in Database")
|
||||||
|
|
||||||
def getFields(obj, fields):
|
def getFields(obj, fields):
|
||||||
jsonWalker = fields.split('.')
|
jsonWalker = fields.split('.')
|
||||||
|
@ -110,6 +118,10 @@ def handler_log(zmq_name, jsonevent):
|
||||||
print('sending', 'log')
|
print('sending', 'log')
|
||||||
return
|
return
|
||||||
|
|
||||||
|
def handler_dispatcher(zmq_name, jsonObj):
|
||||||
|
if "Event" in jsonObj:
|
||||||
|
handler_event(zmq_name, jsonObj)
|
||||||
|
|
||||||
def handler_keepalive(zmq_name, jsonevent):
|
def handler_keepalive(zmq_name, jsonevent):
|
||||||
print('sending', 'keepalive')
|
print('sending', 'keepalive')
|
||||||
to_push = [ jsonevent['uptime'] ]
|
to_push = [ jsonevent['uptime'] ]
|
||||||
|
@ -119,15 +131,17 @@ def handler_sighting(zmq_name, jsonsight):
|
||||||
print('sending' ,'sighting')
|
print('sending' ,'sighting')
|
||||||
return
|
return
|
||||||
|
|
||||||
def handler_event(zmq_name, jsonevent):
|
def handler_event(zmq_name, jsonobj):
|
||||||
#fields: threat_level_id, id, info
|
#fields: threat_level_id, id, info
|
||||||
jsonevent = jsonevent['Event']
|
jsonevent = jsonobj['Event']
|
||||||
#redirect to handler_attribute
|
#redirect to handler_attribute
|
||||||
if 'Attribute' in jsonevent:
|
if 'Attribute' in jsonevent:
|
||||||
attributes = jsonevent['Attribute']
|
attributes = jsonevent['Attribute']
|
||||||
if type(attributes) is list:
|
if type(attributes) is list:
|
||||||
for attr in attributes:
|
for attr in attributes:
|
||||||
handler_attribute(zmq_name, attr)
|
jsoncopy = copy.deepcopy(jsonobj)
|
||||||
|
jsoncopy['Attribute'] = attr
|
||||||
|
handler_attribute(zmq_name, jsoncopy)
|
||||||
else:
|
else:
|
||||||
handler_attribute(zmq_name, attributes)
|
handler_attribute(zmq_name, attributes)
|
||||||
|
|
||||||
|
@ -176,7 +190,7 @@ def main(zmqName):
|
||||||
|
|
||||||
|
|
||||||
dico_action = {
|
dico_action = {
|
||||||
"misp_json": handler_log,
|
"misp_json": handler_dispatcher,
|
||||||
"misp_json_event": handler_event,
|
"misp_json_event": handler_event,
|
||||||
"misp_json_self": handler_keepalive,
|
"misp_json_self": handler_keepalive,
|
||||||
"misp_json_attribute": handler_attribute,
|
"misp_json_attribute": handler_attribute,
|
||||||
|
|
Loading…
Reference in New Issue