MISP
/
misp-dashboard
mirror of https://github.com/MISP/misp-dashboard
2
0
Fork 0
Code Issues Releases Wiki Activity

Added draft support of org overtime + Added clickable orgName + minor fix in live dashbaord + draft support of sighting in zmq_sub

pull/3/head
Sami Mokaddem 2017-11-09 10:32:39 +01:00
parent 10c59fc220
commit 97a9020716
6 changed files with 106 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
contributor_helper.py
View File

@ -336,12 +336,27 @@ class Contributor_helper:
return data2 return data2
def TEST_getTop5OvertimeFromRedis(self): def TEST_getTop5OvertimeFromRedis(self):
import time
now = time.time()
ONE_DAY = 60*60*24
data2 = [ data2 = [
{'label': 'CIRCL', 'data': [[0, 4], [1, 7], [2,14]]}, {'label': 'CIRCL', 'data': [[now, random.randint(1,50)], [now-ONE_DAY, random.randint(1,50)], [now-ONE_DAY*2, random.randint(1,50)], [now-ONE_DAY*3, random.randint(1,50)], [now-ONE_DAY*4, random.randint(1,50)]]},
{'label': 'CASES', 'data': [[0, 1], [1, 5], [2,2]]} {'label': 'CASES', 'data': [[now, random.randint(1,50)], [now-ONE_DAY, random.randint(1,50)], [now-ONE_DAY*2, random.randint(1,50)], [now-ONE_DAY*3, random.randint(1,50)], [now-ONE_DAY*4, random.randint(1,50)]]},
{'label': 'Org1', 'data': [[now, random.randint(1,50)], [now-ONE_DAY, random.randint(1,50)], [now-ONE_DAY*2, random.randint(1,50)], [now-ONE_DAY*3, random.randint(1,50)], [now-ONE_DAY*4, random.randint(1,50)]]},
{'label': 'Org2', 'data': [[now, random.randint(1,50)], [now-ONE_DAY, random.randint(1,50)], [now-ONE_DAY*2, random.randint(1,50)], [now-ONE_DAY*3, random.randint(1,50)], [now-ONE_DAY*4, random.randint(1,50)]]},
{'label': 'SMILE', 'data': [[now, random.randint(1,50)], [now-ONE_DAY, random.randint(1,50)], [now-ONE_DAY*2, random.randint(1,50)], [now-ONE_DAY*3, random.randint(1,50)], [now-ONE_DAY*4, random.randint(1,50)]]},
] ]
return data2 return data2
def TEST_getOrgOvertime(self, org):
import time
now = time.time()
ONE_DAY = 60*60*24
data = [
{'label': org, 'data': [[now, random.randint(1,30)], [now-ONE_DAY, random.randint(1,30)], [now-ONE_DAY*2, random.randint(1,30)], [now-ONE_DAY*3, random.randint(1,30)], [now-ONE_DAY*4, random.randint(1,40)]]}
]
return data
def TEST_getTopContributorFromRedis(self, date): def TEST_getTopContributorFromRedis(self, date):
data2 = [ data2 = [
{ {

8
server.py
View File

@ -331,6 +331,14 @@ def getFameContributor():
def getTop5Overtime(): def getTop5Overtime():
return jsonify(contributor_helper.TEST_getTop5OvertimeFromRedis()) return jsonify(contributor_helper.TEST_getTop5OvertimeFromRedis())
@app.route("/_getOrgOvertime")
def getOrgOvertime():
try:
org = request.args.get('org')
except:
org = ''
return jsonify(contributor_helper.TEST_getOrgOvertime(org))
@app.route("/_getCategPerContrib") @app.route("/_getCategPerContrib")
def getCategPerContrib(): def getCategPerContrib():
try: try:

66
static/js/contrib.js
View File

@ -5,6 +5,8 @@ var datatableFame;
var refresh_speed = min_between_reload*60; var refresh_speed = min_between_reload*60;
var will_reload = $("#reloadCheckbox").is(':checked'); var will_reload = $("#reloadCheckbox").is(':checked');
var sec_before_reload = refresh_speed; var sec_before_reload = refresh_speed;
var dataTop5Overtime;
var plotLineChart
/* CONFIG */ /* CONFIG */
var maxRank = 16; var maxRank = 16;
@ -24,6 +26,7 @@ var optionsLineChart = {
} }
} }
}, },
colors: ["#2F4F4F", "#778899", "#696969", "#A9A9A9", "#D3D3D3", "#337ab7"],
points: { show: true }, points: { show: true },
lines: { show: true, fill: true }, lines: { show: true, fill: true },
grid: { grid: {
@ -33,6 +36,11 @@ var optionsLineChart = {
legend: { legend: {
show: true, show: true,
position: "nw" position: "nw"
},
xaxis: {
mode: "time",
timeformat: "%m/%d",
minTickSize: [1, "day"]
} }
}; };
var optionDatatable_light = { var optionDatatable_light = {
@ -171,6 +179,13 @@ function createHonorImg(array, size) {
return div.outerHTML; return div.outerHTML;
} }
function createOrgLink(org) {
var a = document.createElement('a');
a.innerHTML = org;
a.href = "?org="+org;
return a.outerHTML;
}
function generateRankingSheet(rank, rankDec, stepPnt, pnt, Rpnt) { function generateRankingSheet(rank, rankDec, stepPnt, pnt, Rpnt) {
var Cpnt = pnt - stepPnt; var Cpnt = pnt - stepPnt;
var Tpnt = Cpnt + Rpnt; var Tpnt = Cpnt + Rpnt;
@ -267,7 +282,7 @@ function addToTableFromJson(datatable, url) {
getOrgRankIcon(row.orgRank, 60), getOrgRankIcon(row.orgRank, 60),
createHonorImg(row.honorBadge, 20), createHonorImg(row.honorBadge, 20),
createImg(row.logo_path, 32), createImg(row.logo_path, 32),
row.org createOrgLink(row.org)
]; ];
datatable.row.add(to_add); datatable.row.add(to_add);
} }
@ -293,7 +308,7 @@ function addLastContributor(datatable, data, update) {
getOrgRankIcon(data.orgRank, 60), getOrgRankIcon(data.orgRank, 60),
createHonorImg(data.honorBadge, 20), createHonorImg(data.honorBadge, 20),
createImg(data.logo_path, 32), createImg(data.logo_path, 32),
data.org, createOrgLink(data.org),
data.epoch data.epoch
]; ];
if (update == undefined || update == false) { if (update == undefined || update == false) {
@ -347,7 +362,6 @@ function updateProgressHeader(org) {
// colorize row contribution rank help // colorize row contribution rank help
$.getJSON( url_getContributionOrgStatus+'?org='+org, function( data ) { $.getJSON( url_getContributionOrgStatus+'?org='+org, function( data ) {
var status = data['status']; var status = data['status'];
console.log(data);
var curContributionOrgRank = data['rank']; var curContributionOrgRank = data['rank'];
var totNumPoints = data['totPoints'] var totNumPoints = data['totPoints']
$('#orgTotNumOfPoint').text(totNumPoints); $('#orgTotNumOfPoint').text(totNumPoints);
@ -386,23 +400,42 @@ function updateProgressHeader(org) {
}); });
//update overtake points //update overtake points
var prevOrg = ""; var prevOrgName = "";
var prevOrgPnts = 0; var prevOrgPnts = 0;
datatableTop.rows().every( function() { datatableTop.rows().every( function() {
var row = this.node(); var row = this.node();
if(this.data()[5] == currOrg) { var orgRowName = $(this.data()[5])[0].text; // contained in <a>
if(prevOrg == ""){ //already first var orgRowPnts = this.data()[0]
if(orgRowName == currOrg) {
if(prevOrgName == ""){ //already first
$('#orgToOverTake').text(this.data()[5]); $('#orgToOverTake').text(this.data()[5]);
$('#pntsToOvertakeNext').text(0); $('#pntsToOvertakeNext').text(0);
} else { } else {
$('#orgToOverTake').text(prevOrg); $('#orgToOverTake').text(prevOrgName);
$('#pntsToOvertakeNext').text(parseInt(prevOrgPnts)-this.data()[0]); $('#pntsToOvertakeNext').text(parseInt(prevOrgPnts)-orgRowPnts);
} }
} else { } else {
prevOrg = this.data()[5]; prevOrgName = orgRowName;
prevOrgPnts = this.data()[0]; prevOrgPnts = orgRowPnts;
} }
}); });
//Add new data to linechart
$.getJSON( url_getOrgOvertime+'?org='+org, function( data ) {
var toPlot = dataTop5Overtime.slice(0); //cloning data
// transform secs into date
for(i in data){
var new_data = [];
for(list of data[i]['data']) {
new_data.push([new Date(list[0]*1000), list[1]]);
}
data[i]['data'] = new_data;
toPlot.push(data[i]);
}
plotLineChart.setData(toPlot);
plotLineChart.setupGrid();
plotLineChart.draw();
});
} }
function showOnlyOrg() { function showOnlyOrg() {
@ -462,7 +495,7 @@ $(document).ready(function() {
getOrgRankIcon(row.orgRank, 44), getOrgRankIcon(row.orgRank, 44),
createHonorImg(row.honorBadge, 20), createHonorImg(row.honorBadge, 20),
createImg(row.logo_path, 32), createImg(row.logo_path, 32),
row.org, createOrgLink(row.org),
]; ];
for (categ of categ_list) { for (categ of categ_list) {
to_add.push(row[categ]); to_add.push(row[categ]);
@ -474,7 +507,16 @@ $(document).ready(function() {
}); });
// top 5 contrib overtime // top 5 contrib overtime
$.getJSON( url_getTop5Overtime, function( data ) { $.getJSON( url_getTop5Overtime, function( data ) {
var plotLineChart = $.plot("#divTop5Overtime", data, optionsLineChart); // transform secs into date
for(i in data){
var new_data = [];
for(list of data[i]['data']) {
new_data.push([new Date(list[0]*1000), list[1]]);
}
data[i]['data'] = new_data;
}
dataTop5Overtime = data;
plotLineChart = $.plot("#divTop5Overtime", data, optionsLineChart);
}); });
if(currOrg != "") // currOrg selected if(currOrg != "") // currOrg selected
//FIXME: timeout used to wait that all datatables are draw. //FIXME: timeout used to wait that all datatables are draw.

10
static/js/index/index_map.js
View File

@ -125,7 +125,7 @@ class MapEventManager {
// Add and Manage markers on the map + make Animation // Add and Manage markers on the map + make Animation
popupCoord(coord, regionCode) { popupCoord(coord, regionCode) {
var coord = [coord.lat, coord.lon]; var coord = [coord.lat, coord.lon];
var color = Math.random()*180; var color = 0.5*180;
var pnts = openStreetMapObj.latLngToPoint(coord[0], coord[1]) var pnts = openStreetMapObj.latLngToPoint(coord[0], coord[1])
if (pnts != false) { //sometimes latLngToPoint return false if (pnts != false) { //sometimes latLngToPoint return false
var addedMarker = openStreetMapObj.addMarker(this._curMarkerNum, coord, [color]); var addedMarker = openStreetMapObj.addMarker(this._curMarkerNum, coord, [color]);
@ -178,7 +178,8 @@ $(function(){
series: { series: {
markers: [{ markers: [{
attribute: 'fill', attribute: 'fill',
scale: ['#1A0DAB', '#e50000', '#62ff41'], //scale: ['#1A0DAB', '#e50000', '#62ff41'],
scale: ['#ffff66'],
values: [], values: [],
min: 0, min: 0,
max: 180 max: 180
@ -207,10 +208,10 @@ function connect_source_map() {
var marker = L.marker([json.coord.lat, json.coord.lon]).addTo(myOpenStreetMap); var marker = L.marker([json.coord.lat, json.coord.lon]).addTo(myOpenStreetMap);
var mapEvent = new MapEvent(json, marker); var mapEvent = new MapEvent(json, marker);
mapEventManager.addMapEvent(mapEvent); mapEventManager.addMapEvent(mapEvent);
}; };
source_map.onopen = function(){ source_map.onopen = function(){
console.log('connection is opened. '+source_map.readyState); console.log('connection is opened. '+source_map.readyState);
}; };
source_map.onerror = function(){ source_map.onerror = function(){
console.log('error: '+source_map.readyState); console.log('error: '+source_map.readyState);
@ -240,4 +241,3 @@ $(document).ready(function () {
mapEventManager.directZoom(); mapEventManager.directZoom();
}); });
}); });

8
templates/contrib.html
View File

@ -24,6 +24,7 @@
<script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script> <script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.pie.min.js') }}"></script> <script src="{{ url_for('static', filename='js/jquery.flot.pie.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.resize.js') }}"></script> <script src="{{ url_for('static', filename='js/jquery.flot.resize.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
<!-- Bootstrap Core JavaScript --> <!-- Bootstrap Core JavaScript -->
<script src="{{ url_for('static', filename='js/bootstrap.js') }}"></script> <script src="{{ url_for('static', filename='js/bootstrap.js') }}"></script>
<script src="{{ url_for('static', filename='js/bootstrap3-typeahead.min.js') }}"></script> <script src="{{ url_for('static', filename='js/bootstrap3-typeahead.min.js') }}"></script>
@ -225,7 +226,7 @@
<div class="col-lg-6"> <div class="col-lg-6">
<div class="panel panel-default" style="height: 100%;"> <div class="panel panel-default" style="height: 100%;">
<div class="panel-heading bg-info" style="font-weight: bold;"> <div class="panel-heading bg-info" style="font-weight: bold;">
<i class="fa fa-asterisk " style="margin-right: 5px; color: #f0ad4e;"></i><b>Contributor Ranking</b> <i class="fa fa-asterisk " style="margin-right: 5px; color: #f0ad4e;"></i><b>Contributor ranking (monthly)</b>
</div> </div>
<div id="panelRanking" class="panel-body" style="height: 100%;"> <div id="panelRanking" class="panel-body" style="height: 100%;">
@ -283,7 +284,7 @@
<div class="col-lg-12"> <div class="col-lg-12">
<div class="panel panel-default" style="height: 100%;"> <div class="panel panel-default" style="height: 100%;">
<div class="panel-heading bg-info" style="font-weight: bold;"> <div class="panel-heading bg-info" style="font-weight: bold;">
<i class="fa fa-th-list " style="margin-right: 5px; color: #f0ad4e;"></i><b>Contributors and Categories</b> <i class="fa fa-th-list " style="margin-right: 5px; color: #f0ad4e;"></i><b>Contributors and Categories (total)</b>
</div> </div>
<div id="panelRanking" class="panel-body" style="height: 100%;"> <div id="panelRanking" class="panel-body" style="height: 100%;">
@ -320,7 +321,7 @@
<div class="panel panel-default" style="height: 100%;"> <div class="panel panel-default" style="height: 100%;">
<div class="panel-heading bg-info" style="font-weight: bold;"> <div class="panel-heading bg-info" style="font-weight: bold;">
<i class="fa fa-star" style="margin-right: 5px; color: #f0ad4e;"></i><b>Hall Of Fame</b> <i class="fa fa-star" style="margin-right: 5px; color: #f0ad4e;"></i><b>Hall Of Fame (previous month)</b>
</div> </div>
<div id="panelRanking" class="panel-body" style="height: 100%;"> <div id="panelRanking" class="panel-body" style="height: 100%;">
@ -375,6 +376,7 @@
var url_getFameContributor = "{{ url_for('getFameContributor') }}"; var url_getFameContributor = "{{ url_for('getFameContributor') }}";
var url_getCategPerContrib = "{{ url_for('getCategPerContrib') }}"; var url_getCategPerContrib = "{{ url_for('getCategPerContrib') }}";
var url_getTop5Overtime = "{{ url_for('getTop5Overtime') }}"; var url_getTop5Overtime = "{{ url_for('getTop5Overtime') }}";
var url_getOrgOvertime = "{{ url_for('getOrgOvertime') }}";
var url_getLastContributor = "{{ url_for('getLastContributors') }}"; var url_getLastContributor = "{{ url_for('getLastContributors') }}";
var url_eventStreamLastContributor = "{{ url_for('getLastContributor') }}"; var url_eventStreamLastContributor = "{{ url_for('getLastContributor') }}";
var url_getAllOrg = "{{ url_for('getAllOrg') }}"; var url_getAllOrg = "{{ url_for('getAllOrg') }}";

28
zmq_subscriber.py
View File

@ -86,13 +86,16 @@ def getCoordAndPublish(zmq_name, supposed_ip, categ):
rep = ip_to_coord(supposed_ip) rep = ip_to_coord(supposed_ip)
coord = rep['coord'] coord = rep['coord']
coord_dic = {'lat': coord['lat'], 'lon': coord['lon']} coord_dic = {'lat': coord['lat'], 'lon': coord['lon']}
ordDic = OrderedDict() ordDic = OrderedDict() #keep fields with the same layout in redis
ordDic['lat'] = coord_dic['lat'] ordDic['lat'] = coord_dic['lat']
ordDic['lon'] = coord_dic['lon'] ordDic['lon'] = coord_dic['lon']
coord_list = [coord['lat'], coord['lon']] coord_list = [coord['lat'], coord['lon']]
push_to_redis_zset('GEO_COORD', json.dumps(ordDic)) push_to_redis_zset('GEO_COORD', json.dumps(ordDic))
push_to_redis_zset('GEO_COUNTRY', rep['full_rep'].country.iso_code) push_to_redis_zset('GEO_COUNTRY', rep['full_rep'].country.iso_code)
push_to_redis_geo('GEO_RAD', coord['lon'], coord['lat'], json.dumps({ 'categ': categ, 'value': supposed_ip })) ordDic = OrderedDict() #keep fields with the same layout in redis
ordDic['categ'] = categ
ordDic['value'] = supposed_ip
push_to_redis_geo('GEO_RAD', coord['lon'], coord['lat'], json.dumps(ordDic))
to_send = { to_send = {
"coord": coord, "coord": coord,
"categ": categ, "categ": categ,
@ -170,13 +173,14 @@ def handler_keepalive(zmq_name, jsonevent):
to_push = [ jsonevent['uptime'] ] to_push = [ jsonevent['uptime'] ]
publish_log(zmq_name, 'Keepalive', to_push) publish_log(zmq_name, 'Keepalive', to_push)
def handler_sighting(zmq_name, jsonsight): def handler_sighting(zmq_name, jsondata):
print('sending' ,'sighting') print('sending' ,'sighting')
org = jsonsight['org'] jsonsight = jsondata['Sighting']
categ = jsonsight['categ'] org = jsonsight['Event']['Orgc']['name']
action = jsonsight['action'] categ = jsonsight['Attribute']['category']
action = jsondata['action']
handleContribution(zmq_name, org, categ, action, pntMultiplier=2) handleContribution(zmq_name, org, categ, action, pntMultiplier=2)
return handler_attribute(zmq_name, jsonsight, hasAlreadyBeenContributed=True)
def handler_event(zmq_name, jsonobj): def handler_event(zmq_name, jsonobj):
#fields: threat_level_id, id, info #fields: threat_level_id, id, info
@ -192,7 +196,7 @@ def handler_event(zmq_name, jsonobj):
else: else:
handler_attribute(zmq_name, attributes) handler_attribute(zmq_name, attributes)
def handler_attribute(zmq_name, jsonobj): def handler_attribute(zmq_name, jsonobj, hasAlreadyBeenContributed=False):
# check if jsonattr is an attribute object # check if jsonattr is an attribute object
if 'Attribute' in jsonobj: if 'Attribute' in jsonobj:
jsonattr = jsonobj['Attribute'] jsonattr = jsonobj['Attribute']
@ -212,9 +216,10 @@ def handler_attribute(zmq_name, jsonobj):
if jsonattr['category'] == "Network activity": if jsonattr['category'] == "Network activity":
getCoordAndPublish(zmq_name, jsonattr['value'], jsonattr['category']) getCoordAndPublish(zmq_name, jsonattr['value'], jsonattr['category'])
eventLabeled = False if not hasAlreadyBeenContributed:
#eventLabeled = len(jsonattr[]) > 0 eventLabeled = False
handleContribution(zmq_name, jsonobj['Event']['Orgc']['name'], jsonattr['category'], jsonobj['action'], isLabeled=eventLabeled) #eventLabeled = len(jsonattr[]) > 0
handleContribution(zmq_name, jsonobj['Event']['Orgc']['name'], jsonattr['category'], jsonobj['action'], isLabeled=eventLabeled)
# Push to log # Push to log
publish_log(zmq_name, 'Attribute', to_push) publish_log(zmq_name, 'Attribute', to_push)
@ -227,6 +232,7 @@ def process_log(zmq_name, event):
event = event.decode('utf8') event = event.decode('utf8')
topic, eventdata = event.split(' ', maxsplit=1) topic, eventdata = event.split(' ', maxsplit=1)
jsonevent = json.loads(eventdata) jsonevent = json.loads(eventdata)
print(event)
dico_action[topic](zmq_name, jsonevent) dico_action[topic](zmq_name, jsonevent)