MISP
/
misp-dashboard
mirror of https://github.com/MISP/misp-dashboard
2
0
Fork 0
Code Issues Releases Wiki Activity

Started support of MISP ZMQ

pull/18/head
Sami Mokaddem 2017-10-13 15:03:09 +02:00
parent aec6fa2603
commit 7f9ded5262
6 changed files with 190 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
config.cfg
View File

@ -3,7 +3,7 @@
graph_log_refresh_rate = 1
[Log]
fieldname_order=["time", "level", "name", "message"]
fieldname_order=["time", "id", "category", "type", "value"]
[RedisLog]
host=localhost

2
retreive_map_pic.py
View File

@ -70,7 +70,7 @@ def download_maps(coord):
urls = create_box_around_coord(lon, lat)
map_name = download_and_merge(urls, coord)
print(map_name)
serv.publish(channel_disp, map_name)
serv.publish(channel_disp, json.dumps({ "path": map_name, "coord": coord }))
if __name__ == '__main__':
main()

4
server.py
View File

@ -100,13 +100,13 @@ def getLogHead():
def event_stream_log():
for msg in subscriber_log.listen():
content = msg['data']
print('sending', content)
yield EventMessage(content).to_json()
def event_stream_maps():
for msg in subscriber_map.listen():
content = msg['data'].decode('utf8')
yield 'data: {}\n\n'.format(json.dumps({ 'path': content }))
print('sending map', content)
yield 'data: {}\n\n'.format(content)
if __name__ == '__main__':
app.run(host='localhost', port=8000, threaded=True)

52
static/js/index.js
View File

@ -166,6 +166,7 @@ $(document).ready(function () {
var source_map = new EventSource(urlForMaps);
source_map.onmessage = function(event) {
var json = jQuery.parseJSON( event.data );
popupCoord(json.coord);
var img2 = linkForDefaultMap.replace(/\/[^\/]+$/, "/"+json.path);
$("#img"+img_to_change).fadeOut(400, function(){ $(this).attr('src', img2); }).fadeIn(400);
$("#textMap"+img_to_change).fadeOut(400, function(){ $(this).text(json.path); }).fadeIn(400);
@ -219,20 +220,27 @@ function updateLogTable(feedName, log) {
sources.addIfNotPresent(feedName);
sources.incCountOnSource(feedName);
sources.incCountOnSource('global');
createRow(tableBody, log);
// Remove old row
var logSel = document.getElementById("log_select");
//get height of pannel, find max num of item
var maxNumLogItem = document.getElementById('divLogTable').clientHeight/37;
maxNumLogItem -= 2; //take heading/padding/... into account
if (maxNumLogItem - parseInt(maxNumLogItem) < 0.5) { //beautifier
maxNumLogItem -= 1;
}
if (tableBody.rows.length > maxNumLogItem) {
while (tableBody.rows.length >= maxNumLogItem){
tableBody.deleteRow(0);
// only add row for attribute
if (feedName == "Attribute" ) {
createRow(tableBody, log);
// Remove old row
var logSel = document.getElementById("log_select");
//get height of pannel, find max num of item
var maxNumLogItem = document.getElementById('divLogTable').clientHeight/37;
maxNumLogItem -= 2; //take heading/padding/... into account
if (maxNumLogItem - parseInt(maxNumLogItem) < 0.5) { //beautifier
maxNumLogItem -= 1;
}
if (tableBody.rows.length > maxNumLogItem) {
while (tableBody.rows.length >= maxNumLogItem){
tableBody.deleteRow(0);
}
}
} else {
// do nothing
return;
}
}
@ -252,10 +260,7 @@ function slideAndMax(orig, newData) {
function createRow(tableBody, log) {
var tr = document.createElement('TR');
var action = document.createElement('TD');
var x = log[1];
var y = log[2];
popupCoord([x,y]);
//var action = document.createElement('TD');
for (var key in log) {
if (log.hasOwnProperty(key)) {
@ -277,8 +282,8 @@ function createRow(tableBody, log) {
}
// action
action.appendChild(document.createTextNode("ACTION"));
tr.appendChild(action);
//action.appendChild(document.createTextNode("ACTION"));
//tr.appendChild(action);
tableBody.appendChild(tr);
@ -294,9 +299,9 @@ function createHead(callback) {
th.appendChild(document.createTextNode(head));
tr.appendChild(th);
}
var action = document.createElement('TH');
action.appendChild(document.createTextNode("Actions"));
tr.appendChild(action);
//var action = document.createElement('TH');
//action.appendChild(document.createTextNode("Actions"));
//tr.appendChild(action);
document.getElementById('table_log_head').appendChild(tr);
callback();
});
@ -320,7 +325,8 @@ function marker_animation(x, y, curNumMarker) {
}
function popupCoord(coord) {
var value = coord[0]+coord[1];
var coord = [coord['lat'], coord['lon']];
var value = Math.random()*180;
pnts = mapObj.latLngToPoint(coord[0], coord[1])
if (pnts != false) { //sometimes latLngToPoint return false
mapObj.addMarker(curNumMarker, coord, [value]);
@ -345,7 +351,7 @@ $(function(){
attribute: 'fill',
scale: ['#1A0DAB', '#e50000', '#62ff41'],
values: [],
min: -180,
min: 0,
max: 180
}],
},

53
templates/index.html
View File

@ -97,6 +97,33 @@ small {
</div>
<!-- /.panel-body -->
</div>
<!-- /.panel -->
<div class="panel panel-default" style="margin-top: 15px; height: 29vh;">
<div class="panel-heading">
<i class="fa fa-bar-chart-o fa-fw"></i> Log feed
</div>
<div id="panelbody" class="panel-body" style="width:100%; height: calc(100% - 30px);">
<div id="feedDiv3" style="width:100%; height: calc(100% - 30px); position: relative;"></div>
</div>
</div>
</div>
<!-- /.col-lg-6 -->
<!-- /.col-lg-6 -->
<!-- vh sum = 87 -->
<div class="col-lg-7">
<div class="panel panel-default" style="margin-top: 15px; height: 59vh;">
<!--<div class="panel-heading">
<i class="fa fa-bar-chart-o fa-fw"></i> Feed
</div>-->
<div id="panelbody" class="panel-body" style="height: 100%;">
<div id="feedDiv2" style="width:100%; height: calc(100% - 30px); position: relative;"></div>
</div>
<!-- /.panel-body -->
</div>
<!-- /.panel -->
@ -122,32 +149,6 @@ small {
</div>
</div>
</div>
</div>
<!-- /.col-lg-6 -->
<!-- /.col-lg-6 -->
<!-- vh sum = 87 -->
<div class="col-lg-7">
<div class="panel panel-default" style="margin-top: 15px; height: 59vh;">
<!--<div class="panel-heading">
<i class="fa fa-bar-chart-o fa-fw"></i> Feed
</div>-->
<div id="panelbody" class="panel-body" style="height: 100%;">
<div id="feedDiv2" style="width:100%; height: calc(100% - 30px); position: relative;"></div>
</div>
<!-- /.panel-body -->
</div>
<!-- /.panel -->
<div class="panel panel-default" style="margin-top: 15px; height: 29vh;">
<div class="panel-heading">
<i class="fa fa-bar-chart-o fa-fw"></i> Log feed
</div>
<div id="panelbody" class="panel-body" style="width:100%; height: calc(100% - 30px);">
<div id="feedDiv3" style="width:100%; height: calc(100% - 30px); position: relative;"></div>
</div>
</div>
</div>
<!-- /.col-lg-6 -->

162
zmq_subscriber.py
View File

@ -8,18 +8,20 @@ import configparser
import os
import sys
import json
import geoip2.database
configfile = os.path.join(os.environ['VIRTUAL_ENV'], '../config.cfg')
cfg = configparser.ConfigParser()
cfg.read(configfile)
zmq_url = cfg.get('RedisLog', 'zmq_url')
zmq_url = "tcp://crf.circl.lu:5556"
zmq_url = "tcp://192.168.56.50:50000"
zmq_url = "tcp://localhost:9990"
channel = cfg.get('RedisLog', 'channel')
context = zmq.Context()
socket = context.socket(zmq.SUB)
socket.connect(zmq_url)
socket.setsockopt_string(zmq.SUBSCRIBE, channel)
socket.setsockopt_string(zmq.SUBSCRIBE, '')
redis_server = redis.StrictRedis(
host=cfg.get('RedisLog', 'host'),
@ -29,42 +31,138 @@ serv_coord = redis.StrictRedis(
host='localhost',
port=6250,
db=1)
path_to_db = "/home/sami/Downloads/GeoLite2-City_20171003/GeoLite2-City.mmdb"
reader = geoip2.database.Reader(path_to_db)
channel_proc = "CoordToProcess"
channel_disp = "PicToDisplay"
def publish_coord(coord):
pass
def get_ip(data):
pass
def ip_to_coord(ip):
resp = reader.city(ip)
lat = resp.location.latitude
lon = resp.location.longitude
return {'lat': lat, 'lon': lon}
def default_log(jsonevent):
print('sending', 'log')
return
#redis_server.publish(channel, json.dumps(jsonevent))
def default_keepalive(jsonevent):
print('sending', 'keepalive')
to_push = [ jsonevent['uptime'] ]
to_send = { 'name': 'Keepalive', 'log': json.dumps(to_push) }
redis_server.publish(channel, json.dumps(to_send))
def default_event(jsonevent):
print('sending', 'event')
jsonevent = jsonevent['Event']
to_push = [
jsonevent['threat_level_id'],
jsonevent['id'],
jsonevent['info'],
]
to_send = { 'name': 'Event', 'log': json.dumps(to_push) }
redis_server.publish(channel, json.dumps(to_send))
def default_attribute(jsonevent):
print('sending', 'attribute')
jsonevent = jsonevent['Attribute']
to_push = [
jsonevent['id'],
jsonevent['category'],
jsonevent['type'],
jsonevent['value'],
]
#try to get coord
if jsonevent['category'] == "Network activity":
try:
coord = ip_to_coord(jsonevent['value'])
to_send = {'lat': float(coord['lat']), 'lon': float(coord['lon'])}
serv_coord.publish(channel_proc, json.dumps(to_send))
print('coord sent')
except ValueError:
print("can't resolve ip")
to_send = { 'name': 'Attribute', 'log': json.dumps(to_push) }
redis_server.publish(channel, json.dumps(to_send))
def process_log(event):
event = event.decode('utf8')
topic, eventdata = event.split(' ', maxsplit=1)
jsonevent = json.loads(eventdata)
dico_action[topic](jsonevent)
def main():
while True:
content = socket.recv()
content.replace(b'\n', b'') # remove \n...
process_log(content)
def log_feed():
with open('misp-zmq.2', 'ba') as f:
while True:
time.sleep(1.0)
content = socket.recv()
content.replace(b'\n', b'') # remove \n...
f.write(content)
f.write(b'\n')
print(content)
#redis_server.publish(channel, content)
#if random.randint(1,10)<5:
# time.sleep(0.5)
# redis_server.publish(channel, content)
#if random.randint(1,10)<5:
# time.sleep(0.5)
# redis_server.publish(channel, content)
dico_action = {
"misp_json": default_event,
"misp_json_self": default_keepalive,
"misp_json_attribute": default_attribute,
"misp_json_sighting": default_log,
"misp_json_organisation": default_log,
"misp_json_user": default_log,
"misp_json_conversation": default_log
}
if __name__ == "__main__":
main()
reader.close()
# server side
pubsub = redis_server.pubsub(ignore_subscribe_messages=True)
while True:
rdm = random.randint(1,10)
time.sleep(float(rdm))
#lux
lon = random.uniform(5.7373, 6.4823)
lat = random.uniform(49.4061,49.7449)
#central eur
lon = random.uniform(3.936, 9.890)
lat = random.uniform(47.957, 50.999)
content = ["rdm "+str(rdm)]
content = [lat,lon]
jsonContent = json.dumps(content)
to_send = { 'name': 'feeder'+str(rdm), 'log': jsonContent }
redis_server.publish(channel, json.dumps(to_send))
serv_coord.publish(channel_proc, json.dumps({'lat': float(lat), 'lon': float(lon)}))
#while True:
# rdm = random.randint(1,10)
# time.sleep(float(rdm))
# #lux
# lon = random.uniform(5.7373, 6.4823)
# lat = random.uniform(49.4061,49.7449)
# #central eur
# lon = random.uniform(3.936, 9.890)
# lat = random.uniform(47.957, 50.999)
# content = ["rdm "+str(rdm)]
# content = [lat,lon]
# jsonContent = json.dumps(content)
# to_send = { 'name': 'feeder'+str(rdm), 'log': jsonContent }
# redis_server.publish(channel, json.dumps(to_send))
# serv_coord.publish(channel_proc, json.dumps({'lat': float(lat), 'lon': float(lon)}))
while True:
#FIXME check if sock.recv is blocking
time.sleep(0.1)
content = socket.recv()
print('sending', (content))
redis_server.publish(channel, content)
if random.randint(1,10)<5:
time.sleep(0.5)
redis_server.publish(channel, content)
if random.randint(1,10)<5:
time.sleep(0.5)
redis_server.publish(channel, content)