Started support of MISP ZMQ

pull/18/head
Sami Mokaddem 2017-10-13 15:03:09 +02:00
parent aec6fa2603
commit 7f9ded5262
6 changed files with 190 additions and 85 deletions

View File

@ -3,7 +3,7 @@
graph_log_refresh_rate = 1 graph_log_refresh_rate = 1
[Log] [Log]
fieldname_order=["time", "level", "name", "message"] fieldname_order=["time", "id", "category", "type", "value"]
[RedisLog] [RedisLog]
host=localhost host=localhost

View File

@ -70,7 +70,7 @@ def download_maps(coord):
urls = create_box_around_coord(lon, lat) urls = create_box_around_coord(lon, lat)
map_name = download_and_merge(urls, coord) map_name = download_and_merge(urls, coord)
print(map_name) print(map_name)
serv.publish(channel_disp, map_name) serv.publish(channel_disp, json.dumps({ "path": map_name, "coord": coord }))
if __name__ == '__main__': if __name__ == '__main__':
main() main()

View File

@ -100,13 +100,13 @@ def getLogHead():
def event_stream_log(): def event_stream_log():
for msg in subscriber_log.listen(): for msg in subscriber_log.listen():
content = msg['data'] content = msg['data']
print('sending', content)
yield EventMessage(content).to_json() yield EventMessage(content).to_json()
def event_stream_maps(): def event_stream_maps():
for msg in subscriber_map.listen(): for msg in subscriber_map.listen():
content = msg['data'].decode('utf8') content = msg['data'].decode('utf8')
yield 'data: {}\n\n'.format(json.dumps({ 'path': content })) print('sending map', content)
yield 'data: {}\n\n'.format(content)
if __name__ == '__main__': if __name__ == '__main__':
app.run(host='localhost', port=8000, threaded=True) app.run(host='localhost', port=8000, threaded=True)

View File

@ -166,6 +166,7 @@ $(document).ready(function () {
var source_map = new EventSource(urlForMaps); var source_map = new EventSource(urlForMaps);
source_map.onmessage = function(event) { source_map.onmessage = function(event) {
var json = jQuery.parseJSON( event.data ); var json = jQuery.parseJSON( event.data );
popupCoord(json.coord);
var img2 = linkForDefaultMap.replace(/\/[^\/]+$/, "/"+json.path); var img2 = linkForDefaultMap.replace(/\/[^\/]+$/, "/"+json.path);
$("#img"+img_to_change).fadeOut(400, function(){ $(this).attr('src', img2); }).fadeIn(400); $("#img"+img_to_change).fadeOut(400, function(){ $(this).attr('src', img2); }).fadeIn(400);
$("#textMap"+img_to_change).fadeOut(400, function(){ $(this).text(json.path); }).fadeIn(400); $("#textMap"+img_to_change).fadeOut(400, function(){ $(this).text(json.path); }).fadeIn(400);
@ -219,20 +220,27 @@ function updateLogTable(feedName, log) {
sources.addIfNotPresent(feedName); sources.addIfNotPresent(feedName);
sources.incCountOnSource(feedName); sources.incCountOnSource(feedName);
sources.incCountOnSource('global'); sources.incCountOnSource('global');
createRow(tableBody, log);
// Remove old row // only add row for attribute
var logSel = document.getElementById("log_select"); if (feedName == "Attribute" ) {
//get height of pannel, find max num of item createRow(tableBody, log);
var maxNumLogItem = document.getElementById('divLogTable').clientHeight/37;
maxNumLogItem -= 2; //take heading/padding/... into account // Remove old row
if (maxNumLogItem - parseInt(maxNumLogItem) < 0.5) { //beautifier var logSel = document.getElementById("log_select");
maxNumLogItem -= 1; //get height of pannel, find max num of item
} var maxNumLogItem = document.getElementById('divLogTable').clientHeight/37;
if (tableBody.rows.length > maxNumLogItem) { maxNumLogItem -= 2; //take heading/padding/... into account
while (tableBody.rows.length >= maxNumLogItem){ if (maxNumLogItem - parseInt(maxNumLogItem) < 0.5) { //beautifier
tableBody.deleteRow(0); maxNumLogItem -= 1;
} }
if (tableBody.rows.length > maxNumLogItem) {
while (tableBody.rows.length >= maxNumLogItem){
tableBody.deleteRow(0);
}
}
} else {
// do nothing
return;
} }
} }
@ -252,10 +260,7 @@ function slideAndMax(orig, newData) {
function createRow(tableBody, log) { function createRow(tableBody, log) {
var tr = document.createElement('TR'); var tr = document.createElement('TR');
var action = document.createElement('TD'); //var action = document.createElement('TD');
var x = log[1];
var y = log[2];
popupCoord([x,y]);
for (var key in log) { for (var key in log) {
if (log.hasOwnProperty(key)) { if (log.hasOwnProperty(key)) {
@ -277,8 +282,8 @@ function createRow(tableBody, log) {
} }
// action // action
action.appendChild(document.createTextNode("ACTION")); //action.appendChild(document.createTextNode("ACTION"));
tr.appendChild(action); //tr.appendChild(action);
tableBody.appendChild(tr); tableBody.appendChild(tr);
@ -294,9 +299,9 @@ function createHead(callback) {
th.appendChild(document.createTextNode(head)); th.appendChild(document.createTextNode(head));
tr.appendChild(th); tr.appendChild(th);
} }
var action = document.createElement('TH'); //var action = document.createElement('TH');
action.appendChild(document.createTextNode("Actions")); //action.appendChild(document.createTextNode("Actions"));
tr.appendChild(action); //tr.appendChild(action);
document.getElementById('table_log_head').appendChild(tr); document.getElementById('table_log_head').appendChild(tr);
callback(); callback();
}); });
@ -320,7 +325,8 @@ function marker_animation(x, y, curNumMarker) {
} }
function popupCoord(coord) { function popupCoord(coord) {
var value = coord[0]+coord[1]; var coord = [coord['lat'], coord['lon']];
var value = Math.random()*180;
pnts = mapObj.latLngToPoint(coord[0], coord[1]) pnts = mapObj.latLngToPoint(coord[0], coord[1])
if (pnts != false) { //sometimes latLngToPoint return false if (pnts != false) { //sometimes latLngToPoint return false
mapObj.addMarker(curNumMarker, coord, [value]); mapObj.addMarker(curNumMarker, coord, [value]);
@ -345,7 +351,7 @@ $(function(){
attribute: 'fill', attribute: 'fill',
scale: ['#1A0DAB', '#e50000', '#62ff41'], scale: ['#1A0DAB', '#e50000', '#62ff41'],
values: [], values: [],
min: -180, min: 0,
max: 180 max: 180
}], }],
}, },

View File

@ -97,6 +97,33 @@ small {
</div> </div>
<!-- /.panel-body --> <!-- /.panel-body -->
</div>
<!-- /.panel -->
<div class="panel panel-default" style="margin-top: 15px; height: 29vh;">
<div class="panel-heading">
<i class="fa fa-bar-chart-o fa-fw"></i> Log feed
</div>
<div id="panelbody" class="panel-body" style="width:100%; height: calc(100% - 30px);">
<div id="feedDiv3" style="width:100%; height: calc(100% - 30px); position: relative;"></div>
</div>
</div>
</div>
<!-- /.col-lg-6 -->
<!-- /.col-lg-6 -->
<!-- vh sum = 87 -->
<div class="col-lg-7">
<div class="panel panel-default" style="margin-top: 15px; height: 59vh;">
<!--<div class="panel-heading">
<i class="fa fa-bar-chart-o fa-fw"></i> Feed
</div>-->
<div id="panelbody" class="panel-body" style="height: 100%;">
<div id="feedDiv2" style="width:100%; height: calc(100% - 30px); position: relative;"></div>
</div>
<!-- /.panel-body -->
</div> </div>
<!-- /.panel --> <!-- /.panel -->
@ -122,32 +149,6 @@ small {
</div> </div>
</div> </div>
</div> </div>
</div>
<!-- /.col-lg-6 -->
<!-- /.col-lg-6 -->
<!-- vh sum = 87 -->
<div class="col-lg-7">
<div class="panel panel-default" style="margin-top: 15px; height: 59vh;">
<!--<div class="panel-heading">
<i class="fa fa-bar-chart-o fa-fw"></i> Feed
</div>-->
<div id="panelbody" class="panel-body" style="height: 100%;">
<div id="feedDiv2" style="width:100%; height: calc(100% - 30px); position: relative;"></div>
</div>
<!-- /.panel-body -->
</div>
<!-- /.panel -->
<div class="panel panel-default" style="margin-top: 15px; height: 29vh;">
<div class="panel-heading">
<i class="fa fa-bar-chart-o fa-fw"></i> Log feed
</div>
<div id="panelbody" class="panel-body" style="width:100%; height: calc(100% - 30px);">
<div id="feedDiv3" style="width:100%; height: calc(100% - 30px); position: relative;"></div>
</div>
</div>
</div> </div>
<!-- /.col-lg-6 --> <!-- /.col-lg-6 -->

View File

@ -8,18 +8,20 @@ import configparser
import os import os
import sys import sys
import json import json
import geoip2.database
configfile = os.path.join(os.environ['VIRTUAL_ENV'], '../config.cfg') configfile = os.path.join(os.environ['VIRTUAL_ENV'], '../config.cfg')
cfg = configparser.ConfigParser() cfg = configparser.ConfigParser()
cfg.read(configfile) cfg.read(configfile)
zmq_url = cfg.get('RedisLog', 'zmq_url') zmq_url = cfg.get('RedisLog', 'zmq_url')
zmq_url = "tcp://crf.circl.lu:5556" zmq_url = "tcp://192.168.56.50:50000"
zmq_url = "tcp://localhost:9990"
channel = cfg.get('RedisLog', 'channel') channel = cfg.get('RedisLog', 'channel')
context = zmq.Context() context = zmq.Context()
socket = context.socket(zmq.SUB) socket = context.socket(zmq.SUB)
socket.connect(zmq_url) socket.connect(zmq_url)
socket.setsockopt_string(zmq.SUBSCRIBE, channel) socket.setsockopt_string(zmq.SUBSCRIBE, '')
redis_server = redis.StrictRedis( redis_server = redis.StrictRedis(
host=cfg.get('RedisLog', 'host'), host=cfg.get('RedisLog', 'host'),
@ -29,42 +31,138 @@ serv_coord = redis.StrictRedis(
host='localhost', host='localhost',
port=6250, port=6250,
db=1) db=1)
path_to_db = "/home/sami/Downloads/GeoLite2-City_20171003/GeoLite2-City.mmdb"
reader = geoip2.database.Reader(path_to_db)
channel_proc = "CoordToProcess" channel_proc = "CoordToProcess"
channel_disp = "PicToDisplay" channel_disp = "PicToDisplay"
def publish_coord(coord):
pass
def get_ip(data):
pass
def ip_to_coord(ip):
resp = reader.city(ip)
lat = resp.location.latitude
lon = resp.location.longitude
return {'lat': lat, 'lon': lon}
def default_log(jsonevent):
print('sending', 'log')
return
#redis_server.publish(channel, json.dumps(jsonevent))
def default_keepalive(jsonevent):
print('sending', 'keepalive')
to_push = [ jsonevent['uptime'] ]
to_send = { 'name': 'Keepalive', 'log': json.dumps(to_push) }
redis_server.publish(channel, json.dumps(to_send))
def default_event(jsonevent):
print('sending', 'event')
jsonevent = jsonevent['Event']
to_push = [
jsonevent['threat_level_id'],
jsonevent['id'],
jsonevent['info'],
]
to_send = { 'name': 'Event', 'log': json.dumps(to_push) }
redis_server.publish(channel, json.dumps(to_send))
def default_attribute(jsonevent):
print('sending', 'attribute')
jsonevent = jsonevent['Attribute']
to_push = [
jsonevent['id'],
jsonevent['category'],
jsonevent['type'],
jsonevent['value'],
]
#try to get coord
if jsonevent['category'] == "Network activity":
try:
coord = ip_to_coord(jsonevent['value'])
to_send = {'lat': float(coord['lat']), 'lon': float(coord['lon'])}
serv_coord.publish(channel_proc, json.dumps(to_send))
print('coord sent')
except ValueError:
print("can't resolve ip")
to_send = { 'name': 'Attribute', 'log': json.dumps(to_push) }
redis_server.publish(channel, json.dumps(to_send))
def process_log(event):
event = event.decode('utf8')
topic, eventdata = event.split(' ', maxsplit=1)
jsonevent = json.loads(eventdata)
dico_action[topic](jsonevent)
def main():
while True:
content = socket.recv()
content.replace(b'\n', b'') # remove \n...
process_log(content)
def log_feed():
with open('misp-zmq.2', 'ba') as f:
while True:
time.sleep(1.0)
content = socket.recv()
content.replace(b'\n', b'') # remove \n...
f.write(content)
f.write(b'\n')
print(content)
#redis_server.publish(channel, content)
#if random.randint(1,10)<5:
# time.sleep(0.5)
# redis_server.publish(channel, content)
#if random.randint(1,10)<5:
# time.sleep(0.5)
# redis_server.publish(channel, content)
dico_action = {
"misp_json": default_event,
"misp_json_self": default_keepalive,
"misp_json_attribute": default_attribute,
"misp_json_sighting": default_log,
"misp_json_organisation": default_log,
"misp_json_user": default_log,
"misp_json_conversation": default_log
}
if __name__ == "__main__":
main()
reader.close()
# server side # server side
pubsub = redis_server.pubsub(ignore_subscribe_messages=True) pubsub = redis_server.pubsub(ignore_subscribe_messages=True)
while True: #while True:
rdm = random.randint(1,10) # rdm = random.randint(1,10)
time.sleep(float(rdm)) # time.sleep(float(rdm))
#lux # #lux
lon = random.uniform(5.7373, 6.4823) # lon = random.uniform(5.7373, 6.4823)
lat = random.uniform(49.4061,49.7449) # lat = random.uniform(49.4061,49.7449)
#central eur # #central eur
lon = random.uniform(3.936, 9.890) # lon = random.uniform(3.936, 9.890)
lat = random.uniform(47.957, 50.999) # lat = random.uniform(47.957, 50.999)
content = ["rdm "+str(rdm)] # content = ["rdm "+str(rdm)]
content = [lat,lon] # content = [lat,lon]
jsonContent = json.dumps(content) # jsonContent = json.dumps(content)
to_send = { 'name': 'feeder'+str(rdm), 'log': jsonContent } # to_send = { 'name': 'feeder'+str(rdm), 'log': jsonContent }
redis_server.publish(channel, json.dumps(to_send)) # redis_server.publish(channel, json.dumps(to_send))
serv_coord.publish(channel_proc, json.dumps({'lat': float(lat), 'lon': float(lon)})) # serv_coord.publish(channel_proc, json.dumps({'lat': float(lat), 'lon': float(lon)}))
while True:
#FIXME check if sock.recv is blocking
time.sleep(0.1)
content = socket.recv()
print('sending', (content))
redis_server.publish(channel, content)
if random.randint(1,10)<5:
time.sleep(0.5)
redis_server.publish(channel, content)
if random.randint(1,10)<5:
time.sleep(0.5)
redis_server.publish(channel, content)