From 7f9ded5262ba59bd8c3369cbbec2ec8d0db3c665 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Fri, 13 Oct 2017 15:03:09 +0200 Subject: [PATCH] Started support of MISP ZMQ --- config.cfg | 2 +- retreive_map_pic.py | 2 +- server.py | 4 +- static/js/index.js | 52 ++++++++------ templates/index.html | 53 +++++++------- zmq_subscriber.py | 162 ++++++++++++++++++++++++++++++++++--------- 6 files changed, 190 insertions(+), 85 deletions(-) diff --git a/config.cfg b/config.cfg index d6467a9..1375df8 100644 --- a/config.cfg +++ b/config.cfg @@ -3,7 +3,7 @@ graph_log_refresh_rate = 1 [Log] -fieldname_order=["time", "level", "name", "message"] +fieldname_order=["time", "id", "category", "type", "value"] [RedisLog] host=localhost diff --git a/retreive_map_pic.py b/retreive_map_pic.py index 96d30b0..55929a1 100755 --- a/retreive_map_pic.py +++ b/retreive_map_pic.py @@ -70,7 +70,7 @@ def download_maps(coord): urls = create_box_around_coord(lon, lat) map_name = download_and_merge(urls, coord) print(map_name) - serv.publish(channel_disp, map_name) + serv.publish(channel_disp, json.dumps({ "path": map_name, "coord": coord })) if __name__ == '__main__': main() diff --git a/server.py b/server.py index fc82f05..8a7eaaf 100755 --- a/server.py +++ b/server.py @@ -100,13 +100,13 @@ def getLogHead(): def event_stream_log(): for msg in subscriber_log.listen(): content = msg['data'] - print('sending', content) yield EventMessage(content).to_json() def event_stream_maps(): for msg in subscriber_map.listen(): content = msg['data'].decode('utf8') - yield 'data: {}\n\n'.format(json.dumps({ 'path': content })) + print('sending map', content) + yield 'data: {}\n\n'.format(content) if __name__ == '__main__': app.run(host='localhost', port=8000, threaded=True) diff --git a/static/js/index.js b/static/js/index.js index 1226f21..e9d317a 100644 --- a/static/js/index.js +++ b/static/js/index.js @@ -166,6 +166,7 @@ $(document).ready(function () { var source_map = new EventSource(urlForMaps); source_map.onmessage = function(event) { var json = jQuery.parseJSON( event.data ); + popupCoord(json.coord); var img2 = linkForDefaultMap.replace(/\/[^\/]+$/, "/"+json.path); $("#img"+img_to_change).fadeOut(400, function(){ $(this).attr('src', img2); }).fadeIn(400); $("#textMap"+img_to_change).fadeOut(400, function(){ $(this).text(json.path); }).fadeIn(400); @@ -219,20 +220,27 @@ function updateLogTable(feedName, log) { sources.addIfNotPresent(feedName); sources.incCountOnSource(feedName); sources.incCountOnSource('global'); - createRow(tableBody, log); - // Remove old row - var logSel = document.getElementById("log_select"); - //get height of pannel, find max num of item - var maxNumLogItem = document.getElementById('divLogTable').clientHeight/37; - maxNumLogItem -= 2; //take heading/padding/... into account - if (maxNumLogItem - parseInt(maxNumLogItem) < 0.5) { //beautifier - maxNumLogItem -= 1; - } - if (tableBody.rows.length > maxNumLogItem) { - while (tableBody.rows.length >= maxNumLogItem){ - tableBody.deleteRow(0); + // only add row for attribute + if (feedName == "Attribute" ) { + createRow(tableBody, log); + + // Remove old row + var logSel = document.getElementById("log_select"); + //get height of pannel, find max num of item + var maxNumLogItem = document.getElementById('divLogTable').clientHeight/37; + maxNumLogItem -= 2; //take heading/padding/... into account + if (maxNumLogItem - parseInt(maxNumLogItem) < 0.5) { //beautifier + maxNumLogItem -= 1; } + if (tableBody.rows.length > maxNumLogItem) { + while (tableBody.rows.length >= maxNumLogItem){ + tableBody.deleteRow(0); + } + } + } else { + // do nothing + return; } } @@ -252,10 +260,7 @@ function slideAndMax(orig, newData) { function createRow(tableBody, log) { var tr = document.createElement('TR'); - var action = document.createElement('TD'); - var x = log[1]; - var y = log[2]; - popupCoord([x,y]); + //var action = document.createElement('TD'); for (var key in log) { if (log.hasOwnProperty(key)) { @@ -277,8 +282,8 @@ function createRow(tableBody, log) { } // action - action.appendChild(document.createTextNode("ACTION")); - tr.appendChild(action); + //action.appendChild(document.createTextNode("ACTION")); + //tr.appendChild(action); tableBody.appendChild(tr); @@ -294,9 +299,9 @@ function createHead(callback) { th.appendChild(document.createTextNode(head)); tr.appendChild(th); } - var action = document.createElement('TH'); - action.appendChild(document.createTextNode("Actions")); - tr.appendChild(action); + //var action = document.createElement('TH'); + //action.appendChild(document.createTextNode("Actions")); + //tr.appendChild(action); document.getElementById('table_log_head').appendChild(tr); callback(); }); @@ -320,7 +325,8 @@ function marker_animation(x, y, curNumMarker) { } function popupCoord(coord) { - var value = coord[0]+coord[1]; + var coord = [coord['lat'], coord['lon']]; + var value = Math.random()*180; pnts = mapObj.latLngToPoint(coord[0], coord[1]) if (pnts != false) { //sometimes latLngToPoint return false mapObj.addMarker(curNumMarker, coord, [value]); @@ -345,7 +351,7 @@ $(function(){ attribute: 'fill', scale: ['#1A0DAB', '#e50000', '#62ff41'], values: [], - min: -180, + min: 0, max: 180 }], }, diff --git a/templates/index.html b/templates/index.html index dbbfc59..7ec0265 100644 --- a/templates/index.html +++ b/templates/index.html @@ -97,6 +97,33 @@ small { + + +
+
+ Log feed +
+
+
+
+
+ + + + + + +
+ +
+ +
+
+
+ +
@@ -122,32 +149,6 @@ small {
- - - - -
- -
- -
-
-
- - -
- - -
-
- Log feed -
-
-
-
-
diff --git a/zmq_subscriber.py b/zmq_subscriber.py index d9120a8..4b80161 100755 --- a/zmq_subscriber.py +++ b/zmq_subscriber.py @@ -8,18 +8,20 @@ import configparser import os import sys import json +import geoip2.database configfile = os.path.join(os.environ['VIRTUAL_ENV'], '../config.cfg') cfg = configparser.ConfigParser() cfg.read(configfile) zmq_url = cfg.get('RedisLog', 'zmq_url') -zmq_url = "tcp://crf.circl.lu:5556" +zmq_url = "tcp://192.168.56.50:50000" +zmq_url = "tcp://localhost:9990" channel = cfg.get('RedisLog', 'channel') context = zmq.Context() socket = context.socket(zmq.SUB) socket.connect(zmq_url) -socket.setsockopt_string(zmq.SUBSCRIBE, channel) +socket.setsockopt_string(zmq.SUBSCRIBE, '') redis_server = redis.StrictRedis( host=cfg.get('RedisLog', 'host'), @@ -29,42 +31,138 @@ serv_coord = redis.StrictRedis( host='localhost', port=6250, db=1) +path_to_db = "/home/sami/Downloads/GeoLite2-City_20171003/GeoLite2-City.mmdb" +reader = geoip2.database.Reader(path_to_db) channel_proc = "CoordToProcess" channel_disp = "PicToDisplay" + +def publish_coord(coord): + pass + +def get_ip(data): + pass + +def ip_to_coord(ip): + resp = reader.city(ip) + lat = resp.location.latitude + lon = resp.location.longitude + return {'lat': lat, 'lon': lon} + +def default_log(jsonevent): + print('sending', 'log') + return + #redis_server.publish(channel, json.dumps(jsonevent)) + +def default_keepalive(jsonevent): + print('sending', 'keepalive') + to_push = [ jsonevent['uptime'] ] + to_send = { 'name': 'Keepalive', 'log': json.dumps(to_push) } + redis_server.publish(channel, json.dumps(to_send)) + +def default_event(jsonevent): + print('sending', 'event') + jsonevent = jsonevent['Event'] + to_push = [ + jsonevent['threat_level_id'], + jsonevent['id'], + jsonevent['info'], + ] + to_send = { 'name': 'Event', 'log': json.dumps(to_push) } + redis_server.publish(channel, json.dumps(to_send)) + +def default_attribute(jsonevent): + print('sending', 'attribute') + jsonevent = jsonevent['Attribute'] + to_push = [ + jsonevent['id'], + jsonevent['category'], + jsonevent['type'], + jsonevent['value'], + ] + + #try to get coord + if jsonevent['category'] == "Network activity": + try: + coord = ip_to_coord(jsonevent['value']) + to_send = {'lat': float(coord['lat']), 'lon': float(coord['lon'])} + serv_coord.publish(channel_proc, json.dumps(to_send)) + print('coord sent') + except ValueError: + print("can't resolve ip") + + to_send = { 'name': 'Attribute', 'log': json.dumps(to_push) } + redis_server.publish(channel, json.dumps(to_send)) + + +def process_log(event): + event = event.decode('utf8') + topic, eventdata = event.split(' ', maxsplit=1) + jsonevent = json.loads(eventdata) + dico_action[topic](jsonevent) + + +def main(): + while True: + content = socket.recv() + content.replace(b'\n', b'') # remove \n... + process_log(content) + +def log_feed(): + with open('misp-zmq.2', 'ba') as f: + + while True: + time.sleep(1.0) + content = socket.recv() + content.replace(b'\n', b'') # remove \n... + f.write(content) + f.write(b'\n') + print(content) + #redis_server.publish(channel, content) + + #if random.randint(1,10)<5: + # time.sleep(0.5) + # redis_server.publish(channel, content) + + #if random.randint(1,10)<5: + # time.sleep(0.5) + # redis_server.publish(channel, content) + +dico_action = { + "misp_json": default_event, + "misp_json_self": default_keepalive, + "misp_json_attribute": default_attribute, + "misp_json_sighting": default_log, + "misp_json_organisation": default_log, + "misp_json_user": default_log, + "misp_json_conversation": default_log + } + + +if __name__ == "__main__": + main() + reader.close() + + + # server side pubsub = redis_server.pubsub(ignore_subscribe_messages=True) -while True: - rdm = random.randint(1,10) - time.sleep(float(rdm)) - #lux - lon = random.uniform(5.7373, 6.4823) - lat = random.uniform(49.4061,49.7449) - #central eur - lon = random.uniform(3.936, 9.890) - lat = random.uniform(47.957, 50.999) - content = ["rdm "+str(rdm)] - content = [lat,lon] - jsonContent = json.dumps(content) - to_send = { 'name': 'feeder'+str(rdm), 'log': jsonContent } - redis_server.publish(channel, json.dumps(to_send)) - serv_coord.publish(channel_proc, json.dumps({'lat': float(lat), 'lon': float(lon)})) +#while True: +# rdm = random.randint(1,10) +# time.sleep(float(rdm)) +# #lux +# lon = random.uniform(5.7373, 6.4823) +# lat = random.uniform(49.4061,49.7449) +# #central eur +# lon = random.uniform(3.936, 9.890) +# lat = random.uniform(47.957, 50.999) +# content = ["rdm "+str(rdm)] +# content = [lat,lon] +# jsonContent = json.dumps(content) +# to_send = { 'name': 'feeder'+str(rdm), 'log': jsonContent } +# redis_server.publish(channel, json.dumps(to_send)) +# serv_coord.publish(channel_proc, json.dumps({'lat': float(lat), 'lon': float(lon)})) - -while True: - #FIXME check if sock.recv is blocking - time.sleep(0.1) - content = socket.recv() - print('sending', (content)) - redis_server.publish(channel, content) - - if random.randint(1,10)<5: - time.sleep(0.5) - redis_server.publish(channel, content) - - if random.randint(1,10)<5: - time.sleep(0.5) - redis_server.publish(channel, content)