From 62522f822f192f3faed118a3e2c32ecf8809f1b3 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Fri, 27 Oct 2017 15:33:09 +0200 Subject: [PATCH] Loads of bug fixes: Added possibility to plot any field in lineChart + Fixed bug GEO where data was not from the correct date + Clear view when no data + Probably solved unsorted dic stored in redis + Support of publish_event_to_zmq --- config.cfg | 3 ++- server.py | 20 +++++++++----------- static/js/geo.js | 26 +++++++++++++++++--------- static/js/index/index.js | 13 ++++++++----- static/js/index/index_lineChart.js | 2 +- templates/index.html | 3 ++- zmq_subscriber.py | 24 +++++++++++++++++++----- 7 files changed, 58 insertions(+), 33 deletions(-) diff --git a/config.cfg b/config.cfg index 0e9e81c..1ebf02f 100644 --- a/config.cfg +++ b/config.cfg @@ -6,6 +6,7 @@ rotation_wait_time = 30 max_img_rotation = 10 hours_spanned = 48 zoomlevel = 15 +item_to_plot = Attribute.category # [1->12] size_dashboard_left_width = 5 size_openStreet_pannel_perc = 55 @@ -19,8 +20,8 @@ zoomlevel = 11 clusteringDistance = 10 [Log] +field_to_plot = Attribute.category fieldname_order=["Event.id", "Attribute.Tag", "Attribute.category", "Attribute.type", ["Attribute.value", "Attribute.comment"]] -#fieldname_order=["id", "category", "type", ["value", "comment"]] char_separator=|| [RedisLog] diff --git a/server.py b/server.py index 1bfcf24..f2dc463 100755 --- a/server.py +++ b/server.py @@ -91,11 +91,8 @@ class EventMessage(): to_ret = { 'log': self.feed, 'feedName': self.feedName, 'zmqName': self.zmqName } return 'data: {}\n\n'.format(json.dumps(to_ret)) -def getZrange(keyCateg, dayNum, topNum): - aDateTime = datetime.datetime.now() - correctDatetime = aDateTime - datetime.timedelta(days = dayNum) - - date_str = str(correctDatetime.year)+str(correctDatetime.month)+str(correctDatetime.day) +def getZrange(keyCateg, date, topNum): + date_str = str(date.year)+str(date.month)+str(date.day) keyname = "{}:{}".format(keyCateg, date_str) data = serv_redis_db.zrange(keyname, 0, 5, desc=True, withscores=True) data = [ [record[0].decode('utf8'), record[1]] for record in data ] @@ -114,6 +111,7 @@ def index(): return render_template('index.html', pannelSize=pannelSize, size_dashboard_width=[cfg.getint('Dashboard' ,'size_dashboard_left_width'), 12-cfg.getint('Dashboard', 'size_dashboard_left_width')], + itemToPlot=cfg.get('Dashboard', 'item_to_plot'), graph_log_refresh_rate=cfg.getint('Dashboard' ,'graph_log_refresh_rate'), char_separator=cfg.get('Log', 'char_separator'), rotation_wait_time=cfg.getint('Dashboard' ,'rotation_wait_time'), @@ -133,23 +131,23 @@ def geo(): @app.route("/_getTopCoord") def getTopCoord(): try: - dayNum = int(request.args.get('dayNum')) + date = datetime.datetime.fromtimestamp(float(request.args.get('date'))) except: - dayNum = 0 + date = datetime.datetime.now() keyCateg = "GEO_COORD" topNum = 6 # default Num - data = getZrange(keyCateg, dayNum, topNum) + data = getZrange(keyCateg, date, topNum) return jsonify(data) @app.route("/_getHitMap") def getHitMap(): try: - dayNum = int(request.args.get('dayNum')) + date = datetime.datetime.fromtimestamp(float(request.args.get('date'))) except: - dayNum = 0 + date = datetime.datetime.now() keyCateg = "GEO_COUNTRY" topNum = -1 # default Num - data = getZrange(keyCateg, dayNum, topNum) + data = getZrange(keyCateg, date, topNum) return jsonify(data) def isCloseTo(coord1, coord2): diff --git a/static/js/geo.js b/static/js/geo.js index 6301e03..bffb5d2 100644 --- a/static/js/geo.js +++ b/static/js/geo.js @@ -110,8 +110,15 @@ $(document).ready(function () { /* TOP LOCATION */ -function updateTopMaps(dayNum) { - $.getJSON(urlTopCoord+"?dayNum="+dayNum, function(list){ +function updateTopMaps(date) { + $.getJSON(urlTopCoord+"?date="+date.getTime()/1000, function(list){ + if (list.length==0) { + for(var i=0; i<6; i++) { // clear maps + allOpenStreetMap[i+1].setView([0, 0], 1); + savedMarker[i+1].remove(); // remove marker + } + savedMarker = {}; + } for(var i=0; i<6 && i'+list[i][1]+')'); markerToUpdate.update(); } else { // create new marker var marker = L.marker([coordJson.lat, coordJson.lon]).addTo(allOpenStreetMap[i+1]); @@ -135,8 +142,10 @@ function updateTopMaps(dayNum) { /* WORLD MAP */ -function updateWorldMap(dayNum) { - $.getJSON(urlHitMap+"?dayNum="+dayNum, function(list){ +function updateWorldMap(date) { + $.getJSON(urlHitMap+"?date="+date.getTime()/1000, function(list){ + regionhits = {}; + worldMapObj.series.regions[0].clear(); for(var i=0; i= regionhits[regionCode] ? regionhitsMax : regionhits[regionCode]; - worldMapObj.series.regions[0].params.max = regionhitsMax; + worldMapObj.series.regions[0].params.max = undefined; worldMapObj.series.regions[0].legend.render(); // Update data worldMapObj.series.regions[0].setValues(regionhits); @@ -210,6 +218,6 @@ function updateAll() { var currentDate = datePickerWidget.datepicker( "getDate" ); var now = new Date(); var numDay = days_between(now, currentDate); - updateTopMaps(numDay); - updateWorldMap(numDay); + updateTopMaps(currentDate); + updateWorldMap(currentDate); } diff --git a/static/js/index/index.js b/static/js/index/index.js index b873ab8..704b45c 100644 --- a/static/js/index/index.js +++ b/static/js/index/index.js @@ -3,6 +3,7 @@ var maxNumPoint = hours_spanned; var keepaliveTime = 0; var emptyArray = []; var _timeoutLed; +var toPlotLocationLog; for(i=0; i
- Categories overtime (hours) + {{ itemToPlot }} overtime (hours)
@@ -241,6 +241,7 @@ small { var linkForDefaultMap = "{{ url_for('static', filename='maps/default.png') }}"; /* DATA FROM CONF */ + var itemToPlot = "{{ itemToPlot }}"; var graph_log_refresh_rate = {{ graph_log_refresh_rate }}; var char_separator = "{{ char_separator }}"; var rotation_wait_time = {{ rotation_wait_time }}; diff --git a/zmq_subscriber.py b/zmq_subscriber.py index 3408289..4e70ee8 100755 --- a/zmq_subscriber.py +++ b/zmq_subscriber.py @@ -1,6 +1,9 @@ #!/usr/bin/env python3.5 import time, datetime +import copy +from collections import OrderedDict +from pprint import pprint import zmq import redis import random @@ -69,8 +72,11 @@ def getCoordAndPublish(zmq_name, supposed_ip, categ): rep = ip_to_coord(supposed_ip) coord = rep['coord'] coord_dic = {'lat': coord['lat'], 'lon': coord['lon']} + ordDic = OrderedDict() + ordDic['lat'] = coord_dic['lat'] + ordDic['lon'] = coord_dic['lon'] coord_list = [coord['lat'], coord['lon']] - push_to_redis_zset('GEO_COORD', json.dumps(coord_dic)) + push_to_redis_zset('GEO_COORD', json.dumps(ordDic)) push_to_redis_zset('GEO_COUNTRY', rep['full_rep'].country.iso_code) push_to_redis_geo('GEO_RAD', coord['lon'], coord['lat'], json.dumps({ 'categ': categ, 'value': supposed_ip })) to_send = { @@ -85,6 +91,8 @@ def getCoordAndPublish(zmq_name, supposed_ip, categ): serv_coord.publish(CHANNELDISP, json.dumps(to_send)) except ValueError: print("can't resolve ip") + except ip2.errors.AddressNotFoundError: + print("Address not in Database") def getFields(obj, fields): jsonWalker = fields.split('.') @@ -110,6 +118,10 @@ def handler_log(zmq_name, jsonevent): print('sending', 'log') return +def handler_dispatcher(zmq_name, jsonObj): + if "Event" in jsonObj: + handler_event(zmq_name, jsonObj) + def handler_keepalive(zmq_name, jsonevent): print('sending', 'keepalive') to_push = [ jsonevent['uptime'] ] @@ -119,15 +131,17 @@ def handler_sighting(zmq_name, jsonsight): print('sending' ,'sighting') return -def handler_event(zmq_name, jsonevent): +def handler_event(zmq_name, jsonobj): #fields: threat_level_id, id, info - jsonevent = jsonevent['Event'] + jsonevent = jsonobj['Event'] #redirect to handler_attribute if 'Attribute' in jsonevent: attributes = jsonevent['Attribute'] if type(attributes) is list: for attr in attributes: - handler_attribute(zmq_name, attr) + jsoncopy = copy.deepcopy(jsonobj) + jsoncopy['Attribute'] = attr + handler_attribute(zmq_name, jsoncopy) else: handler_attribute(zmq_name, attributes) @@ -176,7 +190,7 @@ def main(zmqName): dico_action = { - "misp_json": handler_log, + "misp_json": handler_dispatcher, "misp_json_event": handler_event, "misp_json_self": handler_keepalive, "misp_json_attribute": handler_attribute,