misp-dashboard/server.py

270 lines
9.0 KiB
Python
Raw Normal View History

2017-08-24 07:25:13 +02:00
#!/usr/bin/env python3.5
2017-10-25 17:32:06 +02:00
from flask import Flask, render_template, request, Response, jsonify
2017-08-24 11:43:23 +02:00
import json
import redis
2017-10-26 16:59:02 +02:00
import random, math
import configparser
2017-08-24 11:47:54 +02:00
from time import gmtime as now
from time import sleep, strftime
2017-10-25 17:32:06 +02:00
import datetime
import os
configfile = os.path.join(os.environ['DASH_CONFIG'], 'config.cfg')
cfg = configparser.ConfigParser()
cfg.read(configfile)
2017-08-24 11:43:23 +02:00
2017-08-24 07:25:13 +02:00
app = Flask(__name__)
2017-10-11 10:47:11 +02:00
redis_server_log = redis.StrictRedis(
host=cfg.get('RedisGlobal', 'host'),
port=cfg.getint('RedisGlobal', 'port'),
2017-10-11 10:47:11 +02:00
db=cfg.getint('RedisLog', 'db'))
redis_server_map = redis.StrictRedis(
host=cfg.get('RedisGlobal', 'host'),
port=cfg.getint('RedisGlobal', 'port'),
2017-10-11 10:47:11 +02:00
db=cfg.getint('RedisMap', 'db'))
2017-10-25 17:32:06 +02:00
serv_redis_db = redis.StrictRedis(
host=cfg.get('RedisGlobal', 'host'),
port=cfg.getint('RedisGlobal', 'port'),
2017-10-25 17:32:06 +02:00
db=cfg.getint('RedisDB', 'db'))
2017-10-11 10:47:11 +02:00
subscriber_log = redis_server_log.pubsub(ignore_subscribe_messages=True)
subscriber_log.psubscribe(cfg.get('RedisLog', 'channel'))
subscriber_map = redis_server_map.pubsub(ignore_subscribe_messages=True)
subscriber_map.psubscribe(cfg.get('RedisMap', 'channelDisp'))
eventNumber = 0
class LogItem():
2017-08-24 11:43:23 +02:00
2017-09-12 12:27:35 +02:00
FIELDNAME_ORDER = []
2017-10-24 15:17:52 +02:00
FIELDNAME_ORDER_HEADER = []
2017-10-27 12:05:14 +02:00
FIELDNAME_ORDER.append("Time")
FIELDNAME_ORDER_HEADER.append("Time")
2017-09-12 12:27:35 +02:00
for item in json.loads(cfg.get('Log', 'fieldname_order')):
2017-10-24 15:17:52 +02:00
if type(item) is list:
FIELDNAME_ORDER_HEADER.append(" | ".join(item))
else:
FIELDNAME_ORDER_HEADER.append(item)
2017-09-12 12:27:35 +02:00
FIELDNAME_ORDER.append(item)
2017-08-24 11:43:23 +02:00
def __init__(self, feed):
self.time = strftime("%H:%M:%S", now())
2017-09-12 12:34:15 +02:00
#FIXME Parse feed message?
2017-09-12 12:27:35 +02:00
self.fields = []
2017-09-12 12:34:15 +02:00
self.fields.append(self.time)
2017-09-12 12:27:35 +02:00
for f in feed:
self.fields.append(f)
2017-08-24 11:43:23 +02:00
def get_head_row(self):
to_ret = []
2017-10-24 15:17:52 +02:00
for fn in LogItem.FIELDNAME_ORDER_HEADER:
2017-10-27 12:05:14 +02:00
to_ret.append(fn)
2017-08-24 11:43:23 +02:00
return to_ret
def get_row(self):
to_ret = {}
#Number to keep them sorted (jsonify sort keys)
2017-10-24 15:17:52 +02:00
for item in range(len(LogItem.FIELDNAME_ORDER)):
2017-09-12 12:27:35 +02:00
try:
2017-10-24 15:17:52 +02:00
to_ret[item] = self.fields[item]
2017-09-12 12:27:35 +02:00
except IndexError: # not enough field in rcv item
2017-10-24 15:17:52 +02:00
to_ret[item] = ''
2017-08-24 11:43:23 +02:00
return to_ret
class EventMessage():
# Suppose the event message is a json with the format {name: 'feedName', log:'logData'}
2017-08-24 11:43:23 +02:00
def __init__(self, msg):
msg = msg.decode('utf8')
try:
jsonMsg = json.loads(msg)
except json.JSONDecodeError:
2017-09-12 12:27:35 +02:00
print('json decode error')
jsonMsg = { 'name': "undefined" ,'log': json.loads(msg) }
2017-08-24 11:43:23 +02:00
self.feedName = jsonMsg['name']
self.zmqName = jsonMsg['zmqName']
2017-09-12 12:27:35 +02:00
self.feed = json.loads(jsonMsg['log'])
self.feed = LogItem(self.feed).get_row()
2017-08-24 11:43:23 +02:00
def to_json(self):
to_ret = { 'log': self.feed, 'feedName': self.feedName, 'zmqName': self.zmqName }
2017-08-24 11:43:23 +02:00
return 'data: {}\n\n'.format(json.dumps(to_ret))
def getZrange(keyCateg, date, topNum):
date_str = str(date.year)+str(date.month)+str(date.day)
2017-10-25 17:32:06 +02:00
keyname = "{}:{}".format(keyCateg, date_str)
data = serv_redis_db.zrange(keyname, 0, 5, desc=True, withscores=True)
2017-10-30 16:28:32 +01:00
data = [ [record[0].decode('utf8'), record[1]] for record in data ]
2017-10-25 17:32:06 +02:00
return data
2017-08-24 07:25:13 +02:00
@app.route("/")
def index():
ratioCorrection = 88
pannelSize = [
"{:.0f}".format(cfg.getint('Dashboard' ,'size_openStreet_pannel_perc')/100*ratioCorrection),
"{:.0f}".format((100-cfg.getint('Dashboard' ,'size_openStreet_pannel_perc'))/100*ratioCorrection),
"{:.0f}".format(cfg.getint('Dashboard' ,'size_world_pannel_perc')/100*ratioCorrection),
"{:.0f}".format((100-cfg.getint('Dashboard' ,'size_world_pannel_perc'))/100*ratioCorrection)
]
2017-10-30 16:28:32 +01:00
return render_template('index.html',
pannelSize=pannelSize,
size_dashboard_width=[cfg.getint('Dashboard' ,'size_dashboard_left_width'), 12-cfg.getint('Dashboard', 'size_dashboard_left_width')],
itemToPlot=cfg.get('Dashboard', 'item_to_plot'),
graph_log_refresh_rate=cfg.getint('Dashboard' ,'graph_log_refresh_rate'),
2017-10-24 15:17:52 +02:00
char_separator=cfg.get('Log', 'char_separator'),
rotation_wait_time=cfg.getint('Dashboard' ,'rotation_wait_time'),
max_img_rotation=cfg.getint('Dashboard' ,'max_img_rotation'),
2017-10-23 16:56:25 +02:00
hours_spanned=cfg.getint('Dashboard' ,'hours_spanned'),
zoomlevel=cfg.getint('Dashboard' ,'zoomlevel')
)
2017-08-24 07:25:13 +02:00
2017-10-25 16:22:14 +02:00
@app.route("/geo")
def geo():
return render_template('geo.html',
zoomlevel=cfg.getint('GEO' ,'zoomlevel'),
default_updateFrequency=cfg.getint('GEO' ,'updateFrequency')
)
2017-10-25 16:22:14 +02:00
2017-10-30 16:28:32 +01:00
@app.route("/contrib")
def contrib():
return render_template('contrib.html',
)
@app.route("/_getTopContributor")
def getTopContributor():
data = [
{
'progression': '',
'logo_path': 'logo1',
'org': 'CIRCL',
},
{
'progression': '',
'logo_path': 'logo2',
'org': 'CASES',
},
{
'progression': '',
'logo_path': 'logo3',
'org': 'SMILE',
},
{
'progression': '',
'logo_path': 'logo4',
'org': 'ORG4',
},
{
'progression': '',
'logo_path': 'logo5',
'org': 'ORG5',
},
]
return jsonify(data)
@app.route("/_getTop5Overtime")
def getTop5Overtime():
data = [{'label': 'CIRCL', 'data': [[0, 4], [1, 7], [2,14]]}, {'label': 'CASES', 'data': [[0, 1], [1, 5], [2,2]]}]
return jsonify(data)
2017-10-25 17:32:06 +02:00
@app.route("/_getTopCoord")
def getTopCoord():
try:
date = datetime.datetime.fromtimestamp(float(request.args.get('date')))
2017-10-25 17:32:06 +02:00
except:
date = datetime.datetime.now()
2017-10-25 17:32:06 +02:00
keyCateg = "GEO_COORD"
topNum = 6 # default Num
data = getZrange(keyCateg, date, topNum)
2017-10-25 17:32:06 +02:00
return jsonify(data)
@app.route("/_getHitMap")
def getHitMap():
try:
date = datetime.datetime.fromtimestamp(float(request.args.get('date')))
2017-10-25 17:32:06 +02:00
except:
date = datetime.datetime.now()
2017-10-25 17:32:06 +02:00
keyCateg = "GEO_COUNTRY"
topNum = -1 # default Num
data = getZrange(keyCateg, date, topNum)
2017-10-25 17:32:06 +02:00
return jsonify(data)
2017-10-26 17:57:31 +02:00
def isCloseTo(coord1, coord2):
clusterMeter = cfg.getfloat('GEO' ,'clusteringDistance')
clusterThres = math.pow(10, len(str(abs(clusterMeter)))-7) #map meter to coord threshold (~ big approx)
if abs(float(coord1[0]) - float(coord2[0])) <= clusterThres:
if abs(float(coord1[1]) - float(coord2[1])) <= clusterThres:
2017-10-26 17:57:31 +02:00
return True
return False
2017-10-26 16:59:02 +02:00
@app.route("/_getCoordsByRadius")
def getCoordsByRadius():
2017-10-26 17:57:31 +02:00
dico_coord = {}
2017-10-26 16:59:02 +02:00
to_return = []
try:
dateStart = datetime.datetime.fromtimestamp(float(request.args.get('dateStart')))
dateEnd = datetime.datetime.fromtimestamp(float(request.args.get('dateEnd')))
centerLat = request.args.get('centerLat')
centerLon = request.args.get('centerLon')
radius = int(math.ceil(float(request.args.get('radius'))))
except:
return jsonify(to_return)
delta = dateEnd - dateStart
for i in range(delta.days+1):
correctDatetime = dateStart + datetime.timedelta(days=i)
date_str = str(correctDatetime.year)+str(correctDatetime.month)+str(correctDatetime.day)
keyCateg = 'GEO_RAD'
keyname = "{}:{}".format(keyCateg, date_str)
res = serv_redis_db.georadius(keyname, centerLon, centerLat, radius, unit='km', withcoord=True)
2017-10-26 17:57:31 +02:00
#sum up really close coord
for data, coord in res:
flag_added = False
coord = [coord[0], coord[1]]
#list all coord
for dicoCoordStr in dico_coord.keys():
dicoCoord = json.loads(dicoCoordStr)
#if curCoord close to coord
if isCloseTo(dicoCoord, coord):
#add data to dico coord
dico_coord[dicoCoordStr].append(data)
flag_added = True
break
# coord not in dic
if not flag_added:
dico_coord[str(coord)] = [data]
for dicoCoord, array in dico_coord.items():
dicoCoord = json.loads(dicoCoord)
to_return.append([array, dicoCoord])
2017-10-26 16:59:02 +02:00
return jsonify(to_return)
2017-08-24 11:43:23 +02:00
@app.route("/_logs")
def logs():
2017-10-11 10:47:11 +02:00
return Response(event_stream_log(), mimetype="text/event-stream")
@app.route("/_maps")
def maps():
return Response(event_stream_maps(), mimetype="text/event-stream")
2017-08-24 11:43:23 +02:00
@app.route("/_get_log_head")
def getLogHead():
return json.dumps(LogItem('').get_head_row())
2017-08-24 11:43:23 +02:00
2017-10-11 10:47:11 +02:00
def event_stream_log():
for msg in subscriber_log.listen():
content = msg['data']
yield EventMessage(content).to_json()
2017-08-24 07:25:13 +02:00
2017-10-11 10:47:11 +02:00
def event_stream_maps():
for msg in subscriber_map.listen():
content = msg['data'].decode('utf8')
2017-10-13 15:03:09 +02:00
yield 'data: {}\n\n'.format(content)
2017-10-11 10:47:11 +02:00
2017-08-24 07:25:13 +02:00
if __name__ == '__main__':
2017-10-27 12:05:14 +02:00
app.run(host='localhost', port=8001, threaded=True)