mirror of https://github.com/MISP/misp-dashboard
132 lines
5.4 KiB
Python
132 lines
5.4 KiB
Python
import math, random
|
|
import os
|
|
import json
|
|
import datetime, time
|
|
import redis
|
|
from collections import OrderedDict
|
|
import geoip2.database
|
|
|
|
import util
|
|
|
|
class Geo_helper:
|
|
def __init__(self, serv_redis_db, cfg):
|
|
self.serv_redis_db = serv_redis_db
|
|
self.cfg = cfg
|
|
self.serv_coord = redis.StrictRedis(
|
|
host=cfg.get('RedisGlobal', 'host'),
|
|
port=cfg.getint('RedisGlobal', 'port'),
|
|
db=cfg.getint('RedisMap', 'db'))
|
|
|
|
self.keyCategCoord = "GEO_COORD"
|
|
self.keyCategCountry = "GEO_COUNTRY"
|
|
self.keyCategRad = "GEO_RAD"
|
|
self.PATH_TO_DB = cfg.get('RedisMap', 'pathMaxMindDB')
|
|
self.CHANNELDISP = cfg.get('RedisMap', 'channelDisp')
|
|
|
|
self.reader = geoip2.database.Reader(self.PATH_TO_DB)
|
|
|
|
''' GET '''
|
|
def getTopCoord(self, date):
|
|
topNum = 6 # default Num
|
|
data = util.getZrange(self.serv_redis_db, self.keyCategCoord, date, topNum)
|
|
return data
|
|
|
|
def getHitMap(self, date):
|
|
topNum = 0 # all
|
|
data = util.getZrange(self.serv_redis_db, self.keyCategCountry, date, topNum)
|
|
return data
|
|
|
|
def getCoordsByRadius(self, dateStart, dateEnd, centerLat, centerLon, radius):
|
|
dico_coord = {}
|
|
to_return = []
|
|
delta = dateEnd - dateStart
|
|
for i in range(delta.days+1):
|
|
correctDatetime = dateStart + datetime.timedelta(days=i)
|
|
date_str = util.getDateStrFormat(correctDatetime)
|
|
keyname = "{}:{}".format(self.keyCategRad, date_str)
|
|
res = self.serv_redis_db.georadius(keyname, centerLon, centerLat, radius, unit='km', withcoord=True)
|
|
|
|
#sum up really close coord
|
|
for data, coord in res:
|
|
flag_added = False
|
|
coord = [coord[0], coord[1]]
|
|
#list all coord
|
|
for dicoCoordStr in dico_coord.keys():
|
|
dicoCoord = json.loads(dicoCoordStr)
|
|
#if curCoord close to coord
|
|
if self.isCloseTo(dicoCoord, coord):
|
|
#add data to dico coord
|
|
dico_coord[dicoCoordStr].append(data)
|
|
flag_added = True
|
|
break
|
|
# coord not in dic
|
|
if not flag_added:
|
|
dico_coord[str(coord)] = [data]
|
|
|
|
for dicoCoord, array in dico_coord.items():
|
|
dicoCoord = json.loads(dicoCoord)
|
|
to_return.append([array, dicoCoord])
|
|
return to_return
|
|
|
|
''' ADD '''
|
|
def getCoordAndPublish(self, supposed_ip, categ):
|
|
try:
|
|
rep = self.ip_to_coord(supposed_ip)
|
|
coord = rep['coord']
|
|
coord_dic = {'lat': coord['lat'], 'lon': coord['lon']}
|
|
ordDic = OrderedDict() #keep fields with the same layout in redis
|
|
ordDic['lat'] = coord_dic['lat']
|
|
ordDic['lon'] = coord_dic['lon']
|
|
coord_list = [coord['lat'], coord['lon']]
|
|
self.push_to_redis_zset(self.keyCategCoord, json.dumps(ordDic))
|
|
self.push_to_redis_zset(self.keyCategCountry, rep['full_rep'].country.iso_code)
|
|
ordDic = OrderedDict() #keep fields with the same layout in redis
|
|
ordDic['categ'] = categ
|
|
ordDic['value'] = supposed_ip
|
|
self.push_to_redis_geo(self.keyCategRad, coord['lon'], coord['lat'], json.dumps(ordDic))
|
|
to_send = {
|
|
"coord": coord,
|
|
"categ": categ,
|
|
"value": supposed_ip,
|
|
"country": rep['full_rep'].country.name,
|
|
"specifName": rep['full_rep'].subdivisions.most_specific.name,
|
|
"cityName": rep['full_rep'].city.name,
|
|
"regionCode": rep['full_rep'].country.iso_code,
|
|
}
|
|
self.serv_coord.publish(self.CHANNELDISP, json.dumps(to_send))
|
|
except ValueError:
|
|
print("can't resolve ip")
|
|
except geoip2.errors.AddressNotFoundError:
|
|
print("Address not in Database")
|
|
|
|
''' UTIL '''
|
|
def push_to_redis_geo(self, keyCateg, lon, lat, content):
|
|
now = datetime.datetime.now()
|
|
today_str = util.getDateStrFormat(now)
|
|
keyname = "{}:{}".format(keyCateg, today_str)
|
|
self.serv_redis_db.geoadd(keyname, lon, lat, content)
|
|
def push_to_redis_zset(self, keyCateg, toAdd, endSubkey="", count=1):
|
|
now = datetime.datetime.now()
|
|
today_str = util.getDateStrFormat(now)
|
|
keyname = "{}:{}{}".format(keyCateg, today_str, endSubkey)
|
|
self.serv_redis_db.zincrby(keyname, toAdd, count)
|
|
|
|
|
|
def ip_to_coord(self, ip):
|
|
resp = self.reader.city(ip)
|
|
lat = float(resp.location.latitude)
|
|
lon = float(resp.location.longitude)
|
|
# 0.0001 correspond to ~10m
|
|
# Cast the float so that it has the correct float format
|
|
lat_corrected = float("{:.4f}".format(lat))
|
|
lon_corrected = float("{:.4f}".format(lon))
|
|
return { 'coord': {'lat': lat_corrected, 'lon': lon_corrected}, 'full_rep': resp }
|
|
|
|
def isCloseTo(self, coord1, coord2):
|
|
clusterMeter = self.cfg.getfloat('GEO' ,'clusteringDistance')
|
|
clusterThres = math.pow(10, len(str(abs(clusterMeter)))-7) #map meter to coord threshold (~ big approx)
|
|
if abs(float(coord1[0]) - float(coord2[0])) <= clusterThres:
|
|
if abs(float(coord1[1]) - float(coord2[1])) <= clusterThres:
|
|
return True
|
|
return False
|