misp-dashboard/helpers/geo_helper.py

263 lines
12 KiB
Python
Raw Normal View History

import datetime
import json
2017-12-04 16:44:44 +01:00
import logging
import math
import os
import random
2019-06-01 08:08:48 +02:00
import sys
import time
from collections import OrderedDict
import redis
import geoip2.database
import phonenumbers
import pycountry
import util
from helpers import live_helper
from phonenumbers import geocoder
class InvalidCoordinate(Exception):
pass
class Geo_helper:
def __init__(self, serv_redis_db, cfg):
self.serv_redis_db = serv_redis_db
self.cfg = cfg
self.serv_coord = redis.StrictRedis(
host=cfg.get('RedisGlobal', 'host'),
port=cfg.getint('RedisGlobal', 'port'),
db=cfg.getint('RedisMap', 'db'))
self.live_helper = live_helper.Live_helper(serv_redis_db, cfg)
2017-12-05 10:23:40 +01:00
#logger
logDir = cfg.get('Log', 'directory')
logfilename = cfg.get('Log', 'helpers_filename')
2017-12-05 10:23:40 +01:00
logPath = os.path.join(logDir, logfilename)
if not os.path.exists(logDir):
os.makedirs(logDir)
try:
handler = logging.FileHandler(logPath)
except PermissionError as error:
print(error)
print("Please fix the above and try again.")
sys.exit(126)
formatter = logging.Formatter('%(asctime)s:%(levelname)s:%(name)s:%(message)s')
handler.setFormatter(formatter)
2017-12-05 10:23:40 +01:00
self.logger = logging.getLogger(__name__)
self.logger.setLevel(logging.INFO)
self.logger.addHandler(handler)
2017-12-05 10:23:40 +01:00
self.keyCategCoord = "GEO_COORD"
self.keyCategCountry = "GEO_COUNTRY"
self.keyCategRad = "GEO_RAD"
self.PATH_TO_DB = cfg.get('RedisMap', 'pathMaxMindDB')
self.PATH_TO_JSON = cfg.get('RedisMap', 'path_countrycode_to_coord_JSON')
self.CHANNELDISP = cfg.get('RedisMap', 'channelDisp')
try:
self.reader = geoip2.database.Reader(self.PATH_TO_DB)
except PermissionError as error:
print(error)
print("Please fix the above and try again.")
sys.exit(126)
self.country_to_iso = {}
for country in pycountry.countries:
try:
self.country_to_iso[country.name] = country.alpha_2
except AttributeError:
pass
with open(self.PATH_TO_JSON) as f:
self.country_code_to_coord = json.load(f)
''' GET '''
def getTopCoord(self, date):
topNum = 6 # default Num
data = util.getZrange(self.serv_redis_db, self.keyCategCoord, date, topNum)
return data
def getHitMap(self, date):
topNum = 0 # all
data = util.getZrange(self.serv_redis_db, self.keyCategCountry, date, topNum)
return data
def getCoordsByRadius(self, dateStart, dateEnd, centerLat, centerLon, radius):
dico_coord = {}
to_return = []
delta = dateEnd - dateStart
for i in range(delta.days+1):
correctDatetime = dateStart + datetime.timedelta(days=i)
date_str = util.getDateStrFormat(correctDatetime)
keyname = "{}:{}".format(self.keyCategRad, date_str)
res = self.serv_redis_db.georadius(keyname, centerLon, centerLat, radius, unit='km', withcoord=True)
#sum up really close coord
for data, coord in res:
flag_added = False
coord = [coord[0], coord[1]]
#list all coord
for dicoCoordStr in dico_coord.keys():
dicoCoord = json.loads(dicoCoordStr)
#if curCoord close to coord
if self.isCloseTo(dicoCoord, coord):
#add data to dico coord
dico_coord[dicoCoordStr].append(data)
flag_added = True
break
# coord not in dic
if not flag_added:
dico_coord[str(coord)] = [data]
for dicoCoord, array in dico_coord.items():
dicoCoord = json.loads(dicoCoord)
to_return.append([array, dicoCoord])
return to_return
''' ADD '''
def getCoordFromIpAndPublish(self, supposed_ip, categ):
try:
rep = self.ip_to_coord(supposed_ip)
coord = rep['coord']
coord_dic = {'lat': coord['lat'], 'lon': coord['lon']}
ordDic = OrderedDict() #keep fields with the same layout in redis
ordDic['lat'] = coord_dic['lat']
ordDic['lon'] = coord_dic['lon']
ordDic['categ'] = categ
ordDic['value'] = supposed_ip
coord_list = [coord['lat'], coord['lon']]
2018-05-09 13:31:49 +02:00
if not self.coordinate_list_valid(coord_list):
raise InvalidCoordinate("Coordinate do not match EPSG:900913 / EPSG:3785 / OSGEO:41001")
self.push_to_redis_zset(self.keyCategCoord, json.dumps(ordDic))
iso_code = rep['full_rep'].country.iso_code if rep['full_rep'].country.iso_code is not None else rep['full_rep'].registered_country.iso_code
country_name = rep['full_rep'].country.name if rep['full_rep'].country.name is not None else rep['full_rep'].registered_country.name
self.push_to_redis_zset(self.keyCategCountry, iso_code)
ordDic = OrderedDict() #keep fields with the same layout in redis
ordDic['categ'] = categ
ordDic['value'] = supposed_ip
self.push_to_redis_geo(self.keyCategRad, coord['lon'], coord['lat'], json.dumps(ordDic))
to_send = {
"coord": coord,
"categ": categ,
"value": supposed_ip,
"country": country_name,
"specifName": rep['full_rep'].subdivisions.most_specific.name,
"cityName": rep['full_rep'].city.name,
"regionCode": iso_code,
}
j_to_send = json.dumps(to_send)
self.serv_coord.publish(self.CHANNELDISP, j_to_send)
self.live_helper.add_to_stream_log_cache('Map', j_to_send)
2017-12-05 10:23:40 +01:00
self.logger.info('Published: {}'.format(json.dumps(to_send)))
except ValueError:
self.logger.warning("Can't resolve IP: " + str(supposed_ip))
except geoip2.errors.AddressNotFoundError:
2017-12-05 10:23:40 +01:00
self.logger.warning("Address not in Database")
except InvalidCoordinate:
self.logger.warning("Coordinate do not follow redis specification")
def getCoordFromPhoneAndPublish(self, phoneNumber, categ):
try:
rep = phonenumbers.parse(phoneNumber, None)
if not (phonenumbers.is_valid_number(rep) or phonenumbers.is_possible_number(rep)):
2017-12-05 10:23:40 +01:00
self.logger.warning("Phone number not valid")
return
country_name = geocoder.country_name_for_number(rep, "en")
country_code = self.country_to_iso[country_name]
if country_code is None:
2017-12-05 10:23:40 +01:00
self.logger.warning("Non matching ISO_CODE")
return
coord = self.country_code_to_coord[country_code.lower()] # countrycode is in upper case
coord_dic = {'lat': coord['lat'], 'lon': coord['long']}
ordDic = OrderedDict() #keep fields with the same layout in redis
ordDic['lat'] = coord_dic['lat']
ordDic['lon'] = coord_dic['lon']
coord_list = [coord['lat'], coord['long']]
2018-05-09 13:31:49 +02:00
if not self.coordinate_list_valid(coord_list):
raise InvalidCoordinate("Coordinate do not match EPSG:900913 / EPSG:3785 / OSGEO:41001")
self.push_to_redis_zset(self.keyCategCoord, json.dumps(ordDic))
self.push_to_redis_zset(self.keyCategCountry, country_code)
ordDic = OrderedDict() #keep fields with the same layout in redis
ordDic['categ'] = categ
ordDic['value'] = phoneNumber
self.push_to_redis_geo(self.keyCategRad, coord['long'], coord['lat'], json.dumps(ordDic))
to_send = {
"coord": coord_dic,
"categ": categ,
"value": phoneNumber,
"country": country_name,
"specifName": "",
"cityName": "",
"regionCode": country_code,
}
j_to_send = json.dumps(to_send)
self.serv_coord.publish(self.CHANNELDISP, j_to_send)
self.live_helper.add_to_stream_log_cache('Map', j_to_send)
2017-12-05 10:23:40 +01:00
self.logger.info('Published: {}'.format(json.dumps(to_send)))
except phonenumbers.NumberParseException:
2017-12-05 10:23:40 +01:00
self.logger.warning("Can't resolve phone number country")
2018-05-09 13:31:49 +02:00
except InvalidCoordinate:
self.logger.warning("Coordinate do not follow redis specification")
''' UTIL '''
def push_to_redis_geo(self, keyCateg, lon, lat, content):
now = datetime.datetime.now()
today_str = util.getDateStrFormat(now)
keyname = "{}:{}".format(keyCateg, today_str)
try:
self.serv_redis_db.geoadd(keyname, lon, lat, content)
except redis.exceptions.ResponseError as error:
print(error)
print("Please fix the above, and make sure you use a redis version that supports the GEOADD command.")
print("To test for support: echo \"help GEOADD\"| redis-cli")
2017-12-05 10:23:40 +01:00
self.logger.debug('Added to redis: keyname={}, lon={}, lat={}, content={}'.format(keyname, lon, lat, content))
def push_to_redis_zset(self, keyCateg, toAdd, endSubkey="", count=1):
if not isinstance(toAdd, str):
self.logger.warning('Can\'t add to redis, element is not of type String. {}'.format(type(toAdd)))
return
now = datetime.datetime.now()
today_str = util.getDateStrFormat(now)
keyname = "{}:{}{}".format(keyCateg, today_str, endSubkey)
self.serv_redis_db.zincrby(keyname, count, toAdd)
2017-12-05 10:23:40 +01:00
self.logger.debug('Added to redis: keyname={}, toAdd={}, count={}'.format(keyname, toAdd, count))
def ip_to_coord(self, ip):
resp = self.reader.city(ip)
try:
lat = float(resp.location.latitude)
lon = float(resp.location.longitude)
except TypeError: # No location, try to use iso_code instead
self.logger.info('no location in geIP.database response for ip: {}'.format(ip))
iso_code = resp.registered_country.iso_code #if no iso_code, throws
coord = self.country_code_to_coord[iso_code.lower()] # countrycode is in upper case
lat = float(coord['lat'])
lon = float(coord['long'])
# 0.0001 correspond to ~10m
# Cast the float so that it has the correct float format
lat_corrected = float("{:.4f}".format(lat))
lon_corrected = float("{:.4f}".format(lon))
return { 'coord': {'lat': lat_corrected, 'lon': lon_corrected}, 'full_rep': resp }
def isCloseTo(self, coord1, coord2):
clusterMeter = self.cfg.getfloat('GEO' ,'clusteringDistance')
clusterThres = math.pow(10, len(str(abs(clusterMeter)))-7) #map meter to coord threshold (~ big approx)
if abs(float(coord1[0]) - float(coord2[0])) <= clusterThres:
if abs(float(coord1[1]) - float(coord2[1])) <= clusterThres:
return True
return False
# adjust latitude and longitude to fit the limit, as specified
# by EPSG:900913 / EPSG:3785 / OSGEO:41001
# coord_list = [lat, lon]
2018-05-09 11:57:31 +02:00
def coordinate_list_valid(self, coord_list):
2018-05-09 13:04:42 +02:00
lat = float(coord_list[0])
lon = float(coord_list[1])
if (-180 <= lon <= 180) and (-85.05112878 <= lat <= 85.05112878):
return True
else:
return False