misp-dashboard/helpers/geo_helper.py

import datetime
import json
import logging
import math
import os
import random
import sys
import time
from collections import OrderedDict

import redis

import geoip2.database
import phonenumbers
import pycountry
import util
from helpers import live_helper
from phonenumbers import geocoder


class InvalidCoordinate(Exception):
    pass


class Geo_helper:
    def __init__(self, serv_redis_db, cfg):
        self.serv_redis_db = serv_redis_db
        self.cfg = cfg
        self.serv_coord = redis.StrictRedis(
                host=cfg.get('RedisGlobal', 'host'),
                port=cfg.getint('RedisGlobal', 'port'),
                db=cfg.getint('RedisMap', 'db'))
        self.live_helper = live_helper.Live_helper(serv_redis_db, cfg)

        #logger
        logDir = cfg.get('Log', 'directory')
        logfilename = cfg.get('Log', 'helpers_filename')
        logPath = os.path.join(logDir, logfilename)
        if not os.path.exists(logDir):
            os.makedirs(logDir)
        try:
            handler = logging.FileHandler(logPath)
        except PermissionError as error:
            print(error)
            print("Please fix the above and try again.")
            sys.exit(126)
        formatter = logging.Formatter('%(asctime)s:%(levelname)s:%(name)s:%(message)s')
        handler.setFormatter(formatter)
        self.logger = logging.getLogger(__name__)
        self.logger.setLevel(logging.INFO)
        self.logger.addHandler(handler)

        self.keyCategCoord = "GEO_COORD"
        self.keyCategCountry = "GEO_COUNTRY"
        self.keyCategRad = "GEO_RAD"
        self.PATH_TO_DB = cfg.get('RedisMap', 'pathMaxMindDB')
        self.PATH_TO_JSON = cfg.get('RedisMap', 'path_countrycode_to_coord_JSON')
        self.CHANNELDISP = cfg.get('RedisMap', 'channelDisp')

        try:
            self.reader = geoip2.database.Reader(self.PATH_TO_DB)
        except PermissionError as error:
            print(error)
            print("Please fix the above and try again.")
            sys.exit(126)
        self.country_to_iso = {}
        for country in pycountry.countries:
            try:
                self.country_to_iso[country.name] = country.alpha_2
            except AttributeError:
                pass
        with open(self.PATH_TO_JSON) as f:
            self.country_code_to_coord = json.load(f)

    ''' GET '''
    def getTopCoord(self, date):
            topNum = 6 # default Num
            data = util.getZrange(self.serv_redis_db, self.keyCategCoord, date, topNum)
            return data

    def getHitMap(self, date):
        topNum = 0 # all
        data = util.getZrange(self.serv_redis_db, self.keyCategCountry, date, topNum)
        return data

    def getCoordsByRadius(self, dateStart, dateEnd, centerLat, centerLon, radius):
        dico_coord = {}
        to_return = []
        delta = dateEnd - dateStart
        for i in range(delta.days+1):
            correctDatetime = dateStart + datetime.timedelta(days=i)
            date_str = util.getDateStrFormat(correctDatetime)
            keyname = "{}:{}".format(self.keyCategRad, date_str)
            res = self.serv_redis_db.georadius(keyname, centerLon, centerLat, radius, unit='km', withcoord=True)

            #sum up really close coord
            for data, coord in res:
                flag_added = False
                coord = [coord[0], coord[1]]
                #list all coord
                for dicoCoordStr in dico_coord.keys():
                    dicoCoord = json.loads(dicoCoordStr)
                    #if curCoord close to coord
                    if self.isCloseTo(dicoCoord, coord):
                        #add data to dico coord
                        dico_coord[dicoCoordStr].append(data)
                        flag_added = True
                        break
                # coord not in dic
                if not flag_added:
                    dico_coord[str(coord)] = [data]

            for dicoCoord, array in dico_coord.items():
                dicoCoord = json.loads(dicoCoord)
                to_return.append([array, dicoCoord])
        return to_return

    ''' ADD '''
    def getCoordFromIpAndPublish(self, supposed_ip, categ):
        try:
            rep = self.ip_to_coord(supposed_ip)
            coord = rep['coord']
            coord_dic = {'lat': coord['lat'], 'lon': coord['lon']}
            ordDic = OrderedDict() #keep fields with the same layout in redis
            ordDic['lat'] = coord_dic['lat']
            ordDic['lon'] = coord_dic['lon']
            ordDic['categ'] = categ
            ordDic['value'] = supposed_ip
            coord_list = [coord['lat'], coord['lon']]
            if not self.coordinate_list_valid(coord_list):
                raise InvalidCoordinate("Coordinate do not match EPSG:900913 / EPSG:3785 / OSGEO:41001")
            self.push_to_redis_zset(self.keyCategCoord, json.dumps(ordDic))
            iso_code = rep['full_rep'].country.iso_code if rep['full_rep'].country.iso_code is not None else rep['full_rep'].registered_country.iso_code
            country_name = rep['full_rep'].country.name if rep['full_rep'].country.name is not None else rep['full_rep'].registered_country.name
            self.push_to_redis_zset(self.keyCategCountry, iso_code)
            ordDic = OrderedDict() #keep fields with the same layout in redis
            ordDic['categ'] = categ
            ordDic['value'] = supposed_ip
            self.push_to_redis_geo(self.keyCategRad, coord['lon'], coord['lat'], json.dumps(ordDic))
            to_send = {
                    "coord": coord,
                    "categ": categ,
                    "value": supposed_ip,
                    "country": country_name,
                    "specifName": rep['full_rep'].subdivisions.most_specific.name,
                    "cityName": rep['full_rep'].city.name,
                    "regionCode": iso_code,
                    }
            j_to_send = json.dumps(to_send)
            self.serv_coord.publish(self.CHANNELDISP, j_to_send)
            self.live_helper.add_to_stream_log_cache('Map', j_to_send)
            self.logger.info('Published: {}'.format(json.dumps(to_send)))
        except ValueError:
            self.logger.warning("Can't resolve IP: " + str(supposed_ip))
        except geoip2.errors.AddressNotFoundError:
            self.logger.warning("Address not in Database")
        except InvalidCoordinate:
            self.logger.warning("Coordinate do not follow redis specification")


    def getCoordFromPhoneAndPublish(self, phoneNumber, categ):
        try:
            rep = phonenumbers.parse(phoneNumber, None)
            if not (phonenumbers.is_valid_number(rep) or phonenumbers.is_possible_number(rep)):
                self.logger.warning("Phone number not valid")
                return
            country_name = geocoder.country_name_for_number(rep, "en")
            country_code = self.country_to_iso[country_name]
            if country_code is None:
                self.logger.warning("Non matching ISO_CODE")
                return
            coord = self.country_code_to_coord[country_code.lower()]  # countrycode is in upper case
            coord_dic = {'lat': coord['lat'], 'lon': coord['long']}

            ordDic = OrderedDict() #keep fields with the same layout in redis
            ordDic['lat'] = coord_dic['lat']
            ordDic['lon'] = coord_dic['lon']
            coord_list = [coord['lat'], coord['long']]
            if not self.coordinate_list_valid(coord_list):
                raise InvalidCoordinate("Coordinate do not match EPSG:900913 / EPSG:3785 / OSGEO:41001")
            self.push_to_redis_zset(self.keyCategCoord, json.dumps(ordDic))
            self.push_to_redis_zset(self.keyCategCountry, country_code)
            ordDic = OrderedDict() #keep fields with the same layout in redis
            ordDic['categ'] = categ
            ordDic['value'] = phoneNumber
            self.push_to_redis_geo(self.keyCategRad, coord['long'], coord['lat'], json.dumps(ordDic))
            to_send = {
                    "coord": coord_dic,
                    "categ": categ,
                    "value": phoneNumber,
                    "country": country_name,
                    "specifName": "",
                    "cityName": "",
                    "regionCode": country_code,
                    }
            j_to_send = json.dumps(to_send)
            self.serv_coord.publish(self.CHANNELDISP, j_to_send)
            self.live_helper.add_to_stream_log_cache('Map', j_to_send)
            self.logger.info('Published: {}'.format(json.dumps(to_send)))
        except phonenumbers.NumberParseException:
            self.logger.warning("Can't resolve phone number country")
        except InvalidCoordinate:
            self.logger.warning("Coordinate do not follow redis specification")

    ''' UTIL '''
    def push_to_redis_geo(self, keyCateg, lon, lat, content):
        now = datetime.datetime.now()
        today_str = util.getDateStrFormat(now)
        keyname = "{}:{}".format(keyCateg, today_str)
        try:
            self.serv_redis_db.geoadd(keyname, lon, lat, content)
        except redis.exceptions.ResponseError as error:
            print(error)
            print("Please fix the above, and make sure you use a redis version that supports the GEOADD command.")
            print("To test for support: echo \"help GEOADD\"| redis-cli")
        self.logger.debug('Added to redis: keyname={}, lon={}, lat={}, content={}'.format(keyname, lon, lat, content))

    def push_to_redis_zset(self, keyCateg, toAdd, endSubkey="", count=1):
        if not isinstance(toAdd, str):
            self.logger.warning('Can\'t add to redis, element is not of type String. {}'.format(type(toAdd)))
            return
        now = datetime.datetime.now()
        today_str = util.getDateStrFormat(now)
        keyname = "{}:{}{}".format(keyCateg, today_str, endSubkey)
        self.serv_redis_db.zincrby(keyname, count, toAdd)
        self.logger.debug('Added to redis: keyname={}, toAdd={}, count={}'.format(keyname, toAdd, count))

    def ip_to_coord(self, ip):
        resp = self.reader.city(ip)
        try:
            lat = float(resp.location.latitude)
            lon = float(resp.location.longitude)
        except TypeError: # No location, try to use iso_code instead
            self.logger.info('no location in geIP.database response for ip: {}'.format(ip))
            iso_code = resp.registered_country.iso_code #if no iso_code, throws
            coord = self.country_code_to_coord[iso_code.lower()]  # countrycode is in upper case
            lat = float(coord['lat'])
            lon = float(coord['long'])
        # 0.0001 correspond to ~10m
        # Cast the float so that it has the correct float format
        lat_corrected = float("{:.4f}".format(lat))
        lon_corrected = float("{:.4f}".format(lon))
        return { 'coord': {'lat': lat_corrected, 'lon': lon_corrected}, 'full_rep': resp }

    def isCloseTo(self, coord1, coord2):
        clusterMeter = self.cfg.getfloat('GEO' ,'clusteringDistance')
        clusterThres = math.pow(10, len(str(abs(clusterMeter)))-7) #map meter to coord threshold (~ big approx)
        if abs(float(coord1[0]) - float(coord2[0])) <= clusterThres:
            if abs(float(coord1[1]) - float(coord2[1])) <= clusterThres:
                return True
        return False

    # adjust latitude and longitude to fit the limit, as specified
    # by EPSG:900913 / EPSG:3785 / OSGEO:41001
    # coord_list = [lat, lon]
    def coordinate_list_valid(self, coord_list):
        lat = float(coord_list[0])
        lon = float(coord_list[1])
        if (-180 <= lon <= 180) and (-85.05112878 <= lat <= 85.05112878):
            return True
        else:
            return False