From 90bdd45a18f5a1a52f87137bb73052fe0a61d781 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Mon, 27 Nov 2017 16:30:49 +0100 Subject: [PATCH] feature: Added geolocation of phone numbers --- config/config.cfg.default | 1 + data/country_code_lat_long.json | 962 ++++++++++++++++++++++++++++++++ geo_helper.py | 47 +- install_dependencies.sh | 2 +- zmq_subscriber.py | 7 +- 5 files changed, 1015 insertions(+), 4 deletions(-) create mode 100644 data/country_code_lat_long.json diff --git a/config/config.cfg.default b/config/config.cfg.default index ff2bbaa..ad88f5f 100644 --- a/config/config.cfg.default +++ b/config/config.cfg.default @@ -53,6 +53,7 @@ channelLastAwards = lastAwards db=1 channelDisp=PicToDisplay pathMaxMindDB=./data/GeoLite2-City_20171003/GeoLite2-City.mmdb +path_countrycode_to_coord_JSON=./data/country_code_lat_long.json [RedisDB] db=2 diff --git a/data/country_code_lat_long.json b/data/country_code_lat_long.json new file mode 100644 index 0000000..eb0f220 --- /dev/null +++ b/data/country_code_lat_long.json @@ -0,0 +1,962 @@ +{ + "ad": { + "lat": "42.5000", + "long": "1.5000" + }, + "ae": { + "lat": "24.0000", + "long": "54.0000" + }, + "af": { + "lat": "33.0000", + "long": "65.0000" + }, + "ag": { + "lat": "17.0500", + "long": "-61.8000" + }, + "ai": { + "lat": "18.2500", + "long": "-63.1667" + }, + "al": { + "lat": "41.0000", + "long": "20.0000" + }, + "am": { + "lat": "40.0000", + "long": "45.0000" + }, + "an": { + "lat": "12.2500", + "long": "-68.7500" + }, + "ao": { + "lat": "-12.5000", + "long": "18.5000" + }, + "ap": { + "lat": "35.0000", + "long": "105.0000" + }, + "aq": { + "lat": "-90.0000", + "long": "0.0000" + }, + "ar": { + "lat": "-34.0000", + "long": "-64.0000" + }, + "as": { + "lat": "-14.3333", + "long": "-170.0000" + }, + "at": { + "lat": "47.3333", + "long": "13.3333" + }, + "au": { + "lat": "-27.0000", + "long": "133.0000" + }, + "aw": { + "lat": "12.5000", + "long": "-69.9667" + }, + "az": { + "lat": "40.5000", + "long": "47.5000" + }, + "ba": { + "lat": "44.0000", + "long": "18.0000" + }, + "bb": { + "lat": "13.1667", + "long": "-59.5333" + }, + "bd": { + "lat": "24.0000", + "long": "90.0000" + }, + "be": { + "lat": "50.8333", + "long": "4.0000" + }, + "bf": { + "lat": "13.0000", + "long": "-2.0000" + }, + "bg": { + "lat": "43.0000", + "long": "25.0000" + }, + "bh": { + "lat": "26.0000", + "long": "50.5500" + }, + "bi": { + "lat": "-3.5000", + "long": "30.0000" + }, + "bj": { + "lat": "9.5000", + "long": "2.2500" + }, + "bm": { + "lat": "32.3333", + "long": "-64.7500" + }, + "bn": { + "lat": "4.5000", + "long": "114.6667" + }, + "bo": { + "lat": "-17.0000", + "long": "-65.0000" + }, + "br": { + "lat": "-10.0000", + "long": "-55.0000" + }, + "bs": { + "lat": "24.2500", + "long": "-76.0000" + }, + "bt": { + "lat": "27.5000", + "long": "90.5000" + }, + "bv": { + "lat": "-54.4333", + "long": "3.4000" + }, + "bw": { + "lat": "-22.0000", + "long": "24.0000" + }, + "by": { + "lat": "53.0000", + "long": "28.0000" + }, + "bz": { + "lat": "17.2500", + "long": "-88.7500" + }, + "ca": { + "lat": "60.0000", + "long": "-95.0000" + }, + "cc": { + "lat": "-12.5000", + "long": "96.8333" + }, + "cd": { + "lat": "0.0000", + "long": "25.0000" + }, + "cf": { + "lat": "7.0000", + "long": "21.0000" + }, + "cg": { + "lat": "-1.0000", + "long": "15.0000" + }, + "ch": { + "lat": "47.0000", + "long": "8.0000" + }, + "ci": { + "lat": "8.0000", + "long": "-5.0000" + }, + "ck": { + "lat": "-21.2333", + "long": "-159.7667" + }, + "cl": { + "lat": "-30.0000", + "long": "-71.0000" + }, + "cm": { + "lat": "6.0000", + "long": "12.0000" + }, + "cn": { + "lat": "35.0000", + "long": "105.0000" + }, + "co": { + "lat": "4.0000", + "long": "-72.0000" + }, + "cr": { + "lat": "10.0000", + "long": "-84.0000" + }, + "cu": { + "lat": "21.5000", + "long": "-80.0000" + }, + "cv": { + "lat": "16.0000", + "long": "-24.0000" + }, + "cx": { + "lat": "-10.5000", + "long": "105.6667" + }, + "cy": { + "lat": "35.0000", + "long": "33.0000" + }, + "cz": { + "lat": "49.7500", + "long": "15.5000" + }, + "de": { + "lat": "51.0000", + "long": "9.0000" + }, + "dj": { + "lat": "11.5000", + "long": "43.0000" + }, + "dk": { + "lat": "56.0000", + "long": "10.0000" + }, + "dm": { + "lat": "15.4167", + "long": "-61.3333" + }, + "do": { + "lat": "19.0000", + "long": "-70.6667" + }, + "dz": { + "lat": "28.0000", + "long": "3.0000" + }, + "ec": { + "lat": "-2.0000", + "long": "-77.5000" + }, + "ee": { + "lat": "59.0000", + "long": "26.0000" + }, + "eg": { + "lat": "27.0000", + "long": "30.0000" + }, + "eh": { + "lat": "24.5000", + "long": "-13.0000" + }, + "er": { + "lat": "15.0000", + "long": "39.0000" + }, + "es": { + "lat": "40.0000", + "long": "-4.0000" + }, + "et": { + "lat": "8.0000", + "long": "38.0000" + }, + "eu": { + "lat": "47.0000", + "long": "8.0000" + }, + "fi": { + "lat": "64.0000", + "long": "26.0000" + }, + "fj": { + "lat": "-18.0000", + "long": "175.0000" + }, + "fk": { + "lat": "-51.7500", + "long": "-59.0000" + }, + "fm": { + "lat": "6.9167", + "long": "158.2500" + }, + "fo": { + "lat": "62.0000", + "long": "-7.0000" + }, + "fr": { + "lat": "46.0000", + "long": "2.0000" + }, + "ga": { + "lat": "-1.0000", + "long": "11.7500" + }, + "gb": { + "lat": "54.0000", + "long": "-2.0000" + }, + "gd": { + "lat": "12.1167", + "long": "-61.6667" + }, + "ge": { + "lat": "42.0000", + "long": "43.5000" + }, + "gf": { + "lat": "4.0000", + "long": "-53.0000" + }, + "gh": { + "lat": "8.0000", + "long": "-2.0000" + }, + "gi": { + "lat": "36.1833", + "long": "-5.3667" + }, + "gl": { + "lat": "72.0000", + "long": "-40.0000" + }, + "gm": { + "lat": "13.4667", + "long": "-16.5667" + }, + "gn": { + "lat": "11.0000", + "long": "-10.0000" + }, + "gp": { + "lat": "16.2500", + "long": "-61.5833" + }, + "gq": { + "lat": "2.0000", + "long": "10.0000" + }, + "gr": { + "lat": "39.0000", + "long": "22.0000" + }, + "gs": { + "lat": "-54.5000", + "long": "-37.0000" + }, + "gt": { + "lat": "15.5000", + "long": "-90.2500" + }, + "gu": { + "lat": "13.4667", + "long": "144.7833" + }, + "gw": { + "lat": "12.0000", + "long": "-15.0000" + }, + "gy": { + "lat": "5.0000", + "long": "-59.0000" + }, + "hk": { + "lat": "22.2500", + "long": "114.1667" + }, + "hm": { + "lat": "-53.1000", + "long": "72.5167" + }, + "hn": { + "lat": "15.0000", + "long": "-86.5000" + }, + "hr": { + "lat": "45.1667", + "long": "15.5000" + }, + "ht": { + "lat": "19.0000", + "long": "-72.4167" + }, + "hu": { + "lat": "47.0000", + "long": "20.0000" + }, + "id": { + "lat": "-5.0000", + "long": "120.0000" + }, + "ie": { + "lat": "53.0000", + "long": "-8.0000" + }, + "il": { + "lat": "31.5000", + "long": "34.7500" + }, + "in": { + "lat": "20.0000", + "long": "77.0000" + }, + "io": { + "lat": "-6.0000", + "long": "71.5000" + }, + "iq": { + "lat": "33.0000", + "long": "44.0000" + }, + "ir": { + "lat": "32.0000", + "long": "53.0000" + }, + "is": { + "lat": "65.0000", + "long": "-18.0000" + }, + "it": { + "lat": "42.8333", + "long": "12.8333" + }, + "jm": { + "lat": "18.2500", + "long": "-77.5000" + }, + "jo": { + "lat": "31.0000", + "long": "36.0000" + }, + "jp": { + "lat": "36.0000", + "long": "138.0000" + }, + "ke": { + "lat": "1.0000", + "long": "38.0000" + }, + "kg": { + "lat": "41.0000", + "long": "75.0000" + }, + "kh": { + "lat": "13.0000", + "long": "105.0000" + }, + "ki": { + "lat": "1.4167", + "long": "173.0000" + }, + "km": { + "lat": "-12.1667", + "long": "44.2500" + }, + "kn": { + "lat": "17.3333", + "long": "-62.7500" + }, + "kp": { + "lat": "40.0000", + "long": "127.0000" + }, + "kr": { + "lat": "37.0000", + "long": "127.5000" + }, + "kw": { + "lat": "29.3375", + "long": "47.6581" + }, + "ky": { + "lat": "19.5000", + "long": "-80.5000" + }, + "kz": { + "lat": "48.0000", + "long": "68.0000" + }, + "la": { + "lat": "18.0000", + "long": "105.0000" + }, + "lb": { + "lat": "33.8333", + "long": "35.8333" + }, + "lc": { + "lat": "13.8833", + "long": "-61.1333" + }, + "li": { + "lat": "47.1667", + "long": "9.5333" + }, + "lk": { + "lat": "7.0000", + "long": "81.0000" + }, + "lr": { + "lat": "6.5000", + "long": "-9.5000" + }, + "ls": { + "lat": "-29.5000", + "long": "28.5000" + }, + "lt": { + "lat": "56.0000", + "long": "24.0000" + }, + "lu": { + "lat": "49.7500", + "long": "6.1667" + }, + "lv": { + "lat": "57.0000", + "long": "25.0000" + }, + "ly": { + "lat": "25.0000", + "long": "17.0000" + }, + "ma": { + "lat": "32.0000", + "long": "-5.0000" + }, + "mc": { + "lat": "43.7333", + "long": "7.4000" + }, + "md": { + "lat": "47.0000", + "long": "29.0000" + }, + "me": { + "lat": "42.0000", + "long": "19.0000" + }, + "mg": { + "lat": "-20.0000", + "long": "47.0000" + }, + "mh": { + "lat": "9.0000", + "long": "168.0000" + }, + "mk": { + "lat": "41.8333", + "long": "22.0000" + }, + "ml": { + "lat": "17.0000", + "long": "-4.0000" + }, + "mm": { + "lat": "22.0000", + "long": "98.0000" + }, + "mn": { + "lat": "46.0000", + "long": "105.0000" + }, + "mo": { + "lat": "22.1667", + "long": "113.5500" + }, + "mp": { + "lat": "15.2000", + "long": "145.7500" + }, + "mq": { + "lat": "14.6667", + "long": "-61.0000" + }, + "mr": { + "lat": "20.0000", + "long": "-12.0000" + }, + "ms": { + "lat": "16.7500", + "long": "-62.2000" + }, + "mt": { + "lat": "35.8333", + "long": "14.5833" + }, + "mu": { + "lat": "-20.2833", + "long": "57.5500" + }, + "mv": { + "lat": "3.2500", + "long": "73.0000" + }, + "mw": { + "lat": "-13.5000", + "long": "34.0000" + }, + "mx": { + "lat": "23.0000", + "long": "-102.0000" + }, + "my": { + "lat": "2.5000", + "long": "112.5000" + }, + "mz": { + "lat": "-18.2500", + "long": "35.0000" + }, + "na": { + "lat": "-22.0000", + "long": "17.0000" + }, + "nc": { + "lat": "-21.5000", + "long": "165.5000" + }, + "ne": { + "lat": "16.0000", + "long": "8.0000" + }, + "nf": { + "lat": "-29.0333", + "long": "167.9500" + }, + "ng": { + "lat": "10.0000", + "long": "8.0000" + }, + "ni": { + "lat": "13.0000", + "long": "-85.0000" + }, + "nl": { + "lat": "52.5000", + "long": "5.7500" + }, + "no": { + "lat": "62.0000", + "long": "10.0000" + }, + "np": { + "lat": "28.0000", + "long": "84.0000" + }, + "nr": { + "lat": "-0.5333", + "long": "166.9167" + }, + "nu": { + "lat": "-19.0333", + "long": "-169.8667" + }, + "nz": { + "lat": "-41.0000", + "long": "174.0000" + }, + "om": { + "lat": "21.0000", + "long": "57.0000" + }, + "pa": { + "lat": "9.0000", + "long": "-80.0000" + }, + "pe": { + "lat": "-10.0000", + "long": "-76.0000" + }, + "pf": { + "lat": "-15.0000", + "long": "-140.0000" + }, + "pg": { + "lat": "-6.0000", + "long": "147.0000" + }, + "ph": { + "lat": "13.0000", + "long": "122.0000" + }, + "pk": { + "lat": "30.0000", + "long": "70.0000" + }, + "pl": { + "lat": "52.0000", + "long": "20.0000" + }, + "pm": { + "lat": "46.8333", + "long": "-56.3333" + }, + "pr": { + "lat": "18.2500", + "long": "-66.5000" + }, + "ps": { + "lat": "32.0000", + "long": "35.2500" + }, + "pt": { + "lat": "39.5000", + "long": "-8.0000" + }, + "pw": { + "lat": "7.5000", + "long": "134.5000" + }, + "py": { + "lat": "-23.0000", + "long": "-58.0000" + }, + "qa": { + "lat": "25.5000", + "long": "51.2500" + }, + "re": { + "lat": "-21.1000", + "long": "55.6000" + }, + "ro": { + "lat": "46.0000", + "long": "25.0000" + }, + "rs": { + "lat": "44.0000", + "long": "21.0000" + }, + "ru": { + "lat": "60.0000", + "long": "100.0000" + }, + "rw": { + "lat": "-2.0000", + "long": "30.0000" + }, + "sa": { + "lat": "25.0000", + "long": "45.0000" + }, + "sb": { + "lat": "-8.0000", + "long": "159.0000" + }, + "sc": { + "lat": "-4.5833", + "long": "55.6667" + }, + "sd": { + "lat": "15.0000", + "long": "30.0000" + }, + "se": { + "lat": "62.0000", + "long": "15.0000" + }, + "sg": { + "lat": "1.3667", + "long": "103.8000" + }, + "sh": { + "lat": "-15.9333", + "long": "-5.7000" + }, + "si": { + "lat": "46.0000", + "long": "15.0000" + }, + "sj": { + "lat": "78.0000", + "long": "20.0000" + }, + "sk": { + "lat": "48.6667", + "long": "19.5000" + }, + "sl": { + "lat": "8.5000", + "long": "-11.5000" + }, + "sm": { + "lat": "43.7667", + "long": "12.4167" + }, + "sn": { + "lat": "14.0000", + "long": "-14.0000" + }, + "so": { + "lat": "10.0000", + "long": "49.0000" + }, + "sr": { + "lat": "4.0000", + "long": "-56.0000" + }, + "st": { + "lat": "1.0000", + "long": "7.0000" + }, + "sv": { + "lat": "13.8333", + "long": "-88.9167" + }, + "sy": { + "lat": "35.0000", + "long": "38.0000" + }, + "sz": { + "lat": "-26.5000", + "long": "31.5000" + }, + "tc": { + "lat": "21.7500", + "long": "-71.5833" + }, + "td": { + "lat": "15.0000", + "long": "19.0000" + }, + "tf": { + "lat": "-43.0000", + "long": "67.0000" + }, + "tg": { + "lat": "8.0000", + "long": "1.1667" + }, + "th": { + "lat": "15.0000", + "long": "100.0000" + }, + "tj": { + "lat": "39.0000", + "long": "71.0000" + }, + "tk": { + "lat": "-9.0000", + "long": "-172.0000" + }, + "tm": { + "lat": "40.0000", + "long": "60.0000" + }, + "tn": { + "lat": "34.0000", + "long": "9.0000" + }, + "to": { + "lat": "-20.0000", + "long": "-175.0000" + }, + "tr": { + "lat": "39.0000", + "long": "35.0000" + }, + "tt": { + "lat": "11.0000", + "long": "-61.0000" + }, + "tv": { + "lat": "-8.0000", + "long": "178.0000" + }, + "tw": { + "lat": "23.5000", + "long": "121.0000" + }, + "tz": { + "lat": "-6.0000", + "long": "35.0000" + }, + "ua": { + "lat": "49.0000", + "long": "32.0000" + }, + "ug": { + "lat": "1.0000", + "long": "32.0000" + }, + "um": { + "lat": "19.2833", + "long": "166.6000" + }, + "us": { + "lat": "38.0000", + "long": "-97.0000" + }, + "uy": { + "lat": "-33.0000", + "long": "-56.0000" + }, + "uz": { + "lat": "41.0000", + "long": "64.0000" + }, + "va": { + "lat": "41.9000", + "long": "12.4500" + }, + "vc": { + "lat": "13.2500", + "long": "-61.2000" + }, + "ve": { + "lat": "8.0000", + "long": "-66.0000" + }, + "vg": { + "lat": "18.5000", + "long": "-64.5000" + }, + "vi": { + "lat": "18.3333", + "long": "-64.8333" + }, + "vn": { + "lat": "16.0000", + "long": "106.0000" + }, + "vu": { + "lat": "-16.0000", + "long": "167.0000" + }, + "wf": { + "lat": "-13.3000", + "long": "-176.2000" + }, + "ws": { + "lat": "-13.5833", + "long": "-172.3333" + }, + "ye": { + "lat": "15.0000", + "long": "48.0000" + }, + "yt": { + "lat": "-12.8333", + "long": "45.1667" + }, + "za": { + "lat": "-29.0000", + "long": "24.0000" + }, + "zm": { + "lat": "-15.0000", + "long": "30.0000" + }, + "zw": { + "lat": "-20.0000", + "long": "30.0000" + } +} diff --git a/geo_helper.py b/geo_helper.py index e8ff2ab..ca1c83f 100644 --- a/geo_helper.py +++ b/geo_helper.py @@ -2,9 +2,13 @@ import math, random import os import json import datetime, time +import json import redis from collections import OrderedDict + import geoip2.database +import phonenumbers, pycountry +from phonenumbers import geocoder import util @@ -21,9 +25,13 @@ class Geo_helper: self.keyCategCountry = "GEO_COUNTRY" self.keyCategRad = "GEO_RAD" self.PATH_TO_DB = cfg.get('RedisMap', 'pathMaxMindDB') + self.PATH_TO_JSON = cfg.get('RedisMap', 'path_countrycode_to_coord_JSON') self.CHANNELDISP = cfg.get('RedisMap', 'channelDisp') self.reader = geoip2.database.Reader(self.PATH_TO_DB) + self.country_to_iso = { country.name: country.alpha_2 for country in pycountry.countries} + with open(self.PATH_TO_JSON) as f: + self.country_code_to_coord = json.load(f) ''' GET ''' def getTopCoord(self, date): @@ -69,7 +77,7 @@ class Geo_helper: return to_return ''' ADD ''' - def getCoordAndPublish(self, supposed_ip, categ): + def getCoordFromIpAndPublish(self, supposed_ip, categ): try: rep = self.ip_to_coord(supposed_ip) coord = rep['coord'] @@ -99,6 +107,43 @@ class Geo_helper: except geoip2.errors.AddressNotFoundError: print("Address not in Database") + def getCoordFromPhoneAndPublish(self, phoneNumber, categ): + try: + print('function accessed') + rep = phonenumbers.parse(phoneNumber, None) + if not (phonenumbers.is_valid_number(rep) or phonenumbers.is_possible_number(rep)): + print("Phone number not valid") + country_name = geocoder.country_name_for_number(rep, "en") + country_code = self.country_to_iso[country_name] + if country_code is None: + print("Non matching ISO_CODE") + coord = self.country_code_to_coord[country_code.lower()] # countrycode is in upper case + coord_dic = {'lat': coord['lat'], 'lon': coord['long']} + + ordDic = OrderedDict() #keep fields with the same layout in redis + ordDic['lat'] = coord_dic['lat'] + ordDic['lon'] = coord_dic['lon'] + coord_list = [coord['lat'], coord['long']] + self.push_to_redis_zset(self.keyCategCoord, json.dumps(ordDic)) + self.push_to_redis_zset(self.keyCategCountry, country_code) + ordDic = OrderedDict() #keep fields with the same layout in redis + ordDic['categ'] = categ + ordDic['value'] = phoneNumber + self.push_to_redis_geo(self.keyCategRad, coord['long'], coord['lat'], json.dumps(ordDic)) + to_send = { + "coord": coord_dic, + "categ": categ, + "value": phoneNumber, + "country": country_name, + "specifName": "", + "cityName": "", + "regionCode": country_code, + } + print(to_send) + self.serv_coord.publish(self.CHANNELDISP, json.dumps(to_send)) + except phonenumbers.NumberParseException: + print("Can't resolve phone number country") + ''' UTIL ''' def push_to_redis_geo(self, keyCateg, lon, lat, content): now = datetime.datetime.now() diff --git a/install_dependencies.sh b/install_dependencies.sh index 8677b05..420fade 100755 --- a/install_dependencies.sh +++ b/install_dependencies.sh @@ -14,7 +14,7 @@ if [ -z "$VIRTUAL_ENV" ]; then . ./DASHENV/bin/activate fi -pip3 install -U pip argparse redis zmq geoip2 flask +pip3 install -U pip argparse redis zmq geoip2 flask phonenumbers pycountry ## config cp -i config/config.cfg.default config/config.cfg diff --git a/zmq_subscriber.py b/zmq_subscriber.py index 81c6383..6e1514c 100755 --- a/zmq_subscriber.py +++ b/zmq_subscriber.py @@ -276,7 +276,11 @@ def handler_attribute(zmq_name, jsonobj, hasAlreadyBeenContributed=False): #try to get coord from ip if jsonattr['category'] == "Network activity": - geo_helper.getCoordAndPublish(jsonattr['value'], jsonattr['category']) + geo_helper.getCoordFromIpAndPublish(jsonattr['value'], jsonattr['category']) + + #try to get coord from ip + if jsonattr['type'] == "phone-number": + geo_helper.getCoordFromPhoneAndPublish(jsonattr['value'], jsonattr['category']) if not hasAlreadyBeenContributed: try: @@ -349,4 +353,3 @@ if __name__ == "__main__": args = parser.parse_args() main(args.zmqname) - reader.close()