From 2ff3eabc47ff20c55fbc941ad063f50937691e63 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Wed, 9 May 2018 11:25:50 +0200 Subject: [PATCH 1/6] fix: [GeoHelper] fix invalid coordinate In some cases, the coordinate given to redis may not match EPSG:900913 / EPSG:3785 / OSGEO:41001, throwing an error. This fix adds a check and skip the addition if it occurs --- helpers/geo_helper.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/helpers/geo_helper.py b/helpers/geo_helper.py index e1ab86f..f87dd25 100644 --- a/helpers/geo_helper.py +++ b/helpers/geo_helper.py @@ -13,6 +13,9 @@ from phonenumbers import geocoder import util +class InvalidCoordinate(Exception): + pass + class Geo_helper: def __init__(self, serv_redis_db, cfg): self.serv_redis_db = serv_redis_db @@ -98,6 +101,8 @@ class Geo_helper: ordDic['categ'] = categ ordDic['value'] = supposed_ip coord_list = [coord['lat'], coord['lon']] + if self.coordinate_list_valid(coord_list): + raise InvalidCoordinate("Coordinate do not match EPSG:900913 / EPSG:3785 / OSGEO:41001") self.push_to_redis_zset(self.keyCategCoord, json.dumps(ordDic)) self.push_to_redis_zset(self.keyCategCountry, rep['full_rep'].country.iso_code) ordDic = OrderedDict() #keep fields with the same layout in redis @@ -119,6 +124,9 @@ class Geo_helper: self.logger.warning("can't resolve ip") except geoip2.errors.AddressNotFoundError: self.logger.warning("Address not in Database") + except InvalidCoordinate: + self.logger.warning("Coordinate do not follow redis specification") + def getCoordFromPhoneAndPublish(self, phoneNumber, categ): try: @@ -138,6 +146,8 @@ class Geo_helper: ordDic['lat'] = coord_dic['lat'] ordDic['lon'] = coord_dic['lon'] coord_list = [coord['lat'], coord['long']] + if self.coordinate_list_valid(coord_list): + raise InvalidCoordinate("Coordinate do not match EPSG:900913 / EPSG:3785 / OSGEO:41001") self.push_to_redis_zset(self.keyCategCoord, json.dumps(ordDic)) self.push_to_redis_zset(self.keyCategCountry, country_code) ordDic = OrderedDict() #keep fields with the same layout in redis @@ -196,3 +206,14 @@ class Geo_helper: if abs(float(coord1[1]) - float(coord2[1])) <= clusterThres: return True return False + + # adjust latitude and longitude to fit the limit, as specified + # by EPSG:900913 / EPSG:3785 / OSGEO:41001 + # coord_list = [lat, lon] + def coordinate_list_valid(coord_list): + lat = coord_list[0] + lon = coord_list[1] + if (-180 <= lon <= 180) and (-85.05112878 <= lat <= 85.05112878): + return True + else: + return False From f2c91c14e651e918e8418fc79bb5b8e1a2a2f270 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Wed, 9 May 2018 11:57:31 +0200 Subject: [PATCH 2/6] fix: typo --- helpers/geo_helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/geo_helper.py b/helpers/geo_helper.py index f87dd25..9b8be6e 100644 --- a/helpers/geo_helper.py +++ b/helpers/geo_helper.py @@ -210,7 +210,7 @@ class Geo_helper: # adjust latitude and longitude to fit the limit, as specified # by EPSG:900913 / EPSG:3785 / OSGEO:41001 # coord_list = [lat, lon] - def coordinate_list_valid(coord_list): + def coordinate_list_valid(self, coord_list): lat = coord_list[0] lon = coord_list[1] if (-180 <= lon <= 180) and (-85.05112878 <= lat <= 85.05112878): From eafc61e058626cc3afba50704507e7c1362eca7c Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Wed, 9 May 2018 12:59:03 +0200 Subject: [PATCH 3/6] fix: solve comparing int and string --- helpers/geo_helper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helpers/geo_helper.py b/helpers/geo_helper.py index 9b8be6e..1850812 100644 --- a/helpers/geo_helper.py +++ b/helpers/geo_helper.py @@ -211,8 +211,8 @@ class Geo_helper: # by EPSG:900913 / EPSG:3785 / OSGEO:41001 # coord_list = [lat, lon] def coordinate_list_valid(self, coord_list): - lat = coord_list[0] - lon = coord_list[1] + lat = int(coord_list[0]) + lon = int(coord_list[1]) if (-180 <= lon <= 180) and (-85.05112878 <= lat <= 85.05112878): return True else: From 55649570bdf2f9d32b868fe99289794a61a75a2e Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Wed, 9 May 2018 13:04:42 +0200 Subject: [PATCH 4/6] fix: use float instead of int --- helpers/geo_helper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helpers/geo_helper.py b/helpers/geo_helper.py index 1850812..dbbb867 100644 --- a/helpers/geo_helper.py +++ b/helpers/geo_helper.py @@ -211,8 +211,8 @@ class Geo_helper: # by EPSG:900913 / EPSG:3785 / OSGEO:41001 # coord_list = [lat, lon] def coordinate_list_valid(self, coord_list): - lat = int(coord_list[0]) - lon = int(coord_list[1]) + lat = float(coord_list[0]) + lon = float(coord_list[1]) if (-180 <= lon <= 180) and (-85.05112878 <= lat <= 85.05112878): return True else: From 069a6d13b34c128cab434860a4c737178d8c8094 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Wed, 9 May 2018 13:12:14 +0200 Subject: [PATCH 5/6] fix: joinning list of int toghether The String's join function expects a list of string, sometimes this list can be a list of int. --- zmq_dispatcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zmq_dispatcher.py b/zmq_dispatcher.py index 0f1f137..e2a9bb7 100755 --- a/zmq_dispatcher.py +++ b/zmq_dispatcher.py @@ -203,7 +203,7 @@ def handler_attribute(zmq_name, jsonobj, hasAlreadyBeenContributed=False): if type(field) is list: to_join = [] for subField in field: - to_join.append(getFields(jsonobj, subField)) + to_join.append(str(getFields(jsonobj, subField))) to_add = cfg.get('Dashboard', 'char_separator').join(to_join) else: to_add = getFields(jsonobj, field) From c4c25557391d426aaf0fe6e9e0d3f58f98db69de Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Wed, 9 May 2018 13:31:49 +0200 Subject: [PATCH 6/6] fix: reverse matching condition --- helpers/geo_helper.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/helpers/geo_helper.py b/helpers/geo_helper.py index dbbb867..aa125ee 100644 --- a/helpers/geo_helper.py +++ b/helpers/geo_helper.py @@ -101,7 +101,7 @@ class Geo_helper: ordDic['categ'] = categ ordDic['value'] = supposed_ip coord_list = [coord['lat'], coord['lon']] - if self.coordinate_list_valid(coord_list): + if not self.coordinate_list_valid(coord_list): raise InvalidCoordinate("Coordinate do not match EPSG:900913 / EPSG:3785 / OSGEO:41001") self.push_to_redis_zset(self.keyCategCoord, json.dumps(ordDic)) self.push_to_redis_zset(self.keyCategCountry, rep['full_rep'].country.iso_code) @@ -146,7 +146,7 @@ class Geo_helper: ordDic['lat'] = coord_dic['lat'] ordDic['lon'] = coord_dic['lon'] coord_list = [coord['lat'], coord['long']] - if self.coordinate_list_valid(coord_list): + if not self.coordinate_list_valid(coord_list): raise InvalidCoordinate("Coordinate do not match EPSG:900913 / EPSG:3785 / OSGEO:41001") self.push_to_redis_zset(self.keyCategCoord, json.dumps(ordDic)) self.push_to_redis_zset(self.keyCategCountry, country_code) @@ -167,6 +167,8 @@ class Geo_helper: self.logger.info('Published: {}'.format(json.dumps(to_send))) except phonenumbers.NumberParseException: self.logger.warning("Can't resolve phone number country") + except InvalidCoordinate: + self.logger.warning("Coordinate do not follow redis specification") ''' UTIL ''' def push_to_redis_geo(self, keyCateg, lon, lat, content):