MISP
/
misp-dashboard
mirror of https://github.com/MISP/misp-dashboard
2
0
Fork 0
Code Issues Releases Wiki Activity

feature: Added geolocation of phone numbers

Browse Source
pull/11/head
Sami Mokaddem 5 years ago
parent 390e8775db
commit 90bdd45a18
5 changed files with 1015 additions and 4 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 1
      config/config.cfg.default
  2. 962
      data/country_code_lat_long.json
  3. 47
      geo_helper.py
  4. 2
      install_dependencies.sh
  5. 7
      zmq_subscriber.py

1
config/config.cfg.default
Unescape View File

@ -53,6 +53,7 @@ channelLastAwards = lastAwards
db=1
channelDisp=PicToDisplay
pathMaxMindDB=./data/GeoLite2-City_20171003/GeoLite2-City.mmdb
path_countrycode_to_coord_JSON=./data/country_code_lat_long.json
[RedisDB]
db=2

962
data/country_code_lat_long.json
Unescape View File

@ -0,0 +1,962 @@
{
"ad": {
"lat": "42.5000",
"long": "1.5000"
},
"ae": {
"lat": "24.0000",
"long": "54.0000"
},
"af": {
"lat": "33.0000",
"long": "65.0000"
},
"ag": {
"lat": "17.0500",
"long": "-61.8000"
},
"ai": {
"lat": "18.2500",
"long": "-63.1667"
},
"al": {
"lat": "41.0000",
"long": "20.0000"
},
"am": {
"lat": "40.0000",
"long": "45.0000"
},
"an": {
"lat": "12.2500",
"long": "-68.7500"
},
"ao": {
"lat": "-12.5000",
"long": "18.5000"
},
"ap": {
"lat": "35.0000",
"long": "105.0000"
},
"aq": {
"lat": "-90.0000",
"long": "0.0000"
},
"ar": {
"lat": "-34.0000",
"long": "-64.0000"
},
"as": {
"lat": "-14.3333",
"long": "-170.0000"
},
"at": {
"lat": "47.3333",
"long": "13.3333"
},
"au": {
"lat": "-27.0000",
"long": "133.0000"
},
"aw": {
"lat": "12.5000",
"long": "-69.9667"
},
"az": {
"lat": "40.5000",
"long": "47.5000"
},
"ba": {
"lat": "44.0000",
"long": "18.0000"
},
"bb": {
"lat": "13.1667",
"long": "-59.5333"
},
"bd": {
"lat": "24.0000",
"long": "90.0000"
},
"be": {
"lat": "50.8333",
"long": "4.0000"
},
"bf": {
"lat": "13.0000",
"long": "-2.0000"
},
"bg": {
"lat": "43.0000",
"long": "25.0000"
},
"bh": {
"lat": "26.0000",
"long": "50.5500"
},
"bi": {
"lat": "-3.5000",
"long": "30.0000"
},
"bj": {
"lat": "9.5000",
"long": "2.2500"
},
"bm": {
"lat": "32.3333",
"long": "-64.7500"
},
"bn": {
"lat": "4.5000",
"long": "114.6667"
},
"bo": {
"lat": "-17.0000",
"long": "-65.0000"
},
"br": {
"lat": "-10.0000",
"long": "-55.0000"
},
"bs": {
"lat": "24.2500",
"long": "-76.0000"
},
"bt": {
"lat": "27.5000",
"long": "90.5000"
},
"bv": {
"lat": "-54.4333",
"long": "3.4000"
},
"bw": {
"lat": "-22.0000",
"long": "24.0000"
},
"by": {
"lat": "53.0000",
"long": "28.0000"
},
"bz": {
"lat": "17.2500",
"long": "-88.7500"
},
"ca": {
"lat": "60.0000",
"long": "-95.0000"
},
"cc": {
"lat": "-12.5000",
"long": "96.8333"
},
"cd": {
"lat": "0.0000",
"long": "25.0000"
},
"cf": {
"lat": "7.0000",
"long": "21.0000"
},
"cg": {
"lat": "-1.0000",
"long": "15.0000"
},
"ch": {
"lat": "47.0000",
"long": "8.0000"
},
"ci": {
"lat": "8.0000",
"long": "-5.0000"
},
"ck": {
"lat": "-21.2333",
"long": "-159.7667"
},
"cl": {
"lat": "-30.0000",
"long": "-71.0000"
},
"cm": {
"lat": "6.0000",
"long": "12.0000"
},
"cn": {
"lat": "35.0000",
"long": "105.0000"
},
"co": {
"lat": "4.0000",
"long": "-72.0000"
},
"cr": {
"lat": "10.0000",
"long": "-84.0000"
},
"cu": {
"lat": "21.5000",
"long": "-80.0000"
},
"cv": {
"lat": "16.0000",
"long": "-24.0000"
},
"cx": {
"lat": "-10.5000",
"long": "105.6667"
},
"cy": {
"lat": "35.0000",
"long": "33.0000"
},
"cz": {
"lat": "49.7500",
"long": "15.5000"
},
"de": {
"lat": "51.0000",
"long": "9.0000"
},
"dj": {
"lat": "11.5000",
"long": "43.0000"
},
"dk": {
"lat": "56.0000",
"long": "10.0000"
},
"dm": {
"lat": "15.4167",
"long": "-61.3333"
},
"do": {
"lat": "19.0000",
"long": "-70.6667"
},
"dz": {
"lat": "28.0000",
"long": "3.0000"
},
"ec": {
"lat": "-2.0000",
"long": "-77.5000"
},
"ee": {
"lat": "59.0000",
"long": "26.0000"
},
"eg": {
"lat": "27.0000",
"long": "30.0000"
},
"eh": {
"lat": "24.5000",
"long": "-13.0000"
},
"er": {
"lat": "15.0000",
"long": "39.0000"
},
"es": {
"lat": "40.0000",
"long": "-4.0000"
},
"et": {
"lat": "8.0000",
"long": "38.0000"
},
"eu": {
"lat": "47.0000",
"long": "8.0000"
},
"fi": {
"lat": "64.0000",
"long": "26.0000"
},
"fj": {
"lat": "-18.0000",
"long": "175.0000"
},
"fk": {
"lat": "-51.7500",
"long": "-59.0000"
},
"fm": {
"lat": "6.9167",
"long": "158.2500"
},
"fo": {
"lat": "62.0000",
"long": "-7.0000"
},
"fr": {
"lat": "46.0000",
"long": "2.0000"
},
"ga": {
"lat": "-1.0000",
"long": "11.7500"
},
"gb": {
"lat": "54.0000",
"long": "-2.0000"
},
"gd": {
"lat": "12.1167",
"long": "-61.6667"
},
"ge": {
"lat": "42.0000",
"long": "43.5000"
},
"gf": {
"lat": "4.0000",
"long": "-53.0000"
},
"gh": {
"lat": "8.0000",
"long": "-2.0000"
},
"gi": {
"lat": "36.1833",
"long": "-5.3667"
},
"gl": {
"lat": "72.0000",
"long": "-40.0000"
},
"gm": {
"lat": "13.4667",
"long": "-16.5667"
},
"gn": {
"lat": "11.0000",
"long": "-10.0000"
},
"gp": {
"lat": "16.2500",
"long": "-61.5833"
},
"gq": {
"lat": "2.0000",
"long": "10.0000"
},
"gr": {
"lat": "39.0000",
"long": "22.0000"
},
"gs": {
"lat": "-54.5000",
"long": "-37.0000"
},
"gt": {
"lat": "15.5000",
"long": "-90.2500"
},
"gu": {
"lat": "13.4667",
"long": "144.7833"
},
"gw": {
"lat": "12.0000",
"long": "-15.0000"
},
"gy": {
"lat": "5.0000",
"long": "-59.0000"
},
"hk": {
"lat": "22.2500",
"long": "114.1667"
},
"hm": {
"lat": "-53.1000",
"long": "72.5167"
},
"hn": {
"lat": "15.0000",
"long": "-86.5000"
},
"hr": {
"lat": "45.1667",
"long": "15.5000"
},
"ht": {
"lat": "19.0000",
"long": "-72.4167"
},
"hu": {
"lat": "47.0000",
"long": "20.0000"
},
"id": {
"lat": "-5.0000",
"long": "120.0000"
},
"ie": {
"lat": "53.0000",
"long": "-8.0000"
},
"il": {
"lat": "31.5000",
"long": "34.7500"
},
"in": {
"lat": "20.0000",
"long": "77.0000"
},
"io": {
"lat": "-6.0000",
"long": "71.5000"
},
"iq": {
"lat": "33.0000",
"long": "44.0000"
},
"ir": {
"lat": "32.0000",
"long": "53.0000"
},
"is": {
"lat": "65.0000",
"long": "-18.0000"
},
"it": {
"lat": "42.8333",
"long": "12.8333"
},
"jm": {
"lat": "18.2500",
"long": "-77.5000"
},
"jo": {
"lat": "31.0000",
"long": "36.0000"
},
"jp": {
"lat": "36.0000",
"long": "138.0000"
},
"ke": {
"lat": "1.0000",
"long": "38.0000"
},
"kg": {
"lat": "41.0000",
"long": "75.0000"
},
"kh": {
"lat": "13.0000",
"long": "105.0000"
},
"ki": {
"lat": "1.4167",
"long": "173.0000"
},
"km": {
"lat": "-12.1667",
"long": "44.2500"
},
"kn": {
"lat": "17.3333",
"long": "-62.7500"
},
"kp": {
"lat": "40.0000",
"long": "127.0000"
},
"kr": {
"lat": "37.0000",
"long": "127.5000"
},
"kw": {
"lat": "29.3375",
"long": "47.6581"
},
"ky": {
"lat": "19.5000",
"long": "-80.5000"
},
"kz": {
"lat": "48.0000",
"long": "68.0000"
},
"la": {
"lat": "18.0000",
"long": "105.0000"
},
"lb": {
"lat": "33.8333",
"long": "35.8333"
},
"lc": {
"lat": "13.8833",
"long": "-61.1333"
},
"li": {
"lat": "47.1667",
"long": "9.5333"
},
"lk": {
"lat": "7.0000",
"long": "81.0000"
},
"lr": {
"lat": "6.5000",
"long": "-9.5000"
},
"ls": {
"lat": "-29.5000",
"long": "28.5000"
},
"lt": {
"lat": "56.0000",
"long": "24.0000"
},
"lu": {
"lat": "49.7500",
"long": "6.1667"
},
"lv": {
"lat": "57.0000",
"long": "25.0000"
},
"ly": {
"lat": "25.0000",
"long": "17.0000"
},
"ma": {
"lat": "32.0000",
"long": "-5.0000"
},
"mc": {
"lat": "43.7333",
"long": "7.4000"
},
"md": {
"lat": "47.0000",
"long": "29.0000"
},
"me": {
"lat": "42.0000",
"long": "19.0000"
},
"mg": {
"lat": "-20.0000",
"long": "47.0000"
},
"mh": {
"lat": "9.0000",
"long": "168.0000"
},
"mk": {
"lat": "41.8333",
"long": "22.0000"
},
"ml": {
"lat": "17.0000",
"long": "-4.0000"
},
"mm": {
"lat": "22.0000",
"long": "98.0000"
},
"mn": {
"lat": "46.0000",
"long": "105.0000"
},
"mo": {
"lat": "22.1667",
"long": "113.5500"
},
"mp": {
"lat": "15.2000",
"long": "145.7500"
},
"mq": {
"lat": "14.6667",
"long": "-61.0000"
},
"mr": {
"lat": "20.0000",
"long": "-12.0000"
},
"ms": {
"lat": "16.7500",
"long": "-62.2000"
},
"mt": {
"lat": "35.8333",
"long": "14.5833"
},
"mu": {
"lat": "-20.2833",
"long": "57.5500"
},
"mv": {
"lat": "3.2500",
"long": "73.0000"
},
"mw": {
"lat": "-13.5000",
"long": "34.0000"
},
"mx": {
"lat": "23.0000",
"long": "-102.0000"
},
"my": {
"lat": "2.5000",
"long": "112.5000"
},
"mz": {
"lat": "-18.2500",
"long": "35.0000"
},
"na": {
"lat": "-22.0000",
"long": "17.0000"
},
"nc": {
"lat": "-21.5000",
"long": "165.5000"
},
"ne": {
"lat": "16.0000",
"long": "8.0000"
},
"nf": {
"lat": "-29.0333",
"long": "167.9500"
},
"ng": {
"lat": "10.0000",
"long": "8.0000"
},
"ni": {
"lat": "13.0000",
"long": "-85.0000"
},
"nl": {
"lat": "52.5000",
"long": "5.7500"
},
"no": {
"lat": "62.0000",
"long": "10.0000"
},
"np": {
"lat": "28.0000",
"long": "84.0000"
},
"nr": {
"lat": "-0.5333",
"long": "166.9167"
},
"nu": {
"lat": "-19.0333",
"long": "-169.8667"
},
"nz": {
"lat": "-41.0000",
"long": "174.0000"
},
"om": {
"lat": "21.0000",
"long": "57.0000"
},
"pa": {
"lat": "9.0000",
"long": "-80.0000"
},
"pe": {
"lat": "-10.0000",
"long": "-76.0000"
},
"pf": {
"lat": "-15.0000",
"long": "-140.0000"
},
"pg": {
"lat": "-6.0000",
"long": "147.0000"
},
"ph": {
"lat": "13.0000",
"long": "122.0000"
},
"pk": {
"lat": "30.0000",
"long": "70.0000"
},
"pl": {
"lat": "52.0000",
"long": "20.0000"
},
"pm": {
"lat": "46.8333",
"long": "-56.3333"
},
"pr": {
"lat": "18.2500",
"long": "-66.5000"
},
"ps": {
"lat": "32.0000",
"long": "35.2500"
},
"pt": {
"lat": "39.5000",
"long": "-8.0000"
},
"pw": {
"lat": "7.5000",
"long": "134.5000"
},
"py": {
"lat": "-23.0000",
"long": "-58.0000"
},
"qa": {
"lat": "25.5000",
"long": "51.2500"
},
"re": {
"lat": "-21.1000",
"long": "55.6000"
},
"ro": {
"lat": "46.0000",
"long": "25.0000"
},
"rs": {
"lat": "44.0000",
"long": "21.0000"
},
"ru": {
"lat": "60.0000",
"long": "100.0000"
},
"rw": {
"lat": "-2.0000",
"long": "30.0000"
},
"sa": {
"lat": "25.0000",
"long": "45.0000"
},
"sb": {
"lat": "-8.0000",
"long": "159.0000"
},
"sc": {
"lat": "-4.5833",
"long": "55.6667"
},
"sd": {
"lat": "15.0000",
"long": "30.0000"
},
"se": {
"lat": "62.0000",
"long": "15.0000"
},
"sg": {
"lat": "1.3667",
"long": "103.8000"
},
"sh": {
"lat": "-15.9333",
"long": "-5.7000"
},
"si": {
"lat": "46.0000",
"long": "15.0000"
},
"sj": {
"lat": "78.0000",
"long": "20.0000"
},
"sk": {
"lat": "48.6667",
"long": "19.5000"
},
"sl": {
"lat": "8.5000",
"long": "-11.5000"
},
"sm": {
"lat": "43.7667",
"long": "12.4167"
},
"sn": {
"lat": "14.0000",
"long": "-14.0000"
},
"so": {
"lat": "10.0000",
"long": "49.0000"
},
"sr": {
"lat": "4.0000",
"long": "-56.0000"
},
"st": {
"lat": "1.0000",
"long": "7.0000"
},
"sv": {
"lat": "13.8333",
"long": "-88.9167"
},
"sy": {
"lat": "35.0000",
"long": "38.0000"
},
"sz": {
"lat": "-26.5000",
"long": "31.5000"
},
"tc": {
"lat": "21.7500",
"long": "-71.5833"
},
"td": {
"lat": "15.0000",
"long": "19.0000"
},
"tf": {
"lat": "-43.0000",
"long": "67.0000"
},
"tg": {
"lat": "8.0000",
"long": "1.1667"
},
"th": {
"lat": "15.0000",
"long": "100.0000"
},
"tj": {
"lat": "39.0000",
"long": "71.0000"
},
"tk": {
"lat": "-9.0000",
"long": "-172.0000"
},
"tm": {
"lat": "40.0000",
"long": "60.0000"
},
"tn": {
"lat": "34.0000",
"long": "9.0000"
},
"to": {
"lat": "-20.0000",
"long": "-175.0000"
},
"tr": {
"lat": "39.0000",
"long": "35.0000"
},
"tt": {
"lat": "11.0000",
"long": "-61.0000"
},
"tv": {
"lat": "-8.0000",
"long": "178.0000"
},
"tw": {
"lat": "23.5000",
"long": "121.0000"
},
"tz": {
"lat": "-6.0000",
"long": "35.0000"
},
"ua": {
"lat": "49.0000",
"long": "32.0000"
},
"ug": {
"lat": "1.0000",
"long": "32.0000"
},
"um": {
"lat": "19.2833",
"long": "166.6000"
},
"us": {
"lat": "38.0000",
"long": "-97.0000"
},
"uy": {
"lat": "-33.0000",
"long": "-56.0000"
},
"uz": {
"lat": "41.0000",
"long": "64.0000"
},
"va": {
"lat": "41.9000",
"long": "12.4500"
},
"vc": {
"lat": "13.2500",
"long": "-61.2000"
},
"ve": {
"lat": "8.0000",
"long": "-66.0000"
},
"vg": {
"lat": "18.5000",
"long": "-64.5000"
},
"vi": {
"lat": "18.3333",
"long": "-64.8333"
},
"vn": {
"lat": "16.0000",
"long": "106.0000"
},
"vu": {
"lat": "-16.0000",
"long": "167.0000"
},
"wf": {
"lat": "-13.3000",
"long": "-176.2000"
},
"ws": {
"lat": "-13.5833",
"long": "-172.3333"
},
"ye": {
"lat": "15.0000",
"long": "48.0000"
},
"yt": {
"lat": "-12.8333",
"long": "45.1667"
},
"za": {
"lat": "-29.0000",
"long": "24.0000"
},
"zm": {
"lat": "-15.0000",
"long": "30.0000"
},
"zw": {
"lat": "-20.0000",
"long": "30.0000"
}
}

47
geo_helper.py
Unescape View File

@ -2,9 +2,13 @@ import math, random
import os
import json
import datetime, time
import json
import redis
from collections import OrderedDict
import geoip2.database
import phonenumbers, pycountry
from phonenumbers import geocoder
import util
@ -21,9 +25,13 @@ class Geo_helper:
self.keyCategCountry = "GEO_COUNTRY"
self.keyCategRad = "GEO_RAD"
self.PATH_TO_DB = cfg.get('RedisMap', 'pathMaxMindDB')
self.PATH_TO_JSON = cfg.get('RedisMap', 'path_countrycode_to_coord_JSON')
self.CHANNELDISP = cfg.get('RedisMap', 'channelDisp')
self.reader = geoip2.database.Reader(self.PATH_TO_DB)
self.country_to_iso = { country.name: country.alpha_2 for country in pycountry.countries}
with open(self.PATH_TO_JSON) as f:
self.country_code_to_coord = json.load(f)
''' GET '''
def getTopCoord(self, date):
@ -69,7 +77,7 @@ class Geo_helper:
return to_return
''' ADD '''
def getCoordAndPublish(self, supposed_ip, categ):
def getCoordFromIpAndPublish(self, supposed_ip, categ):
try:
rep = self.ip_to_coord(supposed_ip)
coord = rep['coord']
@ -99,6 +107,43 @@ class Geo_helper:
except geoip2.errors.AddressNotFoundError:
print("Address not in Database")
def getCoordFromPhoneAndPublish(self, phoneNumber, categ):
try:
print('function accessed')
rep = phonenumbers.parse(phoneNumber, None)
if not (phonenumbers.is_valid_number(rep) or phonenumbers.is_possible_number(rep)):
print("Phone number not valid")
country_name = geocoder.country_name_for_number(rep, "en")
country_code = self.country_to_iso[country_name]
if country_code is None:
print("Non matching ISO_CODE")
coord = self.country_code_to_coord[country_code.lower()] # countrycode is in upper case
coord_dic = {'lat': coord['lat'], 'lon': coord['long']}
ordDic = OrderedDict() #keep fields with the same layout in redis
ordDic['lat'] = coord_dic['lat']
ordDic['lon'] = coord_dic['lon']
coord_list = [coord['lat'], coord['long']]
self.push_to_redis_zset(self.keyCategCoord, json.dumps(ordDic))
self.push_to_redis_zset(self.keyCategCountry, country_code)
ordDic = OrderedDict() #keep fields with the same layout in redis
ordDic['categ'] = categ
ordDic['value'] = phoneNumber
self.push_to_redis_geo(self.keyCategRad, coord['long'], coord['lat'], json.dumps(ordDic))
to_send = {
"coord": coord_dic,
"categ": categ,
"value": phoneNumber,
"country": country_name,
"specifName": "",
"cityName": "",
"regionCode": country_code,
}
print(to_send)
self.serv_coord.publish(self.CHANNELDISP, json.dumps(to_send))
except phonenumbers.NumberParseException:
print("Can't resolve phone number country")
''' UTIL '''
def push_to_redis_geo(self, keyCateg, lon, lat, content):
now = datetime.datetime.now()

2
install_dependencies.sh
Unescape View File

@ -14,7 +14,7 @@ if [ -z "$VIRTUAL_ENV" ]; then
. ./DASHENV/bin/activate
fi
pip3 install -U pip argparse redis zmq geoip2 flask
pip3 install -U pip argparse redis zmq geoip2 flask phonenumbers pycountry
## config
cp -i config/config.cfg.default config/config.cfg

7
zmq_subscriber.py
Unescape View File

@ -276,7 +276,11 @@ def handler_attribute(zmq_name, jsonobj, hasAlreadyBeenContributed=False):
#try to get coord from ip
if jsonattr['category'] == "Network activity":
geo_helper.getCoordAndPublish(jsonattr['value'], jsonattr['category'])
geo_helper.getCoordFromIpAndPublish(jsonattr['value'], jsonattr['category'])
#try to get coord from ip
if jsonattr['type'] == "phone-number":
geo_helper.getCoordFromPhoneAndPublish(jsonattr['value'], jsonattr['category'])
if not hasAlreadyBeenContributed:
try:
@ -349,4 +353,3 @@ if __name__ == "__main__":
args = parser.parse_args()
main(args.zmqname)
reader.close()

Write Preview
Loading…
Cancel
Save