From 4bc76acd37b6403bd45c4039ef28847c5bbbdb5c Mon Sep 17 00:00:00 2001 From: Andreas Muehlemann Date: Wed, 28 Sep 2016 14:05:43 +0200 Subject: [PATCH 01/12] added geoip_country.py --- .../modules/expansion/geoip_country.py | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 misp_modules/modules/expansion/geoip_country.py diff --git a/misp_modules/modules/expansion/geoip_country.py b/misp_modules/modules/expansion/geoip_country.py new file mode 100644 index 0000000..01e95de --- /dev/null +++ b/misp_modules/modules/expansion/geoip_country.py @@ -0,0 +1,66 @@ +import json, pygeoip +import sys, logging + +log = logging.getLogger('geoip_country') +log.setLevel(logging.DEBUG) +ch = logging.StreamHandler(sys.stdout) +ch.setLevel(logging.DEBUG) +formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +ch.setFormatter(formatter) +log.addHandler(ch) + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['ip-src', 'ip-dst', 'domain|ip'], 'output': ['freetext']} + +# possible module-types: 'expansion', 'hover' or both +moduleinfo = {'version': '0.1', 'author': 'Andreas Muehlemann', + 'description': 'Query a local copy of Maxminds Geolite database', + 'module-type': ['expansion', 'hover']} + +# config fields that your code expects from the site admin +moduleconfig = ['database'] + +# get current db from http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz +gi = pygeoip.GeoIP('/opt/misp-modules/var/GeoIP.dat') + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + + log.debug("BLA") + + if request.get('ip-dst'): + toquery = request['ip-dst'] + elif request.get('ip-src'): + toquery = request['ip-src'] + elif request.get('domain|ip'): + toquery = request['domain|ip'].split('|')[1] + else: + return false + + log.debug(toquery) + + #if request.get('config'): + # if request['config'].get('database'): + # gi = pygeoip.GeoIP(request['config'].get('database')) + #else: + # gi = pygeoip.GeoIP('/opt/misp-modules/var/GeoIP.dat') + + try: + answer = gi.country_code_by_addr(toquery) + except: + misperrors['error'] = "GeoIP resolving error" + return misperrors + + r = {'results': [{'types': mispattributes['output'], + 'values': [str(answer)]}]} + + return r + +def introspection(): + return mispattributes + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo From a568d1a1b3aa992256b08b5e68c092c2e49080f3 Mon Sep 17 00:00:00 2001 From: Andreas Muehlemann Date: Wed, 28 Sep 2016 14:06:18 +0200 Subject: [PATCH 02/12] updated geoip_country to __init__.py --- misp_modules/modules/expansion/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py index e3b8d67..8ed3f35 100644 --- a/misp_modules/modules/expansion/__init__.py +++ b/misp_modules/modules/expansion/__init__.py @@ -1,2 +1,3 @@ __all__ = ['asn_history', 'circl_passivedns', 'circl_passivessl', 'countrycode', 'cve', 'dns', - 'eupi', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal', 'whois', 'shodan', 'reversedns'] + 'eupi', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal', 'whois', 'shodan', 'reversedns' + 'geoip_country'] From 42fc574f85217dcfe724f28631a70e2eb46915a1 Mon Sep 17 00:00:00 2001 From: Andreas Muehlemann Date: Wed, 28 Sep 2016 17:00:11 +0200 Subject: [PATCH 03/12] added pygeoip to the REQUIREMENTS list --- REQUIREMENTS | 1 + 1 file changed, 1 insertion(+) diff --git a/REQUIREMENTS b/REQUIREMENTS index f138191..fd7f901 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -14,3 +14,4 @@ asnhistory git+https://github.com/Rafiot/uwhoisd.git@testing#egg=uwhois&subdirectory=client pillow pytesseract +pygeoip From f8c7271467d84159743d02b1b699fb41bea7e079 Mon Sep 17 00:00:00 2001 From: Andreas Muehlemann Date: Wed, 7 Dec 2016 14:18:21 +0100 Subject: [PATCH 04/12] added config option to geoip_country.py --- misp_modules/modules/expansion/geoip_country.cfg | 2 ++ misp_modules/modules/expansion/geoip_country.py | 13 +++++-------- 2 files changed, 7 insertions(+), 8 deletions(-) create mode 100644 misp_modules/modules/expansion/geoip_country.cfg diff --git a/misp_modules/modules/expansion/geoip_country.cfg b/misp_modules/modules/expansion/geoip_country.cfg new file mode 100644 index 0000000..ab12f90 --- /dev/null +++ b/misp_modules/modules/expansion/geoip_country.cfg @@ -0,0 +1,2 @@ +[defaults] +database=/opt/misp-modules/var/GeoIP.dat diff --git a/misp_modules/modules/expansion/geoip_country.py b/misp_modules/modules/expansion/geoip_country.py index 01e95de..069f20a 100644 --- a/misp_modules/modules/expansion/geoip_country.py +++ b/misp_modules/modules/expansion/geoip_country.py @@ -1,5 +1,6 @@ import json, pygeoip import sys, logging +import ConfigParser log = logging.getLogger('geoip_country') log.setLevel(logging.DEBUG) @@ -21,7 +22,9 @@ moduleinfo = {'version': '0.1', 'author': 'Andreas Muehlemann', moduleconfig = ['database'] # get current db from http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz -gi = pygeoip.GeoIP('/opt/misp-modules/var/GeoIP.dat') +config = ConfigParser.ConfigParser() +config.read('./geoip_contry.cfg') +gi = pygeoip.GeoIP(config.get('defaults', 'database') def handler(q=False): if q is False: @@ -41,12 +44,6 @@ def handler(q=False): log.debug(toquery) - #if request.get('config'): - # if request['config'].get('database'): - # gi = pygeoip.GeoIP(request['config'].get('database')) - #else: - # gi = pygeoip.GeoIP('/opt/misp-modules/var/GeoIP.dat') - try: answer = gi.country_code_by_addr(toquery) except: @@ -55,7 +52,7 @@ def handler(q=False): r = {'results': [{'types': mispattributes['output'], 'values': [str(answer)]}]} - + return r def introspection(): From 6ea7acc5e4983841e87b00345c03be4585a5b632 Mon Sep 17 00:00:00 2001 From: Andreas Muehlemann Date: Wed, 7 Dec 2016 14:28:27 +0100 Subject: [PATCH 05/12] removed debug message --- misp_modules/modules/expansion/geoip_country.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/geoip_country.py b/misp_modules/modules/expansion/geoip_country.py index 069f20a..85dd2d8 100644 --- a/misp_modules/modules/expansion/geoip_country.py +++ b/misp_modules/modules/expansion/geoip_country.py @@ -31,8 +31,6 @@ def handler(q=False): return False request = json.loads(q) - log.debug("BLA") - if request.get('ip-dst'): toquery = request['ip-dst'] elif request.get('ip-src'): @@ -61,3 +59,4 @@ def introspection(): def version(): moduleinfo['config'] = moduleconfig return moduleinfo + From d09c2f3d44d12025b0e8314e75c3cee63267fc0e Mon Sep 17 00:00:00 2001 From: Andreas Muehlemann Date: Wed, 7 Dec 2016 14:29:11 +0100 Subject: [PATCH 06/12] removed unneeded config option for misp --- misp_modules/modules/expansion/geoip_country.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/misp_modules/modules/expansion/geoip_country.py b/misp_modules/modules/expansion/geoip_country.py index 85dd2d8..ae9ccc9 100644 --- a/misp_modules/modules/expansion/geoip_country.py +++ b/misp_modules/modules/expansion/geoip_country.py @@ -18,9 +18,6 @@ moduleinfo = {'version': '0.1', 'author': 'Andreas Muehlemann', 'description': 'Query a local copy of Maxminds Geolite database', 'module-type': ['expansion', 'hover']} -# config fields that your code expects from the site admin -moduleconfig = ['database'] - # get current db from http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz config = ConfigParser.ConfigParser() config.read('./geoip_contry.cfg') @@ -59,4 +56,3 @@ def introspection(): def version(): moduleinfo['config'] = moduleconfig return moduleinfo - From 1e1796b41411c8e17088095fdb457533b364831f Mon Sep 17 00:00:00 2001 From: Andreas Muehlemann Date: Wed, 7 Dec 2016 15:19:54 +0100 Subject: [PATCH 07/12] updated missing parenthesis --- misp_modules/modules/expansion/geoip_country.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/geoip_country.py b/misp_modules/modules/expansion/geoip_country.py index ae9ccc9..4ffc0b9 100644 --- a/misp_modules/modules/expansion/geoip_country.py +++ b/misp_modules/modules/expansion/geoip_country.py @@ -21,7 +21,7 @@ moduleinfo = {'version': '0.1', 'author': 'Andreas Muehlemann', # get current db from http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz config = ConfigParser.ConfigParser() config.read('./geoip_contry.cfg') -gi = pygeoip.GeoIP(config.get('defaults', 'database') +gi = pygeoip.GeoIP(config.get('defaults', 'database')) def handler(q=False): if q is False: From a95af2642455f3d99b055c43b5a143877d32ca05 Mon Sep 17 00:00:00 2001 From: Andreas Muehlemann Date: Wed, 7 Dec 2016 15:30:49 +0100 Subject: [PATCH 08/12] changed configparser from python2 to python3 --- misp_modules/modules/expansion/geoip_country.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/geoip_country.py b/misp_modules/modules/expansion/geoip_country.py index 4ffc0b9..634d4c7 100644 --- a/misp_modules/modules/expansion/geoip_country.py +++ b/misp_modules/modules/expansion/geoip_country.py @@ -1,6 +1,6 @@ import json, pygeoip import sys, logging -import ConfigParser +import configparser log = logging.getLogger('geoip_country') log.setLevel(logging.DEBUG) @@ -19,7 +19,7 @@ moduleinfo = {'version': '0.1', 'author': 'Andreas Muehlemann', 'module-type': ['expansion', 'hover']} # get current db from http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz -config = ConfigParser.ConfigParser() +config = configparser.ConfigParser() config.read('./geoip_contry.cfg') gi = pygeoip.GeoIP(config.get('defaults', 'database')) From 6dcc77ba5d247b1aed1bf22e3866b8bb94a9b37a Mon Sep 17 00:00:00 2001 From: Andreas Muehlemann Date: Wed, 7 Dec 2016 15:48:08 +0100 Subject: [PATCH 09/12] fixed typo --- misp_modules/modules/expansion/geoip_country.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/geoip_country.py b/misp_modules/modules/expansion/geoip_country.py index 634d4c7..a04d739 100644 --- a/misp_modules/modules/expansion/geoip_country.py +++ b/misp_modules/modules/expansion/geoip_country.py @@ -20,7 +20,7 @@ moduleinfo = {'version': '0.1', 'author': 'Andreas Muehlemann', # get current db from http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz config = configparser.ConfigParser() -config.read('./geoip_contry.cfg') +config.read('geoip_country.cfg') gi = pygeoip.GeoIP(config.get('defaults', 'database')) def handler(q=False): From 6853d67a43e4aa04ca51dc6e2e57b6e3b8654db5 Mon Sep 17 00:00:00 2001 From: Andreas Muehlemann Date: Wed, 7 Dec 2016 16:13:46 +0100 Subject: [PATCH 10/12] fixed more typos --- misp_modules/modules/expansion/geoip_country.cfg | 2 +- misp_modules/modules/expansion/geoip_country.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/geoip_country.cfg b/misp_modules/modules/expansion/geoip_country.cfg index ab12f90..bcd1c85 100644 --- a/misp_modules/modules/expansion/geoip_country.cfg +++ b/misp_modules/modules/expansion/geoip_country.cfg @@ -1,2 +1,2 @@ -[defaults] +[DEFAULT] database=/opt/misp-modules/var/GeoIP.dat diff --git a/misp_modules/modules/expansion/geoip_country.py b/misp_modules/modules/expansion/geoip_country.py index a04d739..cac709c 100644 --- a/misp_modules/modules/expansion/geoip_country.py +++ b/misp_modules/modules/expansion/geoip_country.py @@ -21,7 +21,7 @@ moduleinfo = {'version': '0.1', 'author': 'Andreas Muehlemann', # get current db from http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz config = configparser.ConfigParser() config.read('geoip_country.cfg') -gi = pygeoip.GeoIP(config.get('defaults', 'database')) +gi = pygeoip.GeoIP(config.get('DEFAULT', 'database')) def handler(q=False): if q is False: From 98a27ac3ff9743e17163d65f8fd86dc9ff5dfcdc Mon Sep 17 00:00:00 2001 From: Andreas Muehlemann Date: Wed, 7 Dec 2016 16:36:02 +0100 Subject: [PATCH 11/12] removed DEFAULT section from configfile --- misp_modules/modules/expansion/geoip_country.cfg | 4 ++-- misp_modules/modules/expansion/geoip_country.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/misp_modules/modules/expansion/geoip_country.cfg b/misp_modules/modules/expansion/geoip_country.cfg index bcd1c85..f27e7d9 100644 --- a/misp_modules/modules/expansion/geoip_country.cfg +++ b/misp_modules/modules/expansion/geoip_country.cfg @@ -1,2 +1,2 @@ -[DEFAULT] -database=/opt/misp-modules/var/GeoIP.dat +[GEOIP] +database: /opt/misp-modules/var/GeoIP.dat diff --git a/misp_modules/modules/expansion/geoip_country.py b/misp_modules/modules/expansion/geoip_country.py index cac709c..f5a1984 100644 --- a/misp_modules/modules/expansion/geoip_country.py +++ b/misp_modules/modules/expansion/geoip_country.py @@ -21,7 +21,7 @@ moduleinfo = {'version': '0.1', 'author': 'Andreas Muehlemann', # get current db from http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz config = configparser.ConfigParser() config.read('geoip_country.cfg') -gi = pygeoip.GeoIP(config.get('DEFAULT', 'database')) +gi = pygeoip.GeoIP(config.get('GEOIP', 'database')) def handler(q=False): if q is False: From cc58b05d6e1c24468a0770cfccb7a789053163d8 Mon Sep 17 00:00:00 2001 From: Andreas Muehlemann Date: Wed, 7 Dec 2016 17:28:16 +0100 Subject: [PATCH 12/12] added empty line to end of config file --- misp_modules/modules/expansion/geoip_country.cfg | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/geoip_country.cfg b/misp_modules/modules/expansion/geoip_country.cfg index f27e7d9..95037e5 100644 --- a/misp_modules/modules/expansion/geoip_country.cfg +++ b/misp_modules/modules/expansion/geoip_country.cfg @@ -1,2 +1,3 @@ [GEOIP] -database: /opt/misp-modules/var/GeoIP.dat +database = /opt/misp-modules/var/GeoIP.dat +