From e7c6c360893be08db2b72deff739517021bd9c2b Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Fri, 12 Aug 2016 10:55:14 +0200 Subject: [PATCH 1/4] Fix: module_config should be set as introspection relies on it --- misp_modules/modules/export/testexport.py | 1 + 1 file changed, 1 insertion(+) diff --git a/misp_modules/modules/export/testexport.py b/misp_modules/modules/export/testexport.py index 14829cf..ed93228 100755 --- a/misp_modules/modules/export/testexport.py +++ b/misp_modules/modules/export/testexport.py @@ -9,6 +9,7 @@ userConfig = { }; +moduleconfig = [] # fixed for now, options in the future: # event, attribute, event-collection, attribute-collection From 23aedfb6ee6f267e0c0b431f48582f6df6dd2068 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 12 Aug 2016 11:31:23 +0200 Subject: [PATCH 2/4] Use entry_points instead of scripts. --- bin/misp-modules | 168 +------------------------------- misp_modules/__init__.py | 203 +++++++++++++++++++++++++++++++++++++++ setup.py | 2 +- 3 files changed, 206 insertions(+), 167 deletions(-) diff --git a/bin/misp-modules b/bin/misp-modules index 011d767..ad6ab33 100755 --- a/bin/misp-modules +++ b/bin/misp-modules @@ -19,172 +19,8 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import os import sys -import tornado.web -import importlib -import json -import logging -import fnmatch -import argparse -import re - -try: - import misp_modules.modules - HAS_PACKAGE_MODULES = True -except Exception as e: - print(e) - HAS_PACKAGE_MODULES = False - -try: - from misp_modules.helpers import * - HAS_PACKAGE_HELPERS = True -except Exception as e: - print(e) - HAS_PACKAGE_HELPERS = False - - -def init_logger(): - log = logging.getLogger('misp-modules') - formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') - handler = logging.StreamHandler(stream=sys.stdout) - handler.setFormatter(formatter) - handler.setLevel(logging.INFO) - - log.addHandler(handler) - log.setLevel(logging.INFO) - return log - - -def load_helpers(helpersdir): - sys.path.append(helpersdir) - hhandlers = {} - helpers = [] - for root, dirnames, filenames in os.walk(helpersdir): - if os.path.basename(root) == '__pycache__': - continue - if re.match(r'^\.', os.path.basename(root)): - continue - for filename in fnmatch.filter(filenames, '*.py'): - if filename == '__init__.py': - continue - helpername = filename.split(".")[0] - hhandlers[helpername] = importlib.import_module(helpername) - selftest = hhandlers[helpername].selftest() - if selftest is None: - helpers.append(helpername) - log.info('Helpers loaded {} '.format(filename)) - else: - log.info('Helpers failed {} due to {}'.format(filename, selftest)) - - -def load_package_helpers(): - if not HAS_PACKAGE_HELPERS: - log.info('Unable to load MISP helpers from package.') - sys.exit() - mhandlers = {} - helpers = [] - for path, helper in sys.modules.items(): - if not path.startswith('misp_modules.helpers.'): - continue - helpername = path.replace('misp_modules.helpers.', '') - mhandlers[helpername] = helper - helpers.append(helpername) - log.info('Helper loaded {}'.format(helpername)) - return mhandlers, helpers - - -def load_modules(mod_dir): - sys.path.append(mod_dir) - mhandlers = {} - modules = [] - for root, dirnames, filenames in os.walk(mod_dir): - if os.path.basename(root) == '__pycache__': - continue - if os.path.basename(root).startswith("."): - continue - for filename in fnmatch.filter(filenames, '*.py'): - if filename == '__init__.py': - continue - modulename = filename.split(".")[0] - moduletype = os.path.split(modulesdir)[1] - try: - mhandlers[modulename] = importlib.import_module(os.path.basename(root) + '.' + modulename) - except Exception as e: - log.warning('MISP modules {0} failed due to {1}'.format(modulename, e)) - continue - modules.append(modulename) - log.info('MISP modules {0} imported'.format(modulename)) - mhandlers['type:' + modulename] = moduletype - return mhandlers, modules - - -def load_package_modules(): - if not HAS_PACKAGE_MODULES: - log.info('Unable to load MISP modules from package.') - sys.exit() - mhandlers = {} - modules = [] - for path, module in sys.modules.items(): - r = re.findall("misp_modules[.]modules[.](\w+)[.](\w+)", path) - if r and len(r[0]) == 2: - moduletype, modulename = r[0] - mhandlers[modulename] = module - modules.append(modulename) - log.info('MISP modules {0} imported'.format(modulename)) - mhandlers['type:' + modulename] = moduletype - return mhandlers, modules - - -class ListModules(tornado.web.RequestHandler): - def get(self): - ret = [] - for module in modules: - x = {} - x['name'] = module - x['type'] = mhandlers['type:' + module] - x['mispattributes'] = mhandlers[module].introspection() - x['meta'] = mhandlers[module].version() - ret.append(x) - log.debug('MISP ListModules request') - self.write(json.dumps(ret)) - - -class QueryModule(tornado.web.RequestHandler): - def post(self): - jsonpayload = self.request.body.decode('utf-8') - x = json.loads(jsonpayload) - log.debug('MISP QueryModule request {0}'.format(jsonpayload)) - ret = mhandlers[x['module']].handler(q=jsonpayload) - self.write(json.dumps(ret)) - +from misp_modules import main if __name__ == '__main__': - if os.path.dirname(__file__) in ['.', '']: - os.chdir('../') - argParser = argparse.ArgumentParser(description='misp-modules server') - argParser.add_argument('-t', default=False, action='store_true', help='Test mode') - argParser.add_argument('-s', default=False, action='store_true', help='Run a system install (package installed via pip)') - argParser.add_argument('-p', default=6666, help='misp-modules TCP port (default 6666)') - argParser.add_argument('-l', default='localhost', help='misp-modules listen address (default localhost)') - args = argParser.parse_args() - port = args.p - listen = args.l - log = init_logger() - if args.s: - load_package_helpers() - mhandlers, modules = load_package_modules() - else: - modulesdir = 'misp_modules/modules' - helpersdir = 'misp_modules/helpers' - load_helpers(helpersdir=helpersdir) - mhandlers, modules = load_modules(modulesdir) - service = [(r'/modules', ListModules), (r'/query', QueryModule)] - - application = tornado.web.Application(service) - application.listen(port, address=listen) - log.info('MISP modules server started on {0} port {1}'.format(listen, port)) - if args.t: - log.info('MISP modules started in test-mode, quitting immediately.') - sys.exit() - tornado.ioloop.IOLoop.instance().start() + sys.exit(main()) diff --git a/misp_modules/__init__.py b/misp_modules/__init__.py index e69de29..19b44f4 100644 --- a/misp_modules/__init__.py +++ b/misp_modules/__init__.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Core MISP expansion modules loader and web service +# +# Copyright (C) 2016 Alexandre Dulaunoy +# Copyright (C) 2016 CIRCL - Computer Incident Response Center Luxembourg +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import os +import signal +import sys +from tornado.ioloop import IOLoop +import tornado.web +import importlib +import json +import logging +import fnmatch +import argparse +import re + +try: + from . import modules + HAS_PACKAGE_MODULES = True +except Exception as e: + print(e) + HAS_PACKAGE_MODULES = False + +try: + from .helpers import * + HAS_PACKAGE_HELPERS = True +except Exception as e: + print(e) + HAS_PACKAGE_HELPERS = False + +log = logging.getLogger('misp-modules') + +def handle_signal(sig, frame): + IOLoop.instance().add_callback(IOLoop.instance().stop) + + +def init_logger(): + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + handler = logging.StreamHandler(stream=sys.stdout) + handler.setFormatter(formatter) + handler.setLevel(logging.INFO) + + log.addHandler(handler) + log.setLevel(logging.INFO) + return log + + +def load_helpers(helpersdir): + sys.path.append(helpersdir) + hhandlers = {} + helpers = [] + for root, dirnames, filenames in os.walk(helpersdir): + if os.path.basename(root) == '__pycache__': + continue + if re.match(r'^\.', os.path.basename(root)): + continue + for filename in fnmatch.filter(filenames, '*.py'): + if filename == '__init__.py': + continue + helpername = filename.split(".")[0] + hhandlers[helpername] = importlib.import_module(helpername) + selftest = hhandlers[helpername].selftest() + if selftest is None: + helpers.append(helpername) + log.info('Helpers loaded {} '.format(filename)) + else: + log.info('Helpers failed {} due to {}'.format(filename, selftest)) + + +def load_package_helpers(): + if not HAS_PACKAGE_HELPERS: + log.info('Unable to load MISP helpers from package.') + sys.exit() + mhandlers = {} + helpers = [] + for path, helper in sys.modules.items(): + if not path.startswith('misp_modules.helpers.'): + continue + helpername = path.replace('misp_modules.helpers.', '') + mhandlers[helpername] = helper + helpers.append(helpername) + log.info('Helper loaded {}'.format(helpername)) + return mhandlers, helpers + + +def load_modules(mod_dir): + sys.path.append(mod_dir) + mhandlers = {} + modules = [] + for root, dirnames, filenames in os.walk(mod_dir): + if os.path.basename(root) == '__pycache__': + continue + if os.path.basename(root).startswith("."): + continue + for filename in fnmatch.filter(filenames, '*.py'): + if filename == '__init__.py': + continue + modulename = filename.split(".")[0] + moduletype = os.path.split(modulesdir)[1] + try: + mhandlers[modulename] = importlib.import_module(os.path.basename(root) + '.' + modulename) + except Exception as e: + log.warning('MISP modules {0} failed due to {1}'.format(modulename, e)) + continue + modules.append(modulename) + log.info('MISP modules {0} imported'.format(modulename)) + mhandlers['type:' + modulename] = moduletype + return mhandlers, modules + + +def load_package_modules(): + if not HAS_PACKAGE_MODULES: + log.info('Unable to load MISP modules from package.') + sys.exit() + mhandlers = {} + modules = [] + for path, module in sys.modules.items(): + r = re.findall("misp_modules[.]modules[.](\w+)[.](\w+)", path) + if r and len(r[0]) == 2: + moduletype, modulename = r[0] + mhandlers[modulename] = module + modules.append(modulename) + log.info('MISP modules {0} imported'.format(modulename)) + mhandlers['type:' + modulename] = moduletype + return mhandlers, modules + + +class ListModules(tornado.web.RequestHandler): + def get(self): + ret = [] + for module in modules: + x = {} + x['name'] = module + x['type'] = mhandlers['type:' + module] + x['mispattributes'] = mhandlers[module].introspection() + x['meta'] = mhandlers[module].version() + ret.append(x) + log.debug('MISP ListModules request') + self.write(json.dumps(ret)) + + +class QueryModule(tornado.web.RequestHandler): + def post(self): + jsonpayload = self.request.body.decode('utf-8') + x = json.loads(jsonpayload) + log.debug('MISP QueryModule request {0}'.format(jsonpayload)) + ret = mhandlers[x['module']].handler(q=jsonpayload) + self.write(json.dumps(ret)) + +def main(): + signal.signal(signal.SIGINT, handle_signal) + signal.signal(signal.SIGTERM, handle_signal) + if os.path.dirname(__file__) in ['.', '']: + os.chdir('../') + argParser = argparse.ArgumentParser(description='misp-modules server') + argParser.add_argument('-t', default=False, action='store_true', help='Test mode') + argParser.add_argument('-s', default=False, action='store_true', help='Run a system install (package installed via pip)') + argParser.add_argument('-p', default=6666, help='misp-modules TCP port (default 6666)') + argParser.add_argument('-l', default='localhost', help='misp-modules listen address (default localhost)') + args = argParser.parse_args() + port = args.p + listen = args.l + log = init_logger() + if args.s: + load_package_helpers() + mhandlers, modules = load_package_modules() + else: + modulesdir = 'misp_modules/modules' + helpersdir = 'misp_modules/helpers' + load_helpers(helpersdir=helpersdir) + mhandlers, modules = load_modules(modulesdir) + service = [(r'/modules', ListModules), (r'/query', QueryModule)] + + application = tornado.web.Application(service) + application.listen(port, address=listen) + log.info('MISP modules server started on {0} port {1}'.format(listen, port)) + if args.t: + log.info('MISP modules started in test-mode, quitting immediately.') + sys.exit() + IOLoop.instance().start() + IOLoop.instance().stop() + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/setup.py b/setup.py index 1628cba..7d96ba5 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ setup( url='https://github.com/MISP/misp-modules', description='MISP modules are autonomous modules that can be used for expansion and other services in MISP', packages=find_packages(), - scripts=['bin/misp-modules'], + entry_points = {'console_scripts': ['misp-modules = misp_modules:main']}, test_suite="tests", classifiers=[ 'License :: OSI Approved :: GNU Affero General Public License v3', From 59b16950f708a01ebf7c9d6ab56741c7167f41ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 12 Aug 2016 12:35:33 +0200 Subject: [PATCH 3/4] Remove bin script, use cleaner way. Fix last commit. --- REQUIREMENTS | 2 ++ bin/misp-modules | 26 --------------- misp_modules/__init__.py | 32 ++++++++++++------- misp_modules/modules/__init__.py | 2 ++ misp_modules/modules/expansion/__init__.py | 2 +- misp_modules/modules/expansion/whois.py | 5 ++- misp_modules/modules/export_mod/__init__.py | 1 + .../{export => export_mod}/testexport.py | 0 misp_modules/modules/import_mod/__init__.py | 1 + .../modules/{import => import_mod}/ocr.py | 0 .../{import => import_mod}/testimport.py | 0 setup.py | 2 ++ 12 files changed, 34 insertions(+), 39 deletions(-) delete mode 100755 bin/misp-modules create mode 100644 misp_modules/modules/export_mod/__init__.py rename misp_modules/modules/{export => export_mod}/testexport.py (100%) create mode 100644 misp_modules/modules/import_mod/__init__.py rename misp_modules/modules/{import => import_mod}/ocr.py (100%) rename misp_modules/modules/{import => import_mod}/testimport.py (100%) diff --git a/REQUIREMENTS b/REQUIREMENTS index 7656078..482ad1c 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -10,3 +10,5 @@ pyeupi ipasn-redis asnhistory git+https://github.com/Rafiot/uwhoisd.git@testing#egg=uwhois&subdirectory=client +pillow +pytesseract diff --git a/bin/misp-modules b/bin/misp-modules deleted file mode 100755 index ad6ab33..0000000 --- a/bin/misp-modules +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# -# Core MISP expansion modules loader and web service -# -# Copyright (C) 2016 Alexandre Dulaunoy -# Copyright (C) 2016 CIRCL - Computer Incident Response Center Luxembourg -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -import sys -from misp_modules import main - -if __name__ == '__main__': - sys.exit(main()) diff --git a/misp_modules/__init__.py b/misp_modules/__init__.py index 19b44f4..2617fae 100644 --- a/misp_modules/__init__.py +++ b/misp_modules/__init__.py @@ -32,7 +32,7 @@ import argparse import re try: - from . import modules + from .modules import * HAS_PACKAGE_MODULES = True except Exception as e: print(e) @@ -47,6 +47,7 @@ except Exception as e: log = logging.getLogger('misp-modules') + def handle_signal(sig, frame): IOLoop.instance().add_callback(IOLoop.instance().stop) @@ -95,8 +96,12 @@ def load_package_helpers(): continue helpername = path.replace('misp_modules.helpers.', '') mhandlers[helpername] = helper - helpers.append(helpername) - log.info('Helper loaded {}'.format(helpername)) + selftest = mhandlers[helpername].selftest() + if selftest is None: + helpers.append(helpername) + log.info('Helper loaded {}'.format(helpername)) + else: + log.info('Helpers failed {} due to {}'.format(helpername, selftest)) return mhandlers, helpers @@ -113,7 +118,7 @@ def load_modules(mod_dir): if filename == '__init__.py': continue modulename = filename.split(".")[0] - moduletype = os.path.split(modulesdir)[1] + moduletype = os.path.split(mod_dir)[1] try: mhandlers[modulename] = importlib.import_module(os.path.basename(root) + '.' + modulename) except Exception as e: @@ -144,8 +149,10 @@ def load_package_modules(): class ListModules(tornado.web.RequestHandler): def get(self): + global mhandlers + global loaded_modules ret = [] - for module in modules: + for module in loaded_modules: x = {} x['name'] = module x['type'] = mhandlers['type:' + module] @@ -158,17 +165,19 @@ class ListModules(tornado.web.RequestHandler): class QueryModule(tornado.web.RequestHandler): def post(self): + global mhandlers jsonpayload = self.request.body.decode('utf-8') x = json.loads(jsonpayload) log.debug('MISP QueryModule request {0}'.format(jsonpayload)) ret = mhandlers[x['module']].handler(q=jsonpayload) self.write(json.dumps(ret)) + def main(): + global mhandlers + global loaded_modules signal.signal(signal.SIGINT, handle_signal) signal.signal(signal.SIGTERM, handle_signal) - if os.path.dirname(__file__) in ['.', '']: - os.chdir('../') argParser = argparse.ArgumentParser(description='misp-modules server') argParser.add_argument('-t', default=False, action='store_true', help='Test mode') argParser.add_argument('-s', default=False, action='store_true', help='Run a system install (package installed via pip)') @@ -180,12 +189,13 @@ def main(): log = init_logger() if args.s: load_package_helpers() - mhandlers, modules = load_package_modules() + mhandlers, loaded_modules = load_package_modules() else: - modulesdir = 'misp_modules/modules' - helpersdir = 'misp_modules/helpers' + os.chdir(os.path.dirname(__file__)) + modulesdir = 'modules' + helpersdir = 'helpers' load_helpers(helpersdir=helpersdir) - mhandlers, modules = load_modules(modulesdir) + mhandlers, loaded_modules = load_modules(modulesdir) service = [(r'/modules', ListModules), (r'/query', QueryModule)] application = tornado.web.Application(service) diff --git a/misp_modules/modules/__init__.py b/misp_modules/modules/__init__.py index f0f4cdb..65ce6b2 100644 --- a/misp_modules/modules/__init__.py +++ b/misp_modules/modules/__init__.py @@ -1 +1,3 @@ from .expansion import * +from .import_mod import * +from .export_mod import * diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py index 5854e40..79fabff 100644 --- a/misp_modules/modules/expansion/__init__.py +++ b/misp_modules/modules/expansion/__init__.py @@ -1,2 +1,2 @@ __all__ = ['asn_history', 'circl_passivedns', 'circl_passivessl', 'cve', 'dns', - 'eupi', 'ipasn', 'passivetotal', 'sourcecache'] + 'eupi', 'ipasn', 'passivetotal', 'sourcecache', 'whois'] diff --git a/misp_modules/modules/expansion/whois.py b/misp_modules/modules/expansion/whois.py index 5f3602e..4aec40c 100755 --- a/misp_modules/modules/expansion/whois.py +++ b/misp_modules/modules/expansion/whois.py @@ -1,7 +1,10 @@ # -*- coding: utf-8 -*- import json -from uwhois import Uwhois +try: + from uwhois import Uwhois +except ImportError: + print("uwhois module not installed.") misperrors = {'error': 'Error'} mispattributes = {'input': ['domain', 'ip-src', 'ip-dst'], 'output': ['freetext']} diff --git a/misp_modules/modules/export_mod/__init__.py b/misp_modules/modules/export_mod/__init__.py new file mode 100644 index 0000000..35cc7cb --- /dev/null +++ b/misp_modules/modules/export_mod/__init__.py @@ -0,0 +1 @@ +__all__ = ['testexport'] diff --git a/misp_modules/modules/export/testexport.py b/misp_modules/modules/export_mod/testexport.py similarity index 100% rename from misp_modules/modules/export/testexport.py rename to misp_modules/modules/export_mod/testexport.py diff --git a/misp_modules/modules/import_mod/__init__.py b/misp_modules/modules/import_mod/__init__.py new file mode 100644 index 0000000..5716751 --- /dev/null +++ b/misp_modules/modules/import_mod/__init__.py @@ -0,0 +1 @@ +__all__ = ['testimport', 'ocr'] diff --git a/misp_modules/modules/import/ocr.py b/misp_modules/modules/import_mod/ocr.py similarity index 100% rename from misp_modules/modules/import/ocr.py rename to misp_modules/modules/import_mod/ocr.py diff --git a/misp_modules/modules/import/testimport.py b/misp_modules/modules/import_mod/testimport.py similarity index 100% rename from misp_modules/modules/import/testimport.py rename to misp_modules/modules/import_mod/testimport.py diff --git a/setup.py b/setup.py index 7d96ba5..ed92f56 100644 --- a/setup.py +++ b/setup.py @@ -33,5 +33,7 @@ setup( 'pyeupi', 'ipasn-redis', 'asnhistory', + 'pillow', + 'pytesseract', ] ) From bfedd69a817a05ddb4e3ea6dd0254e907eb67098 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Fri, 12 Aug 2016 12:39:46 +0200 Subject: [PATCH 4/4] pip --upgrade must be always called (to have modules updated) --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3d4ad2a..6a4b01a 100644 --- a/README.md +++ b/README.md @@ -30,8 +30,8 @@ sudo apt-get install python3-dev python3-pip libpq5 cd /usr/local/src/ sudo git clone https://github.com/MISP/misp-modules.git cd misp-modules -sudo pip3 install -r REQUIREMENTS -sudo pip3 install . +sudo pip3 install --upgrade -r REQUIREMENTS +sudo pip3 install --upgrade . sudo vi /etc/rc.local, add this line: `sudo -u www-data misp-modules -s` ~~~~