diff --git a/REQUIREMENTS b/REQUIREMENTS index 7656078..482ad1c 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -10,3 +10,5 @@ pyeupi ipasn-redis asnhistory git+https://github.com/Rafiot/uwhoisd.git@testing#egg=uwhois&subdirectory=client +pillow +pytesseract diff --git a/bin/misp-modules b/bin/misp-modules deleted file mode 100755 index ad6ab33..0000000 --- a/bin/misp-modules +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# -# Core MISP expansion modules loader and web service -# -# Copyright (C) 2016 Alexandre Dulaunoy -# Copyright (C) 2016 CIRCL - Computer Incident Response Center Luxembourg -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -import sys -from misp_modules import main - -if __name__ == '__main__': - sys.exit(main()) diff --git a/misp_modules/__init__.py b/misp_modules/__init__.py index 19b44f4..2617fae 100644 --- a/misp_modules/__init__.py +++ b/misp_modules/__init__.py @@ -32,7 +32,7 @@ import argparse import re try: - from . import modules + from .modules import * HAS_PACKAGE_MODULES = True except Exception as e: print(e) @@ -47,6 +47,7 @@ except Exception as e: log = logging.getLogger('misp-modules') + def handle_signal(sig, frame): IOLoop.instance().add_callback(IOLoop.instance().stop) @@ -95,8 +96,12 @@ def load_package_helpers(): continue helpername = path.replace('misp_modules.helpers.', '') mhandlers[helpername] = helper - helpers.append(helpername) - log.info('Helper loaded {}'.format(helpername)) + selftest = mhandlers[helpername].selftest() + if selftest is None: + helpers.append(helpername) + log.info('Helper loaded {}'.format(helpername)) + else: + log.info('Helpers failed {} due to {}'.format(helpername, selftest)) return mhandlers, helpers @@ -113,7 +118,7 @@ def load_modules(mod_dir): if filename == '__init__.py': continue modulename = filename.split(".")[0] - moduletype = os.path.split(modulesdir)[1] + moduletype = os.path.split(mod_dir)[1] try: mhandlers[modulename] = importlib.import_module(os.path.basename(root) + '.' + modulename) except Exception as e: @@ -144,8 +149,10 @@ def load_package_modules(): class ListModules(tornado.web.RequestHandler): def get(self): + global mhandlers + global loaded_modules ret = [] - for module in modules: + for module in loaded_modules: x = {} x['name'] = module x['type'] = mhandlers['type:' + module] @@ -158,17 +165,19 @@ class ListModules(tornado.web.RequestHandler): class QueryModule(tornado.web.RequestHandler): def post(self): + global mhandlers jsonpayload = self.request.body.decode('utf-8') x = json.loads(jsonpayload) log.debug('MISP QueryModule request {0}'.format(jsonpayload)) ret = mhandlers[x['module']].handler(q=jsonpayload) self.write(json.dumps(ret)) + def main(): + global mhandlers + global loaded_modules signal.signal(signal.SIGINT, handle_signal) signal.signal(signal.SIGTERM, handle_signal) - if os.path.dirname(__file__) in ['.', '']: - os.chdir('../') argParser = argparse.ArgumentParser(description='misp-modules server') argParser.add_argument('-t', default=False, action='store_true', help='Test mode') argParser.add_argument('-s', default=False, action='store_true', help='Run a system install (package installed via pip)') @@ -180,12 +189,13 @@ def main(): log = init_logger() if args.s: load_package_helpers() - mhandlers, modules = load_package_modules() + mhandlers, loaded_modules = load_package_modules() else: - modulesdir = 'misp_modules/modules' - helpersdir = 'misp_modules/helpers' + os.chdir(os.path.dirname(__file__)) + modulesdir = 'modules' + helpersdir = 'helpers' load_helpers(helpersdir=helpersdir) - mhandlers, modules = load_modules(modulesdir) + mhandlers, loaded_modules = load_modules(modulesdir) service = [(r'/modules', ListModules), (r'/query', QueryModule)] application = tornado.web.Application(service) diff --git a/misp_modules/modules/__init__.py b/misp_modules/modules/__init__.py index f0f4cdb..65ce6b2 100644 --- a/misp_modules/modules/__init__.py +++ b/misp_modules/modules/__init__.py @@ -1 +1,3 @@ from .expansion import * +from .import_mod import * +from .export_mod import * diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py index 5854e40..79fabff 100644 --- a/misp_modules/modules/expansion/__init__.py +++ b/misp_modules/modules/expansion/__init__.py @@ -1,2 +1,2 @@ __all__ = ['asn_history', 'circl_passivedns', 'circl_passivessl', 'cve', 'dns', - 'eupi', 'ipasn', 'passivetotal', 'sourcecache'] + 'eupi', 'ipasn', 'passivetotal', 'sourcecache', 'whois'] diff --git a/misp_modules/modules/expansion/whois.py b/misp_modules/modules/expansion/whois.py index 5f3602e..4aec40c 100755 --- a/misp_modules/modules/expansion/whois.py +++ b/misp_modules/modules/expansion/whois.py @@ -1,7 +1,10 @@ # -*- coding: utf-8 -*- import json -from uwhois import Uwhois +try: + from uwhois import Uwhois +except ImportError: + print("uwhois module not installed.") misperrors = {'error': 'Error'} mispattributes = {'input': ['domain', 'ip-src', 'ip-dst'], 'output': ['freetext']} diff --git a/misp_modules/modules/export_mod/__init__.py b/misp_modules/modules/export_mod/__init__.py new file mode 100644 index 0000000..35cc7cb --- /dev/null +++ b/misp_modules/modules/export_mod/__init__.py @@ -0,0 +1 @@ +__all__ = ['testexport'] diff --git a/misp_modules/modules/export/testexport.py b/misp_modules/modules/export_mod/testexport.py similarity index 100% rename from misp_modules/modules/export/testexport.py rename to misp_modules/modules/export_mod/testexport.py diff --git a/misp_modules/modules/import_mod/__init__.py b/misp_modules/modules/import_mod/__init__.py new file mode 100644 index 0000000..5716751 --- /dev/null +++ b/misp_modules/modules/import_mod/__init__.py @@ -0,0 +1 @@ +__all__ = ['testimport', 'ocr'] diff --git a/misp_modules/modules/import/ocr.py b/misp_modules/modules/import_mod/ocr.py similarity index 100% rename from misp_modules/modules/import/ocr.py rename to misp_modules/modules/import_mod/ocr.py diff --git a/misp_modules/modules/import/testimport.py b/misp_modules/modules/import_mod/testimport.py similarity index 100% rename from misp_modules/modules/import/testimport.py rename to misp_modules/modules/import_mod/testimport.py diff --git a/setup.py b/setup.py index 7d96ba5..ed92f56 100644 --- a/setup.py +++ b/setup.py @@ -33,5 +33,7 @@ setup( 'pyeupi', 'ipasn-redis', 'asnhistory', + 'pillow', + 'pytesseract', ] )