diff --git a/README.md b/README.md index 15511b8..655e060 100644 --- a/README.md +++ b/README.md @@ -30,8 +30,8 @@ sudo apt-get install python3-dev python3-pip libpq5 cd /usr/local/src/ sudo git clone https://github.com/MISP/misp-modules.git cd misp-modules -sudo pip3 install -r REQUIREMENTS -sudo pip3 install . +sudo pip3 install --upgrade -r REQUIREMENTS +sudo pip3 install --upgrade . sudo vi /etc/rc.local, add this line: `sudo -u www-data misp-modules -s` ~~~~ diff --git a/REQUIREMENTS b/REQUIREMENTS index 063978b..f138191 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -12,3 +12,5 @@ pyeupi ipasn-redis asnhistory git+https://github.com/Rafiot/uwhoisd.git@testing#egg=uwhois&subdirectory=client +pillow +pytesseract diff --git a/bin/misp-modules b/bin/misp-modules deleted file mode 100755 index 011d767..0000000 --- a/bin/misp-modules +++ /dev/null @@ -1,190 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# -# Core MISP expansion modules loader and web service -# -# Copyright (C) 2016 Alexandre Dulaunoy -# Copyright (C) 2016 CIRCL - Computer Incident Response Center Luxembourg -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -import os -import sys -import tornado.web -import importlib -import json -import logging -import fnmatch -import argparse -import re - -try: - import misp_modules.modules - HAS_PACKAGE_MODULES = True -except Exception as e: - print(e) - HAS_PACKAGE_MODULES = False - -try: - from misp_modules.helpers import * - HAS_PACKAGE_HELPERS = True -except Exception as e: - print(e) - HAS_PACKAGE_HELPERS = False - - -def init_logger(): - log = logging.getLogger('misp-modules') - formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') - handler = logging.StreamHandler(stream=sys.stdout) - handler.setFormatter(formatter) - handler.setLevel(logging.INFO) - - log.addHandler(handler) - log.setLevel(logging.INFO) - return log - - -def load_helpers(helpersdir): - sys.path.append(helpersdir) - hhandlers = {} - helpers = [] - for root, dirnames, filenames in os.walk(helpersdir): - if os.path.basename(root) == '__pycache__': - continue - if re.match(r'^\.', os.path.basename(root)): - continue - for filename in fnmatch.filter(filenames, '*.py'): - if filename == '__init__.py': - continue - helpername = filename.split(".")[0] - hhandlers[helpername] = importlib.import_module(helpername) - selftest = hhandlers[helpername].selftest() - if selftest is None: - helpers.append(helpername) - log.info('Helpers loaded {} '.format(filename)) - else: - log.info('Helpers failed {} due to {}'.format(filename, selftest)) - - -def load_package_helpers(): - if not HAS_PACKAGE_HELPERS: - log.info('Unable to load MISP helpers from package.') - sys.exit() - mhandlers = {} - helpers = [] - for path, helper in sys.modules.items(): - if not path.startswith('misp_modules.helpers.'): - continue - helpername = path.replace('misp_modules.helpers.', '') - mhandlers[helpername] = helper - helpers.append(helpername) - log.info('Helper loaded {}'.format(helpername)) - return mhandlers, helpers - - -def load_modules(mod_dir): - sys.path.append(mod_dir) - mhandlers = {} - modules = [] - for root, dirnames, filenames in os.walk(mod_dir): - if os.path.basename(root) == '__pycache__': - continue - if os.path.basename(root).startswith("."): - continue - for filename in fnmatch.filter(filenames, '*.py'): - if filename == '__init__.py': - continue - modulename = filename.split(".")[0] - moduletype = os.path.split(modulesdir)[1] - try: - mhandlers[modulename] = importlib.import_module(os.path.basename(root) + '.' + modulename) - except Exception as e: - log.warning('MISP modules {0} failed due to {1}'.format(modulename, e)) - continue - modules.append(modulename) - log.info('MISP modules {0} imported'.format(modulename)) - mhandlers['type:' + modulename] = moduletype - return mhandlers, modules - - -def load_package_modules(): - if not HAS_PACKAGE_MODULES: - log.info('Unable to load MISP modules from package.') - sys.exit() - mhandlers = {} - modules = [] - for path, module in sys.modules.items(): - r = re.findall("misp_modules[.]modules[.](\w+)[.](\w+)", path) - if r and len(r[0]) == 2: - moduletype, modulename = r[0] - mhandlers[modulename] = module - modules.append(modulename) - log.info('MISP modules {0} imported'.format(modulename)) - mhandlers['type:' + modulename] = moduletype - return mhandlers, modules - - -class ListModules(tornado.web.RequestHandler): - def get(self): - ret = [] - for module in modules: - x = {} - x['name'] = module - x['type'] = mhandlers['type:' + module] - x['mispattributes'] = mhandlers[module].introspection() - x['meta'] = mhandlers[module].version() - ret.append(x) - log.debug('MISP ListModules request') - self.write(json.dumps(ret)) - - -class QueryModule(tornado.web.RequestHandler): - def post(self): - jsonpayload = self.request.body.decode('utf-8') - x = json.loads(jsonpayload) - log.debug('MISP QueryModule request {0}'.format(jsonpayload)) - ret = mhandlers[x['module']].handler(q=jsonpayload) - self.write(json.dumps(ret)) - - -if __name__ == '__main__': - if os.path.dirname(__file__) in ['.', '']: - os.chdir('../') - argParser = argparse.ArgumentParser(description='misp-modules server') - argParser.add_argument('-t', default=False, action='store_true', help='Test mode') - argParser.add_argument('-s', default=False, action='store_true', help='Run a system install (package installed via pip)') - argParser.add_argument('-p', default=6666, help='misp-modules TCP port (default 6666)') - argParser.add_argument('-l', default='localhost', help='misp-modules listen address (default localhost)') - args = argParser.parse_args() - port = args.p - listen = args.l - log = init_logger() - if args.s: - load_package_helpers() - mhandlers, modules = load_package_modules() - else: - modulesdir = 'misp_modules/modules' - helpersdir = 'misp_modules/helpers' - load_helpers(helpersdir=helpersdir) - mhandlers, modules = load_modules(modulesdir) - service = [(r'/modules', ListModules), (r'/query', QueryModule)] - - application = tornado.web.Application(service) - application.listen(port, address=listen) - log.info('MISP modules server started on {0} port {1}'.format(listen, port)) - if args.t: - log.info('MISP modules started in test-mode, quitting immediately.') - sys.exit() - tornado.ioloop.IOLoop.instance().start() diff --git a/misp_modules/__init__.py b/misp_modules/__init__.py index e69de29..2617fae 100644 --- a/misp_modules/__init__.py +++ b/misp_modules/__init__.py @@ -0,0 +1,213 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Core MISP expansion modules loader and web service +# +# Copyright (C) 2016 Alexandre Dulaunoy +# Copyright (C) 2016 CIRCL - Computer Incident Response Center Luxembourg +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import os +import signal +import sys +from tornado.ioloop import IOLoop +import tornado.web +import importlib +import json +import logging +import fnmatch +import argparse +import re + +try: + from .modules import * + HAS_PACKAGE_MODULES = True +except Exception as e: + print(e) + HAS_PACKAGE_MODULES = False + +try: + from .helpers import * + HAS_PACKAGE_HELPERS = True +except Exception as e: + print(e) + HAS_PACKAGE_HELPERS = False + +log = logging.getLogger('misp-modules') + + +def handle_signal(sig, frame): + IOLoop.instance().add_callback(IOLoop.instance().stop) + + +def init_logger(): + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + handler = logging.StreamHandler(stream=sys.stdout) + handler.setFormatter(formatter) + handler.setLevel(logging.INFO) + + log.addHandler(handler) + log.setLevel(logging.INFO) + return log + + +def load_helpers(helpersdir): + sys.path.append(helpersdir) + hhandlers = {} + helpers = [] + for root, dirnames, filenames in os.walk(helpersdir): + if os.path.basename(root) == '__pycache__': + continue + if re.match(r'^\.', os.path.basename(root)): + continue + for filename in fnmatch.filter(filenames, '*.py'): + if filename == '__init__.py': + continue + helpername = filename.split(".")[0] + hhandlers[helpername] = importlib.import_module(helpername) + selftest = hhandlers[helpername].selftest() + if selftest is None: + helpers.append(helpername) + log.info('Helpers loaded {} '.format(filename)) + else: + log.info('Helpers failed {} due to {}'.format(filename, selftest)) + + +def load_package_helpers(): + if not HAS_PACKAGE_HELPERS: + log.info('Unable to load MISP helpers from package.') + sys.exit() + mhandlers = {} + helpers = [] + for path, helper in sys.modules.items(): + if not path.startswith('misp_modules.helpers.'): + continue + helpername = path.replace('misp_modules.helpers.', '') + mhandlers[helpername] = helper + selftest = mhandlers[helpername].selftest() + if selftest is None: + helpers.append(helpername) + log.info('Helper loaded {}'.format(helpername)) + else: + log.info('Helpers failed {} due to {}'.format(helpername, selftest)) + return mhandlers, helpers + + +def load_modules(mod_dir): + sys.path.append(mod_dir) + mhandlers = {} + modules = [] + for root, dirnames, filenames in os.walk(mod_dir): + if os.path.basename(root) == '__pycache__': + continue + if os.path.basename(root).startswith("."): + continue + for filename in fnmatch.filter(filenames, '*.py'): + if filename == '__init__.py': + continue + modulename = filename.split(".")[0] + moduletype = os.path.split(mod_dir)[1] + try: + mhandlers[modulename] = importlib.import_module(os.path.basename(root) + '.' + modulename) + except Exception as e: + log.warning('MISP modules {0} failed due to {1}'.format(modulename, e)) + continue + modules.append(modulename) + log.info('MISP modules {0} imported'.format(modulename)) + mhandlers['type:' + modulename] = moduletype + return mhandlers, modules + + +def load_package_modules(): + if not HAS_PACKAGE_MODULES: + log.info('Unable to load MISP modules from package.') + sys.exit() + mhandlers = {} + modules = [] + for path, module in sys.modules.items(): + r = re.findall("misp_modules[.]modules[.](\w+)[.](\w+)", path) + if r and len(r[0]) == 2: + moduletype, modulename = r[0] + mhandlers[modulename] = module + modules.append(modulename) + log.info('MISP modules {0} imported'.format(modulename)) + mhandlers['type:' + modulename] = moduletype + return mhandlers, modules + + +class ListModules(tornado.web.RequestHandler): + def get(self): + global mhandlers + global loaded_modules + ret = [] + for module in loaded_modules: + x = {} + x['name'] = module + x['type'] = mhandlers['type:' + module] + x['mispattributes'] = mhandlers[module].introspection() + x['meta'] = mhandlers[module].version() + ret.append(x) + log.debug('MISP ListModules request') + self.write(json.dumps(ret)) + + +class QueryModule(tornado.web.RequestHandler): + def post(self): + global mhandlers + jsonpayload = self.request.body.decode('utf-8') + x = json.loads(jsonpayload) + log.debug('MISP QueryModule request {0}'.format(jsonpayload)) + ret = mhandlers[x['module']].handler(q=jsonpayload) + self.write(json.dumps(ret)) + + +def main(): + global mhandlers + global loaded_modules + signal.signal(signal.SIGINT, handle_signal) + signal.signal(signal.SIGTERM, handle_signal) + argParser = argparse.ArgumentParser(description='misp-modules server') + argParser.add_argument('-t', default=False, action='store_true', help='Test mode') + argParser.add_argument('-s', default=False, action='store_true', help='Run a system install (package installed via pip)') + argParser.add_argument('-p', default=6666, help='misp-modules TCP port (default 6666)') + argParser.add_argument('-l', default='localhost', help='misp-modules listen address (default localhost)') + args = argParser.parse_args() + port = args.p + listen = args.l + log = init_logger() + if args.s: + load_package_helpers() + mhandlers, loaded_modules = load_package_modules() + else: + os.chdir(os.path.dirname(__file__)) + modulesdir = 'modules' + helpersdir = 'helpers' + load_helpers(helpersdir=helpersdir) + mhandlers, loaded_modules = load_modules(modulesdir) + service = [(r'/modules', ListModules), (r'/query', QueryModule)] + + application = tornado.web.Application(service) + application.listen(port, address=listen) + log.info('MISP modules server started on {0} port {1}'.format(listen, port)) + if args.t: + log.info('MISP modules started in test-mode, quitting immediately.') + sys.exit() + IOLoop.instance().start() + IOLoop.instance().stop() + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/misp_modules/modules/__init__.py b/misp_modules/modules/__init__.py index f0f4cdb..65ce6b2 100644 --- a/misp_modules/modules/__init__.py +++ b/misp_modules/modules/__init__.py @@ -1 +1,3 @@ from .expansion import * +from .import_mod import * +from .export_mod import * diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py index 5854e40..79fabff 100644 --- a/misp_modules/modules/expansion/__init__.py +++ b/misp_modules/modules/expansion/__init__.py @@ -1,2 +1,2 @@ __all__ = ['asn_history', 'circl_passivedns', 'circl_passivessl', 'cve', 'dns', - 'eupi', 'ipasn', 'passivetotal', 'sourcecache'] + 'eupi', 'ipasn', 'passivetotal', 'sourcecache', 'whois'] diff --git a/misp_modules/modules/expansion/whois.py b/misp_modules/modules/expansion/whois.py index 5f3602e..4aec40c 100755 --- a/misp_modules/modules/expansion/whois.py +++ b/misp_modules/modules/expansion/whois.py @@ -1,7 +1,10 @@ # -*- coding: utf-8 -*- import json -from uwhois import Uwhois +try: + from uwhois import Uwhois +except ImportError: + print("uwhois module not installed.") misperrors = {'error': 'Error'} mispattributes = {'input': ['domain', 'ip-src', 'ip-dst'], 'output': ['freetext']} diff --git a/misp_modules/modules/export_mod/__init__.py b/misp_modules/modules/export_mod/__init__.py new file mode 100644 index 0000000..35cc7cb --- /dev/null +++ b/misp_modules/modules/export_mod/__init__.py @@ -0,0 +1 @@ +__all__ = ['testexport'] diff --git a/misp_modules/modules/export/testexport.py b/misp_modules/modules/export_mod/testexport.py similarity index 98% rename from misp_modules/modules/export/testexport.py rename to misp_modules/modules/export_mod/testexport.py index 14829cf..ed93228 100755 --- a/misp_modules/modules/export/testexport.py +++ b/misp_modules/modules/export_mod/testexport.py @@ -9,6 +9,7 @@ userConfig = { }; +moduleconfig = [] # fixed for now, options in the future: # event, attribute, event-collection, attribute-collection diff --git a/misp_modules/modules/import_mod/__init__.py b/misp_modules/modules/import_mod/__init__.py new file mode 100644 index 0000000..5716751 --- /dev/null +++ b/misp_modules/modules/import_mod/__init__.py @@ -0,0 +1 @@ +__all__ = ['testimport', 'ocr'] diff --git a/misp_modules/modules/import/ocr.py b/misp_modules/modules/import_mod/ocr.py similarity index 100% rename from misp_modules/modules/import/ocr.py rename to misp_modules/modules/import_mod/ocr.py diff --git a/misp_modules/modules/import/testimport.py b/misp_modules/modules/import_mod/testimport.py similarity index 100% rename from misp_modules/modules/import/testimport.py rename to misp_modules/modules/import_mod/testimport.py diff --git a/setup.py b/setup.py index 241f689..ec30f92 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ setup( url='https://github.com/MISP/misp-modules', description='MISP modules are autonomous modules that can be used for expansion and other services in MISP', packages=find_packages(), - scripts=['bin/misp-modules'], + entry_points = {'console_scripts': ['misp-modules = misp_modules:main']}, test_suite="tests", classifiers=[ 'License :: OSI Approved :: GNU Affero General Public License v3', @@ -35,5 +35,7 @@ setup( 'asnhistory', 'stix', 'cybox' + 'pillow', + 'pytesseract', ] )