misp-modules/misp_modules/__init__.py

301 lines
11 KiB
Python
Raw Normal View History

2016-08-12 11:31:23 +02:00
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Core MISP expansion modules loader and web service
#
# Copyright (C) 2016 Alexandre Dulaunoy
# Copyright (C) 2016 CIRCL - Computer Incident Response Center Luxembourg
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import os
import signal
import sys
import importlib
import json
import logging
import fnmatch
import argparse
import re
2016-08-25 17:36:28 +02:00
import datetime
import psutil
2016-08-12 11:31:23 +02:00
2016-08-24 00:22:03 +02:00
import tornado.web
2016-08-25 17:36:28 +02:00
import tornado.process
2016-08-24 00:22:03 +02:00
from tornado.ioloop import IOLoop
from tornado.concurrent import run_on_executor
from concurrent.futures import ThreadPoolExecutor
2016-08-12 11:31:23 +02:00
try:
2018-12-11 15:29:09 +01:00
from .modules import * # noqa
2016-08-12 11:31:23 +02:00
HAS_PACKAGE_MODULES = True
except Exception as e:
print(e)
HAS_PACKAGE_MODULES = False
try:
2018-12-11 15:29:09 +01:00
from .helpers import * # noqa
2016-08-12 11:31:23 +02:00
HAS_PACKAGE_HELPERS = True
except Exception as e:
print(e)
HAS_PACKAGE_HELPERS = False
log = logging.getLogger('misp-modules')
2016-08-12 11:31:23 +02:00
def handle_signal(sig, frame):
2019-04-26 11:35:03 +02:00
IOLoop.instance().add_callback_from_signal(IOLoop.instance().stop)
2016-08-12 11:31:23 +02:00
def init_logger(level=False):
2016-08-12 11:31:23 +02:00
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler = logging.StreamHandler(stream=sys.stdout)
handler.setFormatter(formatter)
handler.setLevel(logging.INFO)
if level:
handler.setLevel(logging.DEBUG)
2016-08-12 11:31:23 +02:00
log.addHandler(handler)
log.setLevel(logging.INFO)
if level:
log.setLevel(logging.DEBUG)
2016-08-12 11:31:23 +02:00
return log
def load_helpers(helpersdir):
sys.path.append(helpersdir)
hhandlers = {}
helpers = []
for root, dirnames, filenames in os.walk(helpersdir):
if os.path.basename(root) == '__pycache__':
continue
if re.match(r'^\.', os.path.basename(root)):
continue
for filename in fnmatch.filter(filenames, '*.py'):
if filename == '__init__.py':
continue
helpername = filename.split(".")[0]
hhandlers[helpername] = importlib.import_module(helpername)
selftest = hhandlers[helpername].selftest()
if selftest is None:
helpers.append(helpername)
log.info('Helpers loaded {} '.format(filename))
else:
log.info('Helpers failed {} due to {}'.format(filename, selftest))
def load_package_helpers():
if not HAS_PACKAGE_HELPERS:
log.info('Unable to load MISP helpers from package.')
sys.exit()
mhandlers = {}
helpers = []
for path, helper in sys.modules.items():
if not path.startswith('misp_modules.helpers.'):
continue
helpername = path.replace('misp_modules.helpers.', '')
mhandlers[helpername] = helper
selftest = mhandlers[helpername].selftest()
if selftest is None:
helpers.append(helpername)
log.info('Helper loaded {}'.format(helpername))
else:
log.info('Helpers failed {} due to {}'.format(helpername, selftest))
2016-08-12 11:31:23 +02:00
return mhandlers, helpers
def load_modules(mod_dir):
sys.path.append(mod_dir)
mhandlers = {}
modules = []
for root, dirnames, filenames in os.walk(mod_dir):
if os.path.basename(root) == '__pycache__':
continue
if os.path.basename(root).startswith("."):
continue
for filename in fnmatch.filter(filenames, '*.py'):
if root.split('/')[-1].startswith('_'):
continue
2016-08-12 11:31:23 +02:00
if filename == '__init__.py':
continue
modulename = filename.split(".")[0]
moduletype = os.path.split(mod_dir)[1]
2016-08-12 11:31:23 +02:00
try:
mhandlers[modulename] = importlib.import_module(os.path.basename(root) + '.' + modulename)
except Exception as e:
log.warning('MISP modules {0} failed due to {1}'.format(modulename, e))
continue
modules.append(modulename)
log.info('MISP modules {0} imported'.format(modulename))
mhandlers['type:' + modulename] = moduletype
return mhandlers, modules
def load_package_modules():
if not HAS_PACKAGE_MODULES:
log.info('Unable to load MISP modules from package.')
sys.exit()
mhandlers = {}
modules = []
for path, module in sys.modules.items():
2018-12-11 15:29:09 +01:00
r = re.findall(r"misp_modules[.]modules[.](\w+)[.]([^_]\w+)", path)
2016-08-12 11:31:23 +02:00
if r and len(r[0]) == 2:
moduletype, modulename = r[0]
mhandlers[modulename] = module
modules.append(modulename)
log.info('MISP modules {0} imported'.format(modulename))
mhandlers['type:' + modulename] = moduletype
return mhandlers, modules
class ListModules(tornado.web.RequestHandler):
2018-12-11 15:29:09 +01:00
global loaded_modules
global mhandlers
2016-08-12 11:31:23 +02:00
def get(self):
ret = []
for module in loaded_modules:
2016-08-12 11:31:23 +02:00
x = {}
x['name'] = module
x['type'] = mhandlers['type:' + module]
x['mispattributes'] = mhandlers[module].introspection()
x['meta'] = mhandlers[module].version()
ret.append(x)
log.debug('MISP ListModules request')
self.write(json.dumps(ret))
2016-08-24 00:22:03 +02:00
class QueryModule(tornado.web.RequestHandler):
2016-08-25 17:36:28 +02:00
# Default value in Python 3.5
# https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor
nb_threads = tornado.process.cpu_count() * 5
executor = ThreadPoolExecutor(nb_threads)
2016-08-23 18:02:29 +02:00
2016-08-24 00:22:03 +02:00
@run_on_executor
def run_request(self, jsonpayload):
x = json.loads(jsonpayload)
log.debug('MISP QueryModule request {0}'.format(jsonpayload))
2016-08-25 17:36:28 +02:00
response = mhandlers[x['module']].handler(q=jsonpayload)
return json.dumps(response)
2016-08-23 18:02:29 +02:00
@tornado.gen.coroutine
2016-08-12 11:31:23 +02:00
def post(self):
2016-08-23 18:02:29 +02:00
try:
2016-08-24 00:22:03 +02:00
jsonpayload = self.request.body.decode('utf-8')
2016-08-25 17:36:28 +02:00
dict_payload = json.loads(jsonpayload)
if dict_payload.get('timeout'):
timeout = datetime.timedelta(seconds=int(dict_payload.get('timeout')))
else:
2018-06-29 06:02:08 +02:00
timeout = datetime.timedelta(seconds=300)
2016-08-25 17:36:28 +02:00
response = yield tornado.gen.with_timeout(timeout, self.run_request(jsonpayload))
self.write(response)
except tornado.gen.TimeoutError:
log.warning('Timeout on {} '.format(dict_payload['module']))
self.write(json.dumps({'error': 'Timeout.'}))
2016-08-23 18:02:29 +02:00
except Exception:
2016-08-25 17:36:28 +02:00
self.write(json.dumps({'error': 'Something went wrong, look in the server logs for details'}))
2016-08-24 00:22:03 +02:00
log.exception('Something went wrong:')
2016-08-25 17:36:28 +02:00
finally:
self.finish()
2016-08-12 11:31:23 +02:00
2019-04-26 13:48:38 +02:00
def _launch_from_current_dir():
log.info('Launch MISP modules server from current directory.')
os.chdir(os.path.dirname(__file__))
modulesdir = 'modules'
helpersdir = 'helpers'
load_helpers(helpersdir=helpersdir)
return load_modules(modulesdir)
2016-08-12 11:31:23 +02:00
def main():
global mhandlers
global loaded_modules
2016-08-12 11:31:23 +02:00
signal.signal(signal.SIGINT, handle_signal)
signal.signal(signal.SIGTERM, handle_signal)
2019-04-26 13:48:38 +02:00
argParser = argparse.ArgumentParser(description='misp-modules server', formatter_class=argparse.RawTextHelpFormatter)
2016-08-12 11:31:23 +02:00
argParser.add_argument('-t', default=False, action='store_true', help='Test mode')
argParser.add_argument('-s', default=False, action='store_true', help='Run a system install (package installed via pip)')
argParser.add_argument('-d', default=False, action='store_true', help='Enable debugging')
2016-08-12 11:31:23 +02:00
argParser.add_argument('-p', default=6666, help='misp-modules TCP port (default 6666)')
argParser.add_argument('-l', default='localhost', help='misp-modules listen address (default localhost)')
argParser.add_argument('-m', default=[], action='append', help='Register a custom module')
2019-04-26 13:48:38 +02:00
argParser.add_argument('--devel', default=False, action='store_true', help='''Start in development mode, enable debug, start only the module(s) listed in -m.\nExample: -m misp_modules.modules.expansion.bgpranking''')
2016-08-12 11:31:23 +02:00
args = argParser.parse_args()
port = args.p
listen = args.l
2019-04-26 13:48:38 +02:00
if args.devel:
log = init_logger(level=True)
log.info('Launch MISP modules server in developement mode. Enable debug, load a list of modules is -m is used.')
if args.m:
mhandlers = {}
modules = []
for module in args.m:
splitted = module.split(".")
modulename = splitted[-1]
moduletype = splitted[2]
mhandlers[modulename] = importlib.import_module(module)
mhandlers['type:' + modulename] = moduletype
modules.append(modulename)
log.info('MISP modules {0} imported'.format(modulename))
else:
mhandlers, loaded_modules = _launch_from_current_dir()
2016-08-12 11:31:23 +02:00
else:
2019-04-26 13:48:38 +02:00
log = init_logger(level=args.d)
if args.s:
log.info('Launch MISP modules server from package.')
load_package_helpers()
mhandlers, loaded_modules = load_package_modules()
else:
mhandlers, loaded_modules = _launch_from_current_dir()
for module in args.m:
mispmod = importlib.import_module(module)
mispmod.register(mhandlers, loaded_modules)
2018-12-11 15:29:09 +01:00
2016-08-12 11:31:23 +02:00
service = [(r'/modules', ListModules), (r'/query', QueryModule)]
application = tornado.web.Application(service)
try:
application.listen(port, address=listen)
except Exception as e:
if e.errno == 98:
pids = psutil.pids()
for pid in pids:
p = psutil.Process(pid)
if p.name() == "misp-modules":
print("\n\n\n")
print(e)
print("\nmisp-modules is still running as PID: {}\n".format(pid))
print("Please kill accordingly:")
print("sudo kill {}".format(pid))
sys.exit(-1)
print(e)
print("misp-modules might still be running.")
2016-08-12 11:31:23 +02:00
log.info('MISP modules server started on {0} port {1}'.format(listen, port))
if args.t:
log.info('MISP modules started in test-mode, quitting immediately.')
sys.exit()
2019-04-26 11:35:03 +02:00
try:
IOLoop.instance().start()
finally:
IOLoop.instance().stop()
2016-08-12 11:31:23 +02:00
return 0
if __name__ == '__main__':
sys.exit(main())