2016-08-12 11:31:23 +02:00
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Core MISP expansion modules loader and web service
#
# Copyright (C) 2016 Alexandre Dulaunoy
# Copyright (C) 2016 CIRCL - Computer Incident Response Center Luxembourg
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import os
import signal
import sys
import importlib
import json
import logging
import fnmatch
import argparse
import re
2016-08-25 17:36:28 +02:00
import datetime
2018-11-02 02:42:40 +01:00
import psutil
2016-08-12 11:31:23 +02:00
2016-08-24 00:22:03 +02:00
import tornado . web
2016-08-25 17:36:28 +02:00
import tornado . process
2016-08-24 00:22:03 +02:00
from tornado . ioloop import IOLoop
from tornado . concurrent import run_on_executor
from concurrent . futures import ThreadPoolExecutor
2016-08-12 11:31:23 +02:00
try :
2018-12-11 15:29:09 +01:00
from . modules import * # noqa
2016-08-12 11:31:23 +02:00
HAS_PACKAGE_MODULES = True
except Exception as e :
print ( e )
HAS_PACKAGE_MODULES = False
try :
2018-12-11 15:29:09 +01:00
from . helpers import * # noqa
2016-08-12 11:31:23 +02:00
HAS_PACKAGE_HELPERS = True
except Exception as e :
print ( e )
HAS_PACKAGE_HELPERS = False
log = logging . getLogger ( ' misp-modules ' )
2016-08-12 12:35:33 +02:00
2016-08-12 11:31:23 +02:00
def handle_signal ( sig , frame ) :
2019-04-26 11:35:03 +02:00
IOLoop . instance ( ) . add_callback_from_signal ( IOLoop . instance ( ) . stop )
2016-08-12 11:31:23 +02:00
2016-08-17 13:42:58 +02:00
def init_logger ( level = False ) :
2016-08-12 11:31:23 +02:00
formatter = logging . Formatter ( ' %(asctime)s - %(name)s - %(levelname)s - %(message)s ' )
handler = logging . StreamHandler ( stream = sys . stdout )
handler . setFormatter ( formatter )
handler . setLevel ( logging . INFO )
2016-08-17 13:42:58 +02:00
if level :
handler . setLevel ( logging . DEBUG )
2016-08-12 11:31:23 +02:00
log . addHandler ( handler )
log . setLevel ( logging . INFO )
2016-08-17 13:42:58 +02:00
if level :
log . setLevel ( logging . DEBUG )
2016-08-12 11:31:23 +02:00
return log
def load_helpers ( helpersdir ) :
sys . path . append ( helpersdir )
hhandlers = { }
helpers = [ ]
for root , dirnames , filenames in os . walk ( helpersdir ) :
if os . path . basename ( root ) == ' __pycache__ ' :
continue
if re . match ( r ' ^ \ . ' , os . path . basename ( root ) ) :
continue
for filename in fnmatch . filter ( filenames , ' *.py ' ) :
if filename == ' __init__.py ' :
continue
helpername = filename . split ( " . " ) [ 0 ]
hhandlers [ helpername ] = importlib . import_module ( helpername )
selftest = hhandlers [ helpername ] . selftest ( )
if selftest is None :
helpers . append ( helpername )
log . info ( ' Helpers loaded {} ' . format ( filename ) )
else :
log . info ( ' Helpers failed {} due to {} ' . format ( filename , selftest ) )
def load_package_helpers ( ) :
if not HAS_PACKAGE_HELPERS :
log . info ( ' Unable to load MISP helpers from package. ' )
sys . exit ( )
mhandlers = { }
helpers = [ ]
for path , helper in sys . modules . items ( ) :
if not path . startswith ( ' misp_modules.helpers. ' ) :
continue
helpername = path . replace ( ' misp_modules.helpers. ' , ' ' )
mhandlers [ helpername ] = helper
2016-08-12 12:35:33 +02:00
selftest = mhandlers [ helpername ] . selftest ( )
if selftest is None :
helpers . append ( helpername )
log . info ( ' Helper loaded {} ' . format ( helpername ) )
else :
log . info ( ' Helpers failed {} due to {} ' . format ( helpername , selftest ) )
2016-08-12 11:31:23 +02:00
return mhandlers , helpers
def load_modules ( mod_dir ) :
sys . path . append ( mod_dir )
mhandlers = { }
modules = [ ]
for root , dirnames , filenames in os . walk ( mod_dir ) :
if os . path . basename ( root ) == ' __pycache__ ' :
continue
if os . path . basename ( root ) . startswith ( " . " ) :
continue
for filename in fnmatch . filter ( filenames , ' *.py ' ) :
2016-11-15 16:43:11 +01:00
if root . split ( ' / ' ) [ - 1 ] . startswith ( ' _ ' ) :
continue
2016-08-12 11:31:23 +02:00
if filename == ' __init__.py ' :
continue
modulename = filename . split ( " . " ) [ 0 ]
2016-08-12 12:35:33 +02:00
moduletype = os . path . split ( mod_dir ) [ 1 ]
2016-08-12 11:31:23 +02:00
try :
mhandlers [ modulename ] = importlib . import_module ( os . path . basename ( root ) + ' . ' + modulename )
except Exception as e :
log . warning ( ' MISP modules {0} failed due to {1} ' . format ( modulename , e ) )
continue
modules . append ( modulename )
log . info ( ' MISP modules {0} imported ' . format ( modulename ) )
mhandlers [ ' type: ' + modulename ] = moduletype
return mhandlers , modules
def load_package_modules ( ) :
if not HAS_PACKAGE_MODULES :
log . info ( ' Unable to load MISP modules from package. ' )
sys . exit ( )
mhandlers = { }
modules = [ ]
for path , module in sys . modules . items ( ) :
2018-12-11 15:29:09 +01:00
r = re . findall ( r " misp_modules[.]modules[.]( \ w+)[.]([^_] \ w+) " , path )
2016-08-12 11:31:23 +02:00
if r and len ( r [ 0 ] ) == 2 :
moduletype , modulename = r [ 0 ]
mhandlers [ modulename ] = module
modules . append ( modulename )
log . info ( ' MISP modules {0} imported ' . format ( modulename ) )
mhandlers [ ' type: ' + modulename ] = moduletype
return mhandlers , modules
class ListModules ( tornado . web . RequestHandler ) :
2018-12-11 15:29:09 +01:00
global loaded_modules
global mhandlers
2016-08-12 11:31:23 +02:00
def get ( self ) :
ret = [ ]
2016-08-12 12:35:33 +02:00
for module in loaded_modules :
2016-08-12 11:31:23 +02:00
x = { }
x [ ' name ' ] = module
x [ ' type ' ] = mhandlers [ ' type: ' + module ]
x [ ' mispattributes ' ] = mhandlers [ module ] . introspection ( )
x [ ' meta ' ] = mhandlers [ module ] . version ( )
ret . append ( x )
log . debug ( ' MISP ListModules request ' )
self . write ( json . dumps ( ret ) )
2016-08-24 00:22:03 +02:00
class QueryModule ( tornado . web . RequestHandler ) :
2016-08-25 17:36:28 +02:00
# Default value in Python 3.5
# https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor
nb_threads = tornado . process . cpu_count ( ) * 5
executor = ThreadPoolExecutor ( nb_threads )
2016-08-23 18:02:29 +02:00
2016-08-24 00:22:03 +02:00
@run_on_executor
2020-10-22 18:03:29 +02:00
def run_request ( self , module , jsonpayload ) :
2016-08-24 00:22:03 +02:00
log . debug ( ' MISP QueryModule request {0} ' . format ( jsonpayload ) )
2020-10-22 18:03:29 +02:00
response = mhandlers [ module ] . handler ( q = jsonpayload )
2016-08-25 17:36:28 +02:00
return json . dumps ( response )
2016-08-23 18:02:29 +02:00
2016-08-21 10:21:00 +02:00
@tornado.gen.coroutine
2016-08-12 11:31:23 +02:00
def post ( self ) :
2016-08-23 18:02:29 +02:00
try :
2016-08-24 00:22:03 +02:00
jsonpayload = self . request . body . decode ( ' utf-8 ' )
2016-08-25 17:36:28 +02:00
dict_payload = json . loads ( jsonpayload )
if dict_payload . get ( ' timeout ' ) :
timeout = datetime . timedelta ( seconds = int ( dict_payload . get ( ' timeout ' ) ) )
else :
2018-06-29 06:02:08 +02:00
timeout = datetime . timedelta ( seconds = 300 )
2020-10-22 18:03:29 +02:00
response = yield tornado . gen . with_timeout ( timeout , self . run_request ( dict_payload [ ' module ' ] , jsonpayload ) )
2016-08-25 17:36:28 +02:00
self . write ( response )
except tornado . gen . TimeoutError :
log . warning ( ' Timeout on {} ' . format ( dict_payload [ ' module ' ] ) )
self . write ( json . dumps ( { ' error ' : ' Timeout. ' } ) )
2016-08-23 18:02:29 +02:00
except Exception :
2016-08-25 17:36:28 +02:00
self . write ( json . dumps ( { ' error ' : ' Something went wrong, look in the server logs for details ' } ) )
2016-08-24 00:22:03 +02:00
log . exception ( ' Something went wrong: ' )
2016-08-25 17:36:28 +02:00
finally :
self . finish ( )
2016-08-12 11:31:23 +02:00
2016-08-12 12:35:33 +02:00
2019-04-26 13:48:38 +02:00
def _launch_from_current_dir ( ) :
log . info ( ' Launch MISP modules server from current directory. ' )
os . chdir ( os . path . dirname ( __file__ ) )
modulesdir = ' modules '
helpersdir = ' helpers '
load_helpers ( helpersdir = helpersdir )
return load_modules ( modulesdir )
2016-08-12 11:31:23 +02:00
def main ( ) :
2016-08-12 12:35:33 +02:00
global mhandlers
global loaded_modules
2016-08-12 11:31:23 +02:00
signal . signal ( signal . SIGINT , handle_signal )
signal . signal ( signal . SIGTERM , handle_signal )
2019-04-26 13:48:38 +02:00
argParser = argparse . ArgumentParser ( description = ' misp-modules server ' , formatter_class = argparse . RawTextHelpFormatter )
2016-08-12 11:31:23 +02:00
argParser . add_argument ( ' -t ' , default = False , action = ' store_true ' , help = ' Test mode ' )
argParser . add_argument ( ' -s ' , default = False , action = ' store_true ' , help = ' Run a system install (package installed via pip) ' )
2016-08-17 13:42:58 +02:00
argParser . add_argument ( ' -d ' , default = False , action = ' store_true ' , help = ' Enable debugging ' )
2016-08-12 11:31:23 +02:00
argParser . add_argument ( ' -p ' , default = 6666 , help = ' misp-modules TCP port (default 6666) ' )
argParser . add_argument ( ' -l ' , default = ' localhost ' , help = ' misp-modules listen address (default localhost) ' )
2017-09-01 16:44:53 +02:00
argParser . add_argument ( ' -m ' , default = [ ] , action = ' append ' , help = ' Register a custom module ' )
2019-04-26 13:48:38 +02:00
argParser . add_argument ( ' --devel ' , default = False , action = ' store_true ' , help = ''' Start in development mode, enable debug, start only the module(s) listed in -m. \n Example: -m misp_modules.modules.expansion.bgpranking ''' )
2016-08-12 11:31:23 +02:00
args = argParser . parse_args ( )
port = args . p
listen = args . l
2019-04-26 13:48:38 +02:00
if args . devel :
log = init_logger ( level = True )
log . info ( ' Launch MISP modules server in developement mode. Enable debug, load a list of modules is -m is used. ' )
if args . m :
mhandlers = { }
modules = [ ]
for module in args . m :
splitted = module . split ( " . " )
modulename = splitted [ - 1 ]
moduletype = splitted [ 2 ]
mhandlers [ modulename ] = importlib . import_module ( module )
mhandlers [ ' type: ' + modulename ] = moduletype
modules . append ( modulename )
log . info ( ' MISP modules {0} imported ' . format ( modulename ) )
else :
mhandlers , loaded_modules = _launch_from_current_dir ( )
2016-08-12 11:31:23 +02:00
else :
2019-04-26 13:48:38 +02:00
log = init_logger ( level = args . d )
if args . s :
log . info ( ' Launch MISP modules server from package. ' )
load_package_helpers ( )
mhandlers , loaded_modules = load_package_modules ( )
else :
mhandlers , loaded_modules = _launch_from_current_dir ( )
for module in args . m :
mispmod = importlib . import_module ( module )
mispmod . register ( mhandlers , loaded_modules )
2018-12-11 15:29:09 +01:00
2016-08-12 11:31:23 +02:00
service = [ ( r ' /modules ' , ListModules ) , ( r ' /query ' , QueryModule ) ]
application = tornado . web . Application ( service )
2018-11-02 02:42:40 +01:00
try :
application . listen ( port , address = listen )
except Exception as e :
if e . errno == 98 :
pids = psutil . pids ( )
for pid in pids :
p = psutil . Process ( pid )
if p . name ( ) == " misp-modules " :
print ( " \n \n \n " )
print ( e )
print ( " \n misp-modules is still running as PID: {} \n " . format ( pid ) )
print ( " Please kill accordingly: " )
print ( " sudo kill {} " . format ( pid ) )
sys . exit ( - 1 )
print ( e )
print ( " misp-modules might still be running. " )
2016-08-12 11:31:23 +02:00
log . info ( ' MISP modules server started on {0} port {1} ' . format ( listen , port ) )
if args . t :
log . info ( ' MISP modules started in test-mode, quitting immediately. ' )
sys . exit ( )
2019-04-26 11:35:03 +02:00
try :
IOLoop . instance ( ) . start ( )
finally :
IOLoop . instance ( ) . stop ( )
2016-08-12 11:31:23 +02:00
return 0
if __name__ == ' __main__ ' :
sys . exit ( main ( ) )