Merge pull request #40 from Rafiot/master

Remove bin script, use cleaner way. Fix last commit.

REQUIREMENTS

@@ -10,3 +10,5 @@ pyeupi
 ipasn-redis
 asnhistory
 git+https://github.com/Rafiot/uwhoisd.git@testing#egg=uwhois&subdirectory=client
+pillow
+pytesseract

bin/misp-modules

@@ -1,26 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-#
-# Core MISP expansion modules loader and web service
-#
-# Copyright (C) 2016 Alexandre Dulaunoy
-# Copyright (C) 2016 CIRCL - Computer Incident Response Center Luxembourg
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-import sys
-from misp_modules import main
-
-if __name__ == '__main__':
-    sys.exit(main())

misp_modules/__init__.py

@@ -32,7 +32,7 @@ import argparse
 import re
 
 try:
-    from . import modules
+    from .modules import *
     HAS_PACKAGE_MODULES = True
 except Exception as e:
     print(e)
@@ -47,6 +47,7 @@ except Exception as e:
 
 log = logging.getLogger('misp-modules')
 
+
 def handle_signal(sig, frame):
     IOLoop.instance().add_callback(IOLoop.instance().stop)
 
@@ -95,8 +96,12 @@ def load_package_helpers():
             continue
         helpername = path.replace('misp_modules.helpers.', '')
         mhandlers[helpername] = helper
-        helpers.append(helpername)
+        selftest = mhandlers[helpername].selftest()
-        log.info('Helper loaded {}'.format(helpername))
+        if selftest is None:
+            helpers.append(helpername)
+            log.info('Helper loaded {}'.format(helpername))
+        else:
+            log.info('Helpers failed {} due to {}'.format(helpername, selftest))
     return mhandlers, helpers
 
 
@@ -113,7 +118,7 @@ def load_modules(mod_dir):
             if filename == '__init__.py':
                 continue
             modulename = filename.split(".")[0]
-            moduletype = os.path.split(modulesdir)[1]
+            moduletype = os.path.split(mod_dir)[1]
             try:
                 mhandlers[modulename] = importlib.import_module(os.path.basename(root) + '.' + modulename)
             except Exception as e:
@@ -144,8 +149,10 @@ def load_package_modules():
 
 class ListModules(tornado.web.RequestHandler):
     def get(self):
+        global mhandlers
+        global loaded_modules
         ret = []
-        for module in modules:
+        for module in loaded_modules:
             x = {}
             x['name'] = module
             x['type'] = mhandlers['type:' + module]
@@ -158,17 +165,19 @@ class ListModules(tornado.web.RequestHandler):
 
 class QueryModule(tornado.web.RequestHandler):
     def post(self):
+        global mhandlers
         jsonpayload = self.request.body.decode('utf-8')
         x = json.loads(jsonpayload)
         log.debug('MISP QueryModule request {0}'.format(jsonpayload))
         ret = mhandlers[x['module']].handler(q=jsonpayload)
         self.write(json.dumps(ret))
 
+
 def main():
+    global mhandlers
+    global loaded_modules
     signal.signal(signal.SIGINT, handle_signal)
     signal.signal(signal.SIGTERM, handle_signal)
-    if os.path.dirname(__file__) in ['.', '']:
-        os.chdir('../')
     argParser = argparse.ArgumentParser(description='misp-modules server')
     argParser.add_argument('-t', default=False, action='store_true', help='Test mode')
     argParser.add_argument('-s', default=False, action='store_true', help='Run a system install (package installed via pip)')
@@ -180,12 +189,13 @@ def main():
     log = init_logger()
     if args.s:
         load_package_helpers()
-        mhandlers, modules = load_package_modules()
+        mhandlers, loaded_modules = load_package_modules()
     else:
-        modulesdir = 'misp_modules/modules'
+        os.chdir(os.path.dirname(__file__))
-        helpersdir = 'misp_modules/helpers'
+        modulesdir = 'modules'
+        helpersdir = 'helpers'
         load_helpers(helpersdir=helpersdir)
-        mhandlers, modules = load_modules(modulesdir)
+        mhandlers, loaded_modules = load_modules(modulesdir)
     service = [(r'/modules', ListModules), (r'/query', QueryModule)]
 
     application = tornado.web.Application(service)

misp_modules/modules/__init__.py

@@ -1 +1,3 @@
 from .expansion import *
+from .import_mod import *
+from .export_mod import *

misp_modules/modules/expansion/__init__.py

@@ -1,2 +1,2 @@
 __all__ = ['asn_history', 'circl_passivedns', 'circl_passivessl', 'cve', 'dns',
-           'eupi', 'ipasn', 'passivetotal', 'sourcecache']
+           'eupi', 'ipasn', 'passivetotal', 'sourcecache', 'whois']

misp_modules/modules/expansion/whois.py

@@ -1,7 +1,10 @@
 # -*- coding: utf-8 -*-
 
 import json
-from uwhois import Uwhois
+try:
+    from uwhois import Uwhois
+except ImportError:
+    print("uwhois module not installed.")
 
 misperrors = {'error': 'Error'}
 mispattributes = {'input': ['domain', 'ip-src', 'ip-dst'], 'output': ['freetext']}

misp_modules/modules/export_mod/__init__.py

@@ -0,0 +1 @@
+__all__ = ['testexport']

misp_modules/modules/import_mod/__init__.py

@@ -0,0 +1 @@
+__all__ = ['testimport', 'ocr']

setup.py

@@ -33,5 +33,7 @@ setup(
         'pyeupi',
         'ipasn-redis',
         'asnhistory',
+        'pillow',
+        'pytesseract',
     ]
 )