Make sure misp-modules can be launched from anywhere

README.md

@@ -5,7 +5,7 @@
 MISP modules are autonomous modules that can be used for expansion and other services in [MISP](https://github.com/MISP/MISP).
 
 The modules are written in Python 3 following a simple API interface. The objective is to ease the extensions of MISP functionalities
-without modifying core components. The API is available via a simple REST API which is independent from MISP installation or configuration. 
+without modifying core components. The API is available via a simple REST API which is independent from MISP installation or configuration.
 
 MISP modules support is included in MISP starting from version 2.4.28.
 
@@ -13,15 +13,15 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/
 
 ## Existing MISP modules
 
-* [ASN History](/modules/expansion/asn_history.py) - a hover and expansion module to expand an AS number with the ASN description and its history.
+* [ASN History](misp_modules/modules/expansion/asn_history.py) - a hover and expansion module to expand an AS number with the ASN description and its history.
-* [CIRCL Passive SSL](modules/expansion/circl_passivessl.py) - a hover and expansion module to expand IP addresses with the X.509 certificate seen.
+* [CIRCL Passive SSL](misp_modulesmodules/expansion/circl_passivessl.py) - a hover and expansion module to expand IP addresses with the X.509 certificate seen.
-* [CIRCL Passive DNS](modules/expansion/circl_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information.
+* [CIRCL Passive DNS](misp_modules/modules/expansion/circl_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information.
-* [CVE](modules/expansion/cve.py) - a hover module to give more information about a vulnerability (CVE).
+* [CVE](misp_modules/modules/expansion/cve.py) - a hover module to give more information about a vulnerability (CVE).
-* [DNS](modules/expansion/dns.py) - a simple module to resolve MISP attributes like hostname and domain to expand IP addresses attributes.
+* [DNS](misp_modules/modules/expansion/dns.py) - a simple module to resolve MISP attributes like hostname and domain to expand IP addresses attributes.
-* [EUPI](modules/expansion/eupi.py) - a hover and expansion module to get information about an URL from the [Phishing Initiative project](https://phishing-initiative.eu/?lang=en).
+* [EUPI](misp_modules/modules/expansion/eupi.py) - a hover and expansion module to get information about an URL from the [Phishing Initiative project](https://phishing-initiative.eu/?lang=en).
-* [IPASN](modules/expansion/ipasn.py) - a hover and expansion to get the BGP ASN of an IP address.
+* [IPASN](misp_modules/modules/expansion/ipasn.py) - a hover and expansion to get the BGP ASN of an IP address.
-* [passivetotal](modules/expansion/passivetotal.py) - a [passivetotal](https://www.passivetotal.org/) module that queries a number of different PassiveTotal datasets.
+* [passivetotal](misp_modules/modules/expansion/passivetotal.py) - a [passivetotal](https://www.passivetotal.org/) module that queries a number of different PassiveTotal datasets.
-* [sourcecache](modules/expansion/sourcecache.py) - a module to cache a specific link from a MISP instance.
+* [sourcecache](misp_modulesmisp_modules//modules/expansion/sourcecache.py) - a module to cache a specific link from a MISP instance.
 
 ## How to install and start MISP modules?
 
@@ -30,13 +30,12 @@ apt-get install python3-dev python3-pip libpq5
 git clone https://github.com/MISP/misp-modules.git
 cd misp-modules
 pip3 install -r REQUIREMENTS
-cd bin
+python3 bin/misp-modules
-python3 misp-modules.py
 ~~~~
 
 ## How to add your own MISP modules?
 
-Create your module in [modules/expansion/](modules/expansion/). The module should have at minimum three functions:
+Create your module in [misp_modules/modules/expansion/](misp_modules/modules/expansion/). The module should have at minimum three functions:
 
 * **introspection** function that returns a dict of the supported attributes (input and output) by your expansion module.
 * **handler** function which accepts a JSON document to expand the values and return a dictionary of the expanded values.

bin/misp-modules

@@ -29,6 +29,20 @@ import fnmatch
 import argparse
 import re
 
+try:
+    import misp_modules.modules
+    HAS_PACKAGE_MODULES = True
+except Exception as e:
+    print(e)
+    HAS_PACKAGE_MODULES = False
+
+try:
+    from misp_modules.helpers import *
+    HAS_PACKAGE_HELPERS = True
+except Exception as e:
+    print(e)
+    HAS_PACKAGE_HELPERS = False
+
 
 def init_logger():
     log = logging.getLogger('misp-modules')
@@ -64,6 +78,22 @@ def load_helpers(helpersdir):
                 log.info('Helpers failed {} due to {}'.format(filename, selftest))
 
 
+def load_package_helpers():
+    if not HAS_PACKAGE_HELPERS:
+        log.info('Unable to load MISP helpers from package.')
+        sys.exit()
+    mhandlers = {}
+    helpers = []
+    for path, helper in sys.modules.items():
+        if not path.startswith('misp_modules.helpers.'):
+            continue
+        helpername = path.replace('misp_modules.helpers.', '')
+        mhandlers[helpername] = helper
+        helpers.append(helpername)
+        log.info('Helper loaded {}'.format(helpername))
+    return mhandlers, helpers
+
+
 def load_modules(mod_dir):
     sys.path.append(mod_dir)
     mhandlers = {}
@@ -89,6 +119,23 @@ def load_modules(mod_dir):
     return mhandlers, modules
 
 
+def load_package_modules():
+    if not HAS_PACKAGE_MODULES:
+        log.info('Unable to load MISP modules from package.')
+        sys.exit()
+    mhandlers = {}
+    modules = []
+    for path, module in sys.modules.items():
+        r = re.findall("misp_modules[.]modules[.](\w+)[.](\w+)", path)
+        if r and len(r[0]) == 2:
+            moduletype, modulename = r[0]
+            mhandlers[modulename] = module
+            modules.append(modulename)
+            log.info('MISP modules {0} imported'.format(modulename))
+            mhandlers['type:' + modulename] = moduletype
+    return mhandlers, modules
+
+
 class ListModules(tornado.web.RequestHandler):
     def get(self):
         ret = []
@@ -117,16 +164,21 @@ if __name__ == '__main__':
         os.chdir('../')
     argParser = argparse.ArgumentParser(description='misp-modules server')
     argParser.add_argument('-t', default=False, action='store_true', help='Test mode')
+    argParser.add_argument('-s', default=False, action='store_true', help='Run a system install (package installed via pip)')
     argParser.add_argument('-p', default=6666, help='misp-modules TCP port (default 6666)')
     argParser.add_argument('-l', default='localhost', help='misp-modules listen address (default localhost)')
     args = argParser.parse_args()
     port = args.p
     listen = args.l
-    modulesdir = 'modules'
-    helpersdir = 'helpers'
     log = init_logger()
-    load_helpers(helpersdir=helpersdir)
+    if args.s:
-    mhandlers, modules = load_modules(modulesdir)
+        load_package_helpers()
+        mhandlers, modules = load_package_modules()
+    else:
+        modulesdir = 'misp_modules/modules'
+        helpersdir = 'misp_modules/helpers'
+        load_helpers(helpersdir=helpersdir)
+        mhandlers, modules = load_modules(modulesdir)
     service = [(r'/modules', ListModules), (r'/query', QueryModule)]
 
     application = tornado.web.Application(service)

misp_modules/helpers/__init__.py

@@ -0,0 +1 @@
+__all__ = ['cache']

misp_modules/modules/__init__.py

@@ -0,0 +1 @@
+from .expansion import *

misp_modules/modules/expansion/__init__.py

@@ -0,0 +1,2 @@
+__all__ = ['asn_history', 'circl_passivedns', 'circl_passivessl', 'cve', 'dns',
+           'eupi', 'ipasn', 'passivetotal', 'sourcecache']

setup.py

@@ -1,6 +1,6 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
-from setuptools import setup
+from setuptools import setup, find_packages
 
 setup(
     name='misp-modules',
@@ -10,8 +10,8 @@ setup(
     maintainer='Alexandre Dulaunoy',
     url='https://github.com/MISP/misp-modules',
     description='MISP modules are autonomous modules that can be used for expansion and other services in MISP',
-    packages=['modules', 'helpers'],
+    packages=find_packages(),
-    scripts=['bin/misp-modules.py'],
+    scripts=['bin/misp-modules'],
     test_suite="tests",
     classifiers=[
         'License :: OSI Approved :: GNU Affero General Public License v3',