Merge pull request #28 from Rafiot/pip

Make it a package

.gitignore

@@ -1,3 +1,6 @@
 *.pyc
 *.swp
 __pycache__
+build/
+dist/
+misp_modules.egg-info/

.travis.yml

@@ -17,9 +17,16 @@ python:
     - "nightly"
 
 install:
-    - pip install -r REQUIREMENTS
+    - python setup.py install
 
 script:
-    - pushd bin
+    - misp-modules &
-    - ./misp-modules.py -t
+    - sleep 15
+    - python setup.py test
+    - pkill misp-modules
+    - pushd ~/
+    - misp-modules -s &
     - popd
+    - sleep 15
+    - python setup.py test
+    - pkill misp-modules

README.md

@@ -5,7 +5,7 @@
 MISP modules are autonomous modules that can be used for expansion and other services in [MISP](https://github.com/MISP/MISP).
 
 The modules are written in Python 3 following a simple API interface. The objective is to ease the extensions of MISP functionalities
-without modifying core components. The API is available via a simple REST API which is independent from MISP installation or configuration. 
+without modifying core components. The API is available via a simple REST API which is independent from MISP installation or configuration.
 
 MISP modules support is included in MISP starting from version 2.4.28.
 
@@ -13,15 +13,15 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/
 
 ## Existing MISP modules
 
-* [ASN History](/modules/expansion/asn_history.py) - a hover and expansion module to expand an AS number with the ASN description and its history.
+* [ASN History](misp_modules/modules/expansion/asn_history.py) - a hover and expansion module to expand an AS number with the ASN description and its history.
-* [CIRCL Passive SSL](modules/expansion/circl_passivessl.py) - a hover and expansion module to expand IP addresses with the X.509 certificate seen.
+* [CIRCL Passive SSL](misp_modules/modules/expansion/circl_passivessl.py) - a hover and expansion module to expand IP addresses with the X.509 certificate seen.
-* [CIRCL Passive DNS](modules/expansion/circl_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information.
+* [CIRCL Passive DNS](misp_modules/modules/expansion/circl_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information.
-* [CVE](modules/expansion/cve.py) - a hover module to give more information about a vulnerability (CVE).
+* [CVE](misp_modules/modules/expansion/cve.py) - a hover module to give more information about a vulnerability (CVE).
-* [DNS](modules/expansion/dns.py) - a simple module to resolve MISP attributes like hostname and domain to expand IP addresses attributes.
+* [DNS](misp_modules/modules/expansion/dns.py) - a simple module to resolve MISP attributes like hostname and domain to expand IP addresses attributes.
-* [EUPI](modules/expansion/eupi.py) - a hover and expansion module to get information about an URL from the [Phishing Initiative project](https://phishing-initiative.eu/?lang=en).
+* [EUPI](misp_modules/modules/expansion/eupi.py) - a hover and expansion module to get information about an URL from the [Phishing Initiative project](https://phishing-initiative.eu/?lang=en).
-* [IPASN](modules/expansion/ipasn.py) - a hover and expansion to get the BGP ASN of an IP address.
+* [IPASN](misp_modules/modules/expansion/ipasn.py) - a hover and expansion to get the BGP ASN of an IP address.
-* [passivetotal](modules/expansion/passivetotal.py) - a [passivetotal](https://www.passivetotal.org/) module that queries a number of different PassiveTotal datasets.
+* [passivetotal](misp_modules/modules/expansion/passivetotal.py) - a [passivetotal](https://www.passivetotal.org/) module that queries a number of different PassiveTotal datasets.
-* [sourcecache](modules/expansion/sourcecache.py) - a module to cache a specific link from a MISP instance.
+* [sourcecache](misp_modules/modules/expansion/sourcecache.py) - a module to cache a specific link from a MISP instance.
 
 ## How to install and start MISP modules?
 
@@ -30,13 +30,12 @@ apt-get install python3-dev python3-pip libpq5
 git clone https://github.com/MISP/misp-modules.git
 cd misp-modules
 pip3 install -r REQUIREMENTS
-cd bin
+python3 bin/misp-modules
-python3 misp-modules.py
 ~~~~
 
 ## How to add your own MISP modules?
 
-Create your module in [modules/expansion/](modules/expansion/). The module should have at minimum three functions:
+Create your module in [misp_modules/modules/expansion/](misp_modules/modules/expansion/). The module should have at minimum three functions:
 
 * **introspection** function that returns a dict of the supported attributes (input and output) by your expansion module.
 * **handler** function which accepts a JSON document to expand the values and return a dictionary of the expanded values.

README.rst

@@ -0,0 +1 @@
+README.md

bin/misp-modules

@@ -29,6 +29,21 @@ import fnmatch
 import argparse
 import re
 
+try:
+    import misp_modules.modules
+    HAS_PACKAGE_MODULES = True
+except Exception as e:
+    print(e)
+    HAS_PACKAGE_MODULES = False
+
+try:
+    from misp_modules.helpers import *
+    HAS_PACKAGE_HELPERS = True
+except Exception as e:
+    print(e)
+    HAS_PACKAGE_HELPERS = False
+
+
 def init_logger():
     log = logging.getLogger('misp-modules')
     formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
@@ -41,7 +56,7 @@ def init_logger():
     return log
 
 
-def load_helpers(helpersdir='../helpers'):
+def load_helpers(helpersdir):
     sys.path.append(helpersdir)
     hhandlers = {}
     helpers = []
@@ -51,9 +66,11 @@ def load_helpers(helpersdir='../helpers'):
         if re.match(r'^\.', os.path.basename(root)):
             continue
         for filename in fnmatch.filter(filenames, '*.py'):
+            if filename == '__init__.py':
+                continue
             helpername = filename.split(".")[0]
             hhandlers[helpername] = importlib.import_module(helpername)
-            selftest= hhandlers[helpername].selftest()
+            selftest = hhandlers[helpername].selftest()
             if selftest is None:
                 helpers.append(helpername)
                 log.info('Helpers loaded {} '.format(filename))
@@ -61,6 +78,22 @@ def load_helpers(helpersdir='../helpers'):
                 log.info('Helpers failed {} due to {}'.format(filename, selftest))
 
 
+def load_package_helpers():
+    if not HAS_PACKAGE_HELPERS:
+        log.info('Unable to load MISP helpers from package.')
+        sys.exit()
+    mhandlers = {}
+    helpers = []
+    for path, helper in sys.modules.items():
+        if not path.startswith('misp_modules.helpers.'):
+            continue
+        helpername = path.replace('misp_modules.helpers.', '')
+        mhandlers[helpername] = helper
+        helpers.append(helpername)
+        log.info('Helper loaded {}'.format(helpername))
+    return mhandlers, helpers
+
+
 def load_modules(mod_dir):
     sys.path.append(mod_dir)
     mhandlers = {}
@@ -86,6 +119,23 @@ def load_modules(mod_dir):
     return mhandlers, modules
 
 
+def load_package_modules():
+    if not HAS_PACKAGE_MODULES:
+        log.info('Unable to load MISP modules from package.')
+        sys.exit()
+    mhandlers = {}
+    modules = []
+    for path, module in sys.modules.items():
+        r = re.findall("misp_modules[.]modules[.](\w+)[.](\w+)", path)
+        if r and len(r[0]) == 2:
+            moduletype, modulename = r[0]
+            mhandlers[modulename] = module
+            modules.append(modulename)
+            log.info('MISP modules {0} imported'.format(modulename))
+            mhandlers['type:' + modulename] = moduletype
+    return mhandlers, modules
+
+
 class ListModules(tornado.web.RequestHandler):
     def get(self):
         ret = []
@@ -110,20 +160,25 @@ class QueryModule(tornado.web.RequestHandler):
 
 
 if __name__ == '__main__':
-    if os.path.dirname(__file__) is not '':
+    if os.path.dirname(__file__) is '.':
-        os.chdir(os.path.dirname(__file__))
+        os.chdir('../')
     argParser = argparse.ArgumentParser(description='misp-modules server')
     argParser.add_argument('-t', default=False, action='store_true', help='Test mode')
+    argParser.add_argument('-s', default=False, action='store_true', help='Run a system install (package installed via pip)')
     argParser.add_argument('-p', default=6666, help='misp-modules TCP port (default 6666)')
     argParser.add_argument('-l', default='localhost', help='misp-modules listen address (default localhost)')
     args = argParser.parse_args()
     port = args.p
     listen = args.l
-    modulesdir = '../modules'
-    helpersdir = '../helpers'
     log = init_logger()
-    load_helpers(helpersdir=helpersdir)
+    if args.s:
-    mhandlers, modules = load_modules(modulesdir)
+        load_package_helpers()
+        mhandlers, modules = load_package_modules()
+    else:
+        modulesdir = 'misp_modules/modules'
+        helpersdir = 'misp_modules/helpers'
+        load_helpers(helpersdir=helpersdir)
+        mhandlers, modules = load_modules(modulesdir)
     service = [(r'/modules', ListModules), (r'/query', QueryModule)]
 
     application = tornado.web.Application(service)

misp_modules/helpers/__init__.py

@@ -0,0 +1 @@
+__all__ = ['cache']

misp_modules/helpers/cache.py

@@ -32,7 +32,7 @@ def selftest(enable=True):
         return False
     r = redis.StrictRedis(host=hostname, port=port, db=db)
     try:
-        r.set('test', 'selftest')
+        r.ping()
     except:
         return 'Redis not running or not installed. Helper will be disabled.'
 
@@ -44,16 +44,15 @@ def get(modulename=None, query=None, value=None, debug=False):
     h = hashlib.sha1()
     h.update(query.encode('UTF-8'))
     hv = h.hexdigest()
-    key = "m:"+modulename+":"+hv
+    key = "m:" + modulename + ":" + hv
 
     if not r.exists(key):
         if debug:
-            print ("Key {} added in cache".format(key))
+            print("Key {} added in cache".format(key))
-        r.set(key, value)
+        r.setex(key, 86400, value)
-        r.expire(key, 86400)
     else:
         if debug:
-            print ("Cache hit with Key {}".format(key))
+            print("Cache hit with Key {}".format(key))
 
     return r.get(key)
 
@@ -68,14 +67,14 @@ if __name__ == "__main__":
     if selftest() is not None:
         sys.exit()
     else:
-        print ("Selftest ok")
+        print("Selftest ok")
     v = get(modulename="testmodule", query="abcdef", value="barfoo", debug=True)
     if v == b'barfoo':
-        print ("Cache ok")
+        print("Cache ok")
     v = get(modulename="testmodule", query="abcdef")
-    print (v)
+    print(v)
     v = get(modulename="testmodule")
     if (not v):
-        print ("Failed ok")
+        print("Failed ok")
     if flush():
-        print ("Cache flushed ok")
+        print("Cache flushed ok")

misp_modules/modules/__init__.py

@@ -0,0 +1 @@
+from .expansion import *

misp_modules/modules/expansion/__init__.py

@@ -0,0 +1,2 @@
+__all__ = ['asn_history', 'circl_passivedns', 'circl_passivessl', 'cve', 'dns',
+           'eupi', 'ipasn', 'passivetotal', 'sourcecache']

misp_modules/modules/expansion/cve.py

@@ -11,7 +11,6 @@ cveapi_url = 'https://cve.circl.lu/api/cve/'
 def handler(q=False):
     if q is False:
         return False
-    print (q)
     request = json.loads(q)
     if not request.get('vulnerability'):
         misperrors['error'] = 'Vulnerability id missing'

setup.py

@@ -0,0 +1,37 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+from setuptools import setup, find_packages
+
+setup(
+    name='misp-modules',
+    version='1.0',
+    author='Alexandre Dulaunoy',
+    author_email='alexandre.dulaunoy@circl.lu',
+    maintainer='Alexandre Dulaunoy',
+    url='https://github.com/MISP/misp-modules',
+    description='MISP modules are autonomous modules that can be used for expansion and other services in MISP',
+    packages=find_packages(),
+    scripts=['bin/misp-modules'],
+    test_suite="tests",
+    classifiers=[
+        'License :: OSI Approved :: GNU Affero General Public License v3',
+        'Development Status :: 5 - Production/Stable',
+        'Environment :: Console',
+        'Intended Audience :: Science/Research',
+        'Programming Language :: Python :: 3',
+        'Topic :: Security',
+    ],
+    install_requires=[
+        'tornado',
+        'dnspython3',
+        'requests',
+        'urlarchiver',
+        'passivetotal',
+        'PyPDNS',
+        'pypssl',
+        'redis',
+        'pyeupi',
+        'ipasn-redis',
+        'asnhistory',
+    ]
+)

tests/test.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import unittest
+import requests
+
+
+class TestModules(unittest.TestCase):
+
+    def setUp(self):
+        self.maxDiff = None
+        self.headers = {'Content-Type': 'application/json'}
+
+    def test_introspection(self):
+        response = requests.get('http://127.0.0.1:6666/modules')
+        print(response.json())
+
+    def test_cve(self):
+        with open('tests/bodycve.json', 'r') as f:
+            response = requests.post('http://127.0.0.1:6666/query', data=f.read())
+            print(response.json())
+
+    def test_dns(self):
+        with open('tests/body.json', 'r') as f:
+            response = requests.post('http://127.0.0.1:6666/query', data=f.read())
+            print(response.json())