From 0752c5e3f4f27fe528072dbe5728436b87f117f8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= <raphael@vinot.info>
Date: Sat, 18 Jun 2016 11:04:41 +0900
Subject: [PATCH 1/6] Make it a package

---
 .gitignore                      |  3 +++
 README.rst                      |  1 +
 bin/misp-modules.py             | 15 +++++++------
 helpers/__init__.py             |  0
 helpers/cache.py                | 21 +++++++++----------
 modules/expansion/cve.py        |  1 -
 setup.py                        | 37 +++++++++++++++++++++++++++++++++
 tests/query-circl_passivedns.sh |  0
 tests/query-circl_passivessl.sh |  0
 tests/query-cve.sh              |  0
 tests/query-dns.sh              |  0
 tests/query-sourcecache.sh      |  0
 tests/search-modules.sh         |  0
 13 files changed, 60 insertions(+), 18 deletions(-)
 create mode 120000 README.rst
 create mode 100644 helpers/__init__.py
 create mode 100644 setup.py
 mode change 100644 => 100755 tests/query-circl_passivedns.sh
 mode change 100644 => 100755 tests/query-circl_passivessl.sh
 mode change 100644 => 100755 tests/query-cve.sh
 mode change 100644 => 100755 tests/query-dns.sh
 mode change 100644 => 100755 tests/query-sourcecache.sh
 mode change 100644 => 100755 tests/search-modules.sh

diff --git a/.gitignore b/.gitignore
index e69364e..9676396 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,6 @@
 *.pyc
 *.swp
 __pycache__
+build/
+dist/
+misp_modules.egg-info/
diff --git a/README.rst b/README.rst
new file mode 120000
index 0000000..42061c0
--- /dev/null
+++ b/README.rst
@@ -0,0 +1 @@
+README.md
\ No newline at end of file
diff --git a/bin/misp-modules.py b/bin/misp-modules.py
index f5adc83..2eb38d3 100755
--- a/bin/misp-modules.py
+++ b/bin/misp-modules.py
@@ -29,6 +29,7 @@ import fnmatch
 import argparse
 import re
 
+
 def init_logger():
     log = logging.getLogger('misp-modules')
     formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
@@ -41,7 +42,7 @@ def init_logger():
     return log
 
 
-def load_helpers(helpersdir='../helpers'):
+def load_helpers(helpersdir):
     sys.path.append(helpersdir)
     hhandlers = {}
     helpers = []
@@ -51,9 +52,11 @@ def load_helpers(helpersdir='../helpers'):
         if re.match(r'^\.', os.path.basename(root)):
             continue
         for filename in fnmatch.filter(filenames, '*.py'):
+            if filename == '__init__.py':
+                continue
             helpername = filename.split(".")[0]
             hhandlers[helpername] = importlib.import_module(helpername)
-            selftest= hhandlers[helpername].selftest()
+            selftest = hhandlers[helpername].selftest()
             if selftest is None:
                 helpers.append(helpername)
                 log.info('Helpers loaded {} '.format(filename))
@@ -110,8 +113,8 @@ class QueryModule(tornado.web.RequestHandler):
 
 
 if __name__ == '__main__':
-    if os.path.dirname(__file__) is not '':
-        os.chdir(os.path.dirname(__file__))
+    if os.path.dirname(__file__) is '.':
+        os.chdir('../')
     argParser = argparse.ArgumentParser(description='misp-modules server')
     argParser.add_argument('-t', default=False, action='store_true', help='Test mode')
     argParser.add_argument('-p', default=6666, help='misp-modules TCP port (default 6666)')
@@ -119,8 +122,8 @@ if __name__ == '__main__':
     args = argParser.parse_args()
     port = args.p
     listen = args.l
-    modulesdir = '../modules'
-    helpersdir = '../helpers'
+    modulesdir = 'modules'
+    helpersdir = 'helpers'
     log = init_logger()
     load_helpers(helpersdir=helpersdir)
     mhandlers, modules = load_modules(modulesdir)
diff --git a/helpers/__init__.py b/helpers/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/helpers/cache.py b/helpers/cache.py
index da43cd8..7d70159 100644
--- a/helpers/cache.py
+++ b/helpers/cache.py
@@ -32,7 +32,7 @@ def selftest(enable=True):
         return False
     r = redis.StrictRedis(host=hostname, port=port, db=db)
     try:
-        r.set('test', 'selftest')
+        r.ping()
     except:
         return 'Redis not running or not installed. Helper will be disabled.'
 
@@ -44,16 +44,15 @@ def get(modulename=None, query=None, value=None, debug=False):
     h = hashlib.sha1()
     h.update(query.encode('UTF-8'))
     hv = h.hexdigest()
-    key = "m:"+modulename+":"+hv
+    key = "m:" + modulename + ":" + hv
 
     if not r.exists(key):
         if debug:
-            print ("Key {} added in cache".format(key))
-        r.set(key, value)
-        r.expire(key, 86400)
+            print("Key {} added in cache".format(key))
+        r.setex(key, 86400, value)
     else:
         if debug:
-            print ("Cache hit with Key {}".format(key))
+            print("Cache hit with Key {}".format(key))
 
     return r.get(key)
 
@@ -68,14 +67,14 @@ if __name__ == "__main__":
     if selftest() is not None:
         sys.exit()
     else:
-        print ("Selftest ok")
+        print("Selftest ok")
     v = get(modulename="testmodule", query="abcdef", value="barfoo", debug=True)
     if v == b'barfoo':
-        print ("Cache ok")
+        print("Cache ok")
     v = get(modulename="testmodule", query="abcdef")
-    print (v)
+    print(v)
     v = get(modulename="testmodule")
     if (not v):
-        print ("Failed ok")
+        print("Failed ok")
     if flush():
-        print ("Cache flushed ok")
+        print("Cache flushed ok")
diff --git a/modules/expansion/cve.py b/modules/expansion/cve.py
index 48e8f02..e370116 100755
--- a/modules/expansion/cve.py
+++ b/modules/expansion/cve.py
@@ -11,7 +11,6 @@ cveapi_url = 'https://cve.circl.lu/api/cve/'
 def handler(q=False):
     if q is False:
         return False
-    print (q)
     request = json.loads(q)
     if not request.get('vulnerability'):
         misperrors['error'] = 'Vulnerability id missing'
diff --git a/setup.py b/setup.py
new file mode 100644
index 0000000..d9d0f79
--- /dev/null
+++ b/setup.py
@@ -0,0 +1,37 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+from setuptools import setup
+
+setup(
+    name='misp-modules',
+    version='1.0',
+    author='Alexandre Dulaunoy',
+    author_email='alexandre.dulaunoy@circl.lu',
+    maintainer='Alexandre Dulaunoy',
+    url='https://github.com/MISP/misp-modules',
+    description='MISP modules are autonomous modules that can be used for expansion and other services in MISP',
+    packages=['modules', 'helpers'],
+    scripts=['bin/misp-modules.py'],
+    test_suite="tests",
+    classifiers=[
+        'License :: OSI Approved :: GNU Affero General Public License v3',
+        'Development Status :: 5 - Production/Stable',
+        'Environment :: Console',
+        'Intended Audience :: Science/Research',
+        'Programming Language :: Python :: 3',
+        'Topic :: Security',
+    ],
+    install_requires=[
+        'tornado',
+        'dnspython3',
+        'requests',
+        'urlarchiver',
+        'passivetotal',
+        'PyPDNS',
+        'pypssl',
+        'redis',
+        'pyeupi',
+        'ipasn-redis',
+        'asnhistory',
+    ]
+)
diff --git a/tests/query-circl_passivedns.sh b/tests/query-circl_passivedns.sh
old mode 100644
new mode 100755
diff --git a/tests/query-circl_passivessl.sh b/tests/query-circl_passivessl.sh
old mode 100644
new mode 100755
diff --git a/tests/query-cve.sh b/tests/query-cve.sh
old mode 100644
new mode 100755
diff --git a/tests/query-dns.sh b/tests/query-dns.sh
old mode 100644
new mode 100755
diff --git a/tests/query-sourcecache.sh b/tests/query-sourcecache.sh
old mode 100644
new mode 100755
diff --git a/tests/search-modules.sh b/tests/search-modules.sh
old mode 100644
new mode 100755

From ef6e3b27f81bee3b42a0971276c5377e17323f38 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= <raphael@vinot.info>
Date: Sat, 18 Jun 2016 14:53:26 +0900
Subject: [PATCH 2/6] Proper testcases

---
 .travis.yml       |  9 +++++----
 tests/__init__.py |  0
 tests/test.py     | 26 ++++++++++++++++++++++++++
 3 files changed, 31 insertions(+), 4 deletions(-)
 create mode 100644 tests/__init__.py
 create mode 100755 tests/test.py

diff --git a/.travis.yml b/.travis.yml
index d9d892f..abef1f5 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -17,9 +17,10 @@ python:
     - "nightly"
 
 install:
-    - pip install -r REQUIREMENTS
+    - python setup.py install
 
 script:
-    - pushd bin
-    - ./misp-modules.py -t
-    - popd
+    - misp-modules.py &
+    - sleep 15
+    - python setup.py test
+    - pkill misp-modules.py
diff --git a/tests/__init__.py b/tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/test.py b/tests/test.py
new file mode 100755
index 0000000..4732358
--- /dev/null
+++ b/tests/test.py
@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import unittest
+import requests
+
+
+class TestModules(unittest.TestCase):
+
+    def setUp(self):
+        self.maxDiff = None
+        self.headers = {'Content-Type': 'application/json'}
+
+    def test_introspection(self):
+        response = requests.get('http://127.0.0.1:6666/modules')
+        print(response.json())
+
+    def test_cve(self):
+        with open('tests/bodycve.json', 'r') as f:
+            response = requests.post('http://127.0.0.1:6666/query', data=f.read())
+            print(response.json())
+
+    def test_dns(self):
+        with open('tests/body.json', 'r') as f:
+            response = requests.post('http://127.0.0.1:6666/query', data=f.read())
+            print(response.json())

From 22eaba6ab62c55632acfc0005f003370c5315757 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= <raphael@vinot.info>
Date: Thu, 23 Jun 2016 19:51:13 +0900
Subject: [PATCH 3/6] Make sure misp-modules can be launched from anywhere

---
 README.md                                     | 25 ++++----
 bin/{misp-modules.py => misp-modules}         | 60 +++++++++++++++++--
 {helpers => misp_modules}/__init__.py         |  0
 misp_modules/helpers/__init__.py              |  1 +
 {helpers => misp_modules/helpers}/cache.py    |  0
 misp_modules/modules/__init__.py              |  1 +
 misp_modules/modules/expansion/__init__.py    |  2 +
 .../modules}/expansion/asn_history.py         |  0
 .../modules}/expansion/circl_passivedns.py    |  0
 .../modules}/expansion/circl_passivessl.py    |  0
 .../modules}/expansion/cve.py                 |  0
 .../modules}/expansion/dns.py                 |  0
 .../modules}/expansion/eupi.py                |  0
 .../expansion/intelmq_eventdb.py.experimental |  0
 .../modules}/expansion/ipasn.py               |  0
 .../modules}/expansion/passivetotal.py        |  0
 .../modules}/expansion/sourcecache.py         |  0
 modules/__init__.py                           |  0
 modules/expansion/__init__.py                 |  0
 setup.py                                      |  6 +-
 20 files changed, 75 insertions(+), 20 deletions(-)
 rename bin/{misp-modules.py => misp-modules} (72%)
 rename {helpers => misp_modules}/__init__.py (100%)
 create mode 100644 misp_modules/helpers/__init__.py
 rename {helpers => misp_modules/helpers}/cache.py (100%)
 create mode 100644 misp_modules/modules/__init__.py
 create mode 100644 misp_modules/modules/expansion/__init__.py
 rename {modules => misp_modules/modules}/expansion/asn_history.py (100%)
 rename {modules => misp_modules/modules}/expansion/circl_passivedns.py (100%)
 rename {modules => misp_modules/modules}/expansion/circl_passivessl.py (100%)
 rename {modules => misp_modules/modules}/expansion/cve.py (100%)
 rename {modules => misp_modules/modules}/expansion/dns.py (100%)
 rename {modules => misp_modules/modules}/expansion/eupi.py (100%)
 rename {modules => misp_modules/modules}/expansion/intelmq_eventdb.py.experimental (100%)
 rename {modules => misp_modules/modules}/expansion/ipasn.py (100%)
 rename {modules => misp_modules/modules}/expansion/passivetotal.py (100%)
 rename {modules => misp_modules/modules}/expansion/sourcecache.py (100%)
 delete mode 100644 modules/__init__.py
 delete mode 100644 modules/expansion/__init__.py

diff --git a/README.md b/README.md
index 3e2036d..7ca3746 100644
--- a/README.md
+++ b/README.md
@@ -5,7 +5,7 @@
 MISP modules are autonomous modules that can be used for expansion and other services in [MISP](https://github.com/MISP/MISP).
 
 The modules are written in Python 3 following a simple API interface. The objective is to ease the extensions of MISP functionalities
-without modifying core components. The API is available via a simple REST API which is independent from MISP installation or configuration. 
+without modifying core components. The API is available via a simple REST API which is independent from MISP installation or configuration.
 
 MISP modules support is included in MISP starting from version 2.4.28.
 
@@ -13,15 +13,15 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/
 
 ## Existing MISP modules
 
-* [ASN History](/modules/expansion/asn_history.py) - a hover and expansion module to expand an AS number with the ASN description and its history.
-* [CIRCL Passive SSL](modules/expansion/circl_passivessl.py) - a hover and expansion module to expand IP addresses with the X.509 certificate seen.
-* [CIRCL Passive DNS](modules/expansion/circl_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information.
-* [CVE](modules/expansion/cve.py) - a hover module to give more information about a vulnerability (CVE).
-* [DNS](modules/expansion/dns.py) - a simple module to resolve MISP attributes like hostname and domain to expand IP addresses attributes.
-* [EUPI](modules/expansion/eupi.py) - a hover and expansion module to get information about an URL from the [Phishing Initiative project](https://phishing-initiative.eu/?lang=en).
-* [IPASN](modules/expansion/ipasn.py) - a hover and expansion to get the BGP ASN of an IP address.
-* [passivetotal](modules/expansion/passivetotal.py) - a [passivetotal](https://www.passivetotal.org/) module that queries a number of different PassiveTotal datasets.
-* [sourcecache](modules/expansion/sourcecache.py) - a module to cache a specific link from a MISP instance.
+* [ASN History](misp_modules/modules/expansion/asn_history.py) - a hover and expansion module to expand an AS number with the ASN description and its history.
+* [CIRCL Passive SSL](misp_modulesmodules/expansion/circl_passivessl.py) - a hover and expansion module to expand IP addresses with the X.509 certificate seen.
+* [CIRCL Passive DNS](misp_modules/modules/expansion/circl_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information.
+* [CVE](misp_modules/modules/expansion/cve.py) - a hover module to give more information about a vulnerability (CVE).
+* [DNS](misp_modules/modules/expansion/dns.py) - a simple module to resolve MISP attributes like hostname and domain to expand IP addresses attributes.
+* [EUPI](misp_modules/modules/expansion/eupi.py) - a hover and expansion module to get information about an URL from the [Phishing Initiative project](https://phishing-initiative.eu/?lang=en).
+* [IPASN](misp_modules/modules/expansion/ipasn.py) - a hover and expansion to get the BGP ASN of an IP address.
+* [passivetotal](misp_modules/modules/expansion/passivetotal.py) - a [passivetotal](https://www.passivetotal.org/) module that queries a number of different PassiveTotal datasets.
+* [sourcecache](misp_modulesmisp_modules//modules/expansion/sourcecache.py) - a module to cache a specific link from a MISP instance.
 
 ## How to install and start MISP modules?
 
@@ -30,13 +30,12 @@ apt-get install python3-dev python3-pip libpq5
 git clone https://github.com/MISP/misp-modules.git
 cd misp-modules
 pip3 install -r REQUIREMENTS
-cd bin
-python3 misp-modules.py
+python3 bin/misp-modules
 ~~~~
 
 ## How to add your own MISP modules?
 
-Create your module in [modules/expansion/](modules/expansion/). The module should have at minimum three functions:
+Create your module in [misp_modules/modules/expansion/](misp_modules/modules/expansion/). The module should have at minimum three functions:
 
 * **introspection** function that returns a dict of the supported attributes (input and output) by your expansion module.
 * **handler** function which accepts a JSON document to expand the values and return a dictionary of the expanded values.
diff --git a/bin/misp-modules.py b/bin/misp-modules
similarity index 72%
rename from bin/misp-modules.py
rename to bin/misp-modules
index 2eb38d3..d25d7dd 100755
--- a/bin/misp-modules.py
+++ b/bin/misp-modules
@@ -29,6 +29,20 @@ import fnmatch
 import argparse
 import re
 
+try:
+    import misp_modules.modules
+    HAS_PACKAGE_MODULES = True
+except Exception as e:
+    print(e)
+    HAS_PACKAGE_MODULES = False
+
+try:
+    from misp_modules.helpers import *
+    HAS_PACKAGE_HELPERS = True
+except Exception as e:
+    print(e)
+    HAS_PACKAGE_HELPERS = False
+
 
 def init_logger():
     log = logging.getLogger('misp-modules')
@@ -64,6 +78,22 @@ def load_helpers(helpersdir):
                 log.info('Helpers failed {} due to {}'.format(filename, selftest))
 
 
+def load_package_helpers():
+    if not HAS_PACKAGE_HELPERS:
+        log.info('Unable to load MISP helpers from package.')
+        sys.exit()
+    mhandlers = {}
+    helpers = []
+    for path, helper in sys.modules.items():
+        if not path.startswith('misp_modules.helpers.'):
+            continue
+        helpername = path.replace('misp_modules.helpers.', '')
+        mhandlers[helpername] = helper
+        helpers.append(helpername)
+        log.info('Helper loaded {}'.format(helpername))
+    return mhandlers, helpers
+
+
 def load_modules(mod_dir):
     sys.path.append(mod_dir)
     mhandlers = {}
@@ -89,6 +119,23 @@ def load_modules(mod_dir):
     return mhandlers, modules
 
 
+def load_package_modules():
+    if not HAS_PACKAGE_MODULES:
+        log.info('Unable to load MISP modules from package.')
+        sys.exit()
+    mhandlers = {}
+    modules = []
+    for path, module in sys.modules.items():
+        r = re.findall("misp_modules[.]modules[.](\w+)[.](\w+)", path)
+        if r and len(r[0]) == 2:
+            moduletype, modulename = r[0]
+            mhandlers[modulename] = module
+            modules.append(modulename)
+            log.info('MISP modules {0} imported'.format(modulename))
+            mhandlers['type:' + modulename] = moduletype
+    return mhandlers, modules
+
+
 class ListModules(tornado.web.RequestHandler):
     def get(self):
         ret = []
@@ -117,16 +164,21 @@ if __name__ == '__main__':
         os.chdir('../')
     argParser = argparse.ArgumentParser(description='misp-modules server')
     argParser.add_argument('-t', default=False, action='store_true', help='Test mode')
+    argParser.add_argument('-s', default=False, action='store_true', help='Run a system install (package installed via pip)')
     argParser.add_argument('-p', default=6666, help='misp-modules TCP port (default 6666)')
     argParser.add_argument('-l', default='localhost', help='misp-modules listen address (default localhost)')
     args = argParser.parse_args()
     port = args.p
     listen = args.l
-    modulesdir = 'modules'
-    helpersdir = 'helpers'
     log = init_logger()
-    load_helpers(helpersdir=helpersdir)
-    mhandlers, modules = load_modules(modulesdir)
+    if args.s:
+        load_package_helpers()
+        mhandlers, modules = load_package_modules()
+    else:
+        modulesdir = 'misp_modules/modules'
+        helpersdir = 'misp_modules/helpers'
+        load_helpers(helpersdir=helpersdir)
+        mhandlers, modules = load_modules(modulesdir)
     service = [(r'/modules', ListModules), (r'/query', QueryModule)]
 
     application = tornado.web.Application(service)
diff --git a/helpers/__init__.py b/misp_modules/__init__.py
similarity index 100%
rename from helpers/__init__.py
rename to misp_modules/__init__.py
diff --git a/misp_modules/helpers/__init__.py b/misp_modules/helpers/__init__.py
new file mode 100644
index 0000000..cd16c88
--- /dev/null
+++ b/misp_modules/helpers/__init__.py
@@ -0,0 +1 @@
+__all__ = ['cache']
diff --git a/helpers/cache.py b/misp_modules/helpers/cache.py
similarity index 100%
rename from helpers/cache.py
rename to misp_modules/helpers/cache.py
diff --git a/misp_modules/modules/__init__.py b/misp_modules/modules/__init__.py
new file mode 100644
index 0000000..f0f4cdb
--- /dev/null
+++ b/misp_modules/modules/__init__.py
@@ -0,0 +1 @@
+from .expansion import *
diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py
new file mode 100644
index 0000000..5854e40
--- /dev/null
+++ b/misp_modules/modules/expansion/__init__.py
@@ -0,0 +1,2 @@
+__all__ = ['asn_history', 'circl_passivedns', 'circl_passivessl', 'cve', 'dns',
+           'eupi', 'ipasn', 'passivetotal', 'sourcecache']
diff --git a/modules/expansion/asn_history.py b/misp_modules/modules/expansion/asn_history.py
similarity index 100%
rename from modules/expansion/asn_history.py
rename to misp_modules/modules/expansion/asn_history.py
diff --git a/modules/expansion/circl_passivedns.py b/misp_modules/modules/expansion/circl_passivedns.py
similarity index 100%
rename from modules/expansion/circl_passivedns.py
rename to misp_modules/modules/expansion/circl_passivedns.py
diff --git a/modules/expansion/circl_passivessl.py b/misp_modules/modules/expansion/circl_passivessl.py
similarity index 100%
rename from modules/expansion/circl_passivessl.py
rename to misp_modules/modules/expansion/circl_passivessl.py
diff --git a/modules/expansion/cve.py b/misp_modules/modules/expansion/cve.py
similarity index 100%
rename from modules/expansion/cve.py
rename to misp_modules/modules/expansion/cve.py
diff --git a/modules/expansion/dns.py b/misp_modules/modules/expansion/dns.py
similarity index 100%
rename from modules/expansion/dns.py
rename to misp_modules/modules/expansion/dns.py
diff --git a/modules/expansion/eupi.py b/misp_modules/modules/expansion/eupi.py
similarity index 100%
rename from modules/expansion/eupi.py
rename to misp_modules/modules/expansion/eupi.py
diff --git a/modules/expansion/intelmq_eventdb.py.experimental b/misp_modules/modules/expansion/intelmq_eventdb.py.experimental
similarity index 100%
rename from modules/expansion/intelmq_eventdb.py.experimental
rename to misp_modules/modules/expansion/intelmq_eventdb.py.experimental
diff --git a/modules/expansion/ipasn.py b/misp_modules/modules/expansion/ipasn.py
similarity index 100%
rename from modules/expansion/ipasn.py
rename to misp_modules/modules/expansion/ipasn.py
diff --git a/modules/expansion/passivetotal.py b/misp_modules/modules/expansion/passivetotal.py
similarity index 100%
rename from modules/expansion/passivetotal.py
rename to misp_modules/modules/expansion/passivetotal.py
diff --git a/modules/expansion/sourcecache.py b/misp_modules/modules/expansion/sourcecache.py
similarity index 100%
rename from modules/expansion/sourcecache.py
rename to misp_modules/modules/expansion/sourcecache.py
diff --git a/modules/__init__.py b/modules/__init__.py
deleted file mode 100644
index e69de29..0000000
diff --git a/modules/expansion/__init__.py b/modules/expansion/__init__.py
deleted file mode 100644
index e69de29..0000000
diff --git a/setup.py b/setup.py
index d9d0f79..1628cba 100644
--- a/setup.py
+++ b/setup.py
@@ -1,6 +1,6 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
-from setuptools import setup
+from setuptools import setup, find_packages
 
 setup(
     name='misp-modules',
@@ -10,8 +10,8 @@ setup(
     maintainer='Alexandre Dulaunoy',
     url='https://github.com/MISP/misp-modules',
     description='MISP modules are autonomous modules that can be used for expansion and other services in MISP',
-    packages=['modules', 'helpers'],
-    scripts=['bin/misp-modules.py'],
+    packages=find_packages(),
+    scripts=['bin/misp-modules'],
     test_suite="tests",
     classifiers=[
         'License :: OSI Approved :: GNU Affero General Public License v3',

From 77bc05e0ecd1fbb6ae0dd52c40c3e8f5f35f15b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= <raphael@vinot.info>
Date: Thu, 23 Jun 2016 19:53:45 +0900
Subject: [PATCH 4/6] Fix travis

---
 .travis.yml | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index abef1f5..999ab27 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -20,7 +20,12 @@ install:
     - python setup.py install
 
 script:
-    - misp-modules.py &
+    - misp-modules &
     - sleep 15
     - python setup.py test
-    - pkill misp-modules.py
+    - pkill misp-modules
+    - pushd ..
+    - misp-modules -s &
+    - sleep 15
+    - popd
+    - pkill misp-modules

From 123b26274c2f041297704af0c3f6fb7b9120313e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= <Rafiot@users.noreply.github.com>
Date: Fri, 24 Jun 2016 09:15:25 +0900
Subject: [PATCH 5/6] Fix typos in the readme

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 7ca3746..dde3425 100644
--- a/README.md
+++ b/README.md
@@ -14,14 +14,14 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/
 ## Existing MISP modules
 
 * [ASN History](misp_modules/modules/expansion/asn_history.py) - a hover and expansion module to expand an AS number with the ASN description and its history.
-* [CIRCL Passive SSL](misp_modulesmodules/expansion/circl_passivessl.py) - a hover and expansion module to expand IP addresses with the X.509 certificate seen.
+* [CIRCL Passive SSL](misp_modules/modules/expansion/circl_passivessl.py) - a hover and expansion module to expand IP addresses with the X.509 certificate seen.
 * [CIRCL Passive DNS](misp_modules/modules/expansion/circl_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information.
 * [CVE](misp_modules/modules/expansion/cve.py) - a hover module to give more information about a vulnerability (CVE).
 * [DNS](misp_modules/modules/expansion/dns.py) - a simple module to resolve MISP attributes like hostname and domain to expand IP addresses attributes.
 * [EUPI](misp_modules/modules/expansion/eupi.py) - a hover and expansion module to get information about an URL from the [Phishing Initiative project](https://phishing-initiative.eu/?lang=en).
 * [IPASN](misp_modules/modules/expansion/ipasn.py) - a hover and expansion to get the BGP ASN of an IP address.
 * [passivetotal](misp_modules/modules/expansion/passivetotal.py) - a [passivetotal](https://www.passivetotal.org/) module that queries a number of different PassiveTotal datasets.
-* [sourcecache](misp_modulesmisp_modules//modules/expansion/sourcecache.py) - a module to cache a specific link from a MISP instance.
+* [sourcecache](misp_modules/modules/expansion/sourcecache.py) - a module to cache a specific link from a MISP instance.
 
 ## How to install and start MISP modules?
 

From 5efb1dbae2e78f91d2248afcbbbffdffcaeb6de1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= <raphael@vinot.info>
Date: Fri, 24 Jun 2016 10:12:31 +0900
Subject: [PATCH 6/6] Also run travis tests on the system-wide instance

---
 .travis.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 999ab27..ebe717f 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -24,8 +24,9 @@ script:
     - sleep 15
     - python setup.py test
     - pkill misp-modules
-    - pushd ..
+    - pushd ~/
     - misp-modules -s &
-    - sleep 15
     - popd
+    - sleep 15
+    - python setup.py test
     - pkill misp-modules