Make it a package

.gitignore

@@ -1,3 +1,6 @@
 *.pyc
 *.swp
 __pycache__
+build/
+dist/
+misp_modules.egg-info/

README.rst

@@ -0,0 +1 @@
+README.md

bin/misp-modules.py

@@ -29,6 +29,7 @@ import fnmatch
 import argparse
 import re
 
+
 def init_logger():
     log = logging.getLogger('misp-modules')
     formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
@@ -41,7 +42,7 @@ def init_logger():
     return log
 
 
-def load_helpers(helpersdir='../helpers'):
+def load_helpers(helpersdir):
     sys.path.append(helpersdir)
     hhandlers = {}
     helpers = []
@@ -51,9 +52,11 @@ def load_helpers(helpersdir='../helpers'):
         if re.match(r'^\.', os.path.basename(root)):
             continue
         for filename in fnmatch.filter(filenames, '*.py'):
+            if filename == '__init__.py':
+                continue
             helpername = filename.split(".")[0]
             hhandlers[helpername] = importlib.import_module(helpername)
-            selftest= hhandlers[helpername].selftest()
+            selftest = hhandlers[helpername].selftest()
             if selftest is None:
                 helpers.append(helpername)
                 log.info('Helpers loaded {} '.format(filename))
@@ -110,8 +113,8 @@ class QueryModule(tornado.web.RequestHandler):
 
 
 if __name__ == '__main__':
-    if os.path.dirname(__file__) is not '':
+    if os.path.dirname(__file__) is '.':
-        os.chdir(os.path.dirname(__file__))
+        os.chdir('../')
     argParser = argparse.ArgumentParser(description='misp-modules server')
     argParser.add_argument('-t', default=False, action='store_true', help='Test mode')
     argParser.add_argument('-p', default=6666, help='misp-modules TCP port (default 6666)')
@@ -119,8 +122,8 @@ if __name__ == '__main__':
     args = argParser.parse_args()
     port = args.p
     listen = args.l
-    modulesdir = '../modules'
+    modulesdir = 'modules'
-    helpersdir = '../helpers'
+    helpersdir = 'helpers'
     log = init_logger()
     load_helpers(helpersdir=helpersdir)
     mhandlers, modules = load_modules(modulesdir)

helpers/cache.py

@@ -32,7 +32,7 @@ def selftest(enable=True):
         return False
     r = redis.StrictRedis(host=hostname, port=port, db=db)
     try:
-        r.set('test', 'selftest')
+        r.ping()
     except:
         return 'Redis not running or not installed. Helper will be disabled.'
 
@@ -44,16 +44,15 @@ def get(modulename=None, query=None, value=None, debug=False):
     h = hashlib.sha1()
     h.update(query.encode('UTF-8'))
     hv = h.hexdigest()
-    key = "m:"+modulename+":"+hv
+    key = "m:" + modulename + ":" + hv
 
     if not r.exists(key):
         if debug:
-            print ("Key {} added in cache".format(key))
+            print("Key {} added in cache".format(key))
-        r.set(key, value)
+        r.setex(key, 86400, value)
-        r.expire(key, 86400)
     else:
         if debug:
-            print ("Cache hit with Key {}".format(key))
+            print("Cache hit with Key {}".format(key))
 
     return r.get(key)
 
@@ -68,14 +67,14 @@ if __name__ == "__main__":
     if selftest() is not None:
         sys.exit()
     else:
-        print ("Selftest ok")
+        print("Selftest ok")
     v = get(modulename="testmodule", query="abcdef", value="barfoo", debug=True)
     if v == b'barfoo':
-        print ("Cache ok")
+        print("Cache ok")
     v = get(modulename="testmodule", query="abcdef")
-    print (v)
+    print(v)
     v = get(modulename="testmodule")
     if (not v):
-        print ("Failed ok")
+        print("Failed ok")
     if flush():
-        print ("Cache flushed ok")
+        print("Cache flushed ok")

modules/expansion/cve.py

@@ -11,7 +11,6 @@ cveapi_url = 'https://cve.circl.lu/api/cve/'
 def handler(q=False):
     if q is False:
         return False
-    print (q)
     request = json.loads(q)
     if not request.get('vulnerability'):
         misperrors['error'] = 'Vulnerability id missing'

setup.py

@@ -0,0 +1,37 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+from setuptools import setup
+
+setup(
+    name='misp-modules',
+    version='1.0',
+    author='Alexandre Dulaunoy',
+    author_email='alexandre.dulaunoy@circl.lu',
+    maintainer='Alexandre Dulaunoy',
+    url='https://github.com/MISP/misp-modules',
+    description='MISP modules are autonomous modules that can be used for expansion and other services in MISP',
+    packages=['modules', 'helpers'],
+    scripts=['bin/misp-modules.py'],
+    test_suite="tests",
+    classifiers=[
+        'License :: OSI Approved :: GNU Affero General Public License v3',
+        'Development Status :: 5 - Production/Stable',
+        'Environment :: Console',
+        'Intended Audience :: Science/Research',
+        'Programming Language :: Python :: 3',
+        'Topic :: Security',
+    ],
+    install_requires=[
+        'tornado',
+        'dnspython3',
+        'requests',
+        'urlarchiver',
+        'passivetotal',
+        'PyPDNS',
+        'pypssl',
+        'redis',
+        'pyeupi',
+        'ipasn-redis',
+        'asnhistory',
+    ]
+)