diff --git a/.gitignore b/.gitignore index e69364e..9676396 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ *.pyc *.swp __pycache__ +build/ +dist/ +misp_modules.egg-info/ diff --git a/.travis.yml b/.travis.yml index d9d892f..ebe717f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,9 +17,16 @@ python: - "nightly" install: - - pip install -r REQUIREMENTS + - python setup.py install script: - - pushd bin - - ./misp-modules.py -t + - misp-modules & + - sleep 15 + - python setup.py test + - pkill misp-modules + - pushd ~/ + - misp-modules -s & - popd + - sleep 15 + - python setup.py test + - pkill misp-modules diff --git a/README.md b/README.md index 3e2036d..dde3425 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ MISP modules are autonomous modules that can be used for expansion and other services in [MISP](https://github.com/MISP/MISP). The modules are written in Python 3 following a simple API interface. The objective is to ease the extensions of MISP functionalities -without modifying core components. The API is available via a simple REST API which is independent from MISP installation or configuration. +without modifying core components. The API is available via a simple REST API which is independent from MISP installation or configuration. MISP modules support is included in MISP starting from version 2.4.28. @@ -13,15 +13,15 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/ ## Existing MISP modules -* [ASN History](/modules/expansion/asn_history.py) - a hover and expansion module to expand an AS number with the ASN description and its history. -* [CIRCL Passive SSL](modules/expansion/circl_passivessl.py) - a hover and expansion module to expand IP addresses with the X.509 certificate seen. -* [CIRCL Passive DNS](modules/expansion/circl_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information. -* [CVE](modules/expansion/cve.py) - a hover module to give more information about a vulnerability (CVE). -* [DNS](modules/expansion/dns.py) - a simple module to resolve MISP attributes like hostname and domain to expand IP addresses attributes. -* [EUPI](modules/expansion/eupi.py) - a hover and expansion module to get information about an URL from the [Phishing Initiative project](https://phishing-initiative.eu/?lang=en). -* [IPASN](modules/expansion/ipasn.py) - a hover and expansion to get the BGP ASN of an IP address. -* [passivetotal](modules/expansion/passivetotal.py) - a [passivetotal](https://www.passivetotal.org/) module that queries a number of different PassiveTotal datasets. -* [sourcecache](modules/expansion/sourcecache.py) - a module to cache a specific link from a MISP instance. +* [ASN History](misp_modules/modules/expansion/asn_history.py) - a hover and expansion module to expand an AS number with the ASN description and its history. +* [CIRCL Passive SSL](misp_modules/modules/expansion/circl_passivessl.py) - a hover and expansion module to expand IP addresses with the X.509 certificate seen. +* [CIRCL Passive DNS](misp_modules/modules/expansion/circl_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information. +* [CVE](misp_modules/modules/expansion/cve.py) - a hover module to give more information about a vulnerability (CVE). +* [DNS](misp_modules/modules/expansion/dns.py) - a simple module to resolve MISP attributes like hostname and domain to expand IP addresses attributes. +* [EUPI](misp_modules/modules/expansion/eupi.py) - a hover and expansion module to get information about an URL from the [Phishing Initiative project](https://phishing-initiative.eu/?lang=en). +* [IPASN](misp_modules/modules/expansion/ipasn.py) - a hover and expansion to get the BGP ASN of an IP address. +* [passivetotal](misp_modules/modules/expansion/passivetotal.py) - a [passivetotal](https://www.passivetotal.org/) module that queries a number of different PassiveTotal datasets. +* [sourcecache](misp_modules/modules/expansion/sourcecache.py) - a module to cache a specific link from a MISP instance. ## How to install and start MISP modules? @@ -30,13 +30,12 @@ apt-get install python3-dev python3-pip libpq5 git clone https://github.com/MISP/misp-modules.git cd misp-modules pip3 install -r REQUIREMENTS -cd bin -python3 misp-modules.py +python3 bin/misp-modules ~~~~ ## How to add your own MISP modules? -Create your module in [modules/expansion/](modules/expansion/). The module should have at minimum three functions: +Create your module in [misp_modules/modules/expansion/](misp_modules/modules/expansion/). The module should have at minimum three functions: * **introspection** function that returns a dict of the supported attributes (input and output) by your expansion module. * **handler** function which accepts a JSON document to expand the values and return a dictionary of the expanded values. diff --git a/README.rst b/README.rst new file mode 120000 index 0000000..42061c0 --- /dev/null +++ b/README.rst @@ -0,0 +1 @@ +README.md \ No newline at end of file diff --git a/bin/misp-modules.py b/bin/misp-modules similarity index 69% rename from bin/misp-modules.py rename to bin/misp-modules index f5adc83..d25d7dd 100755 --- a/bin/misp-modules.py +++ b/bin/misp-modules @@ -29,6 +29,21 @@ import fnmatch import argparse import re +try: + import misp_modules.modules + HAS_PACKAGE_MODULES = True +except Exception as e: + print(e) + HAS_PACKAGE_MODULES = False + +try: + from misp_modules.helpers import * + HAS_PACKAGE_HELPERS = True +except Exception as e: + print(e) + HAS_PACKAGE_HELPERS = False + + def init_logger(): log = logging.getLogger('misp-modules') formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') @@ -41,7 +56,7 @@ def init_logger(): return log -def load_helpers(helpersdir='../helpers'): +def load_helpers(helpersdir): sys.path.append(helpersdir) hhandlers = {} helpers = [] @@ -51,9 +66,11 @@ def load_helpers(helpersdir='../helpers'): if re.match(r'^\.', os.path.basename(root)): continue for filename in fnmatch.filter(filenames, '*.py'): + if filename == '__init__.py': + continue helpername = filename.split(".")[0] hhandlers[helpername] = importlib.import_module(helpername) - selftest= hhandlers[helpername].selftest() + selftest = hhandlers[helpername].selftest() if selftest is None: helpers.append(helpername) log.info('Helpers loaded {} '.format(filename)) @@ -61,6 +78,22 @@ def load_helpers(helpersdir='../helpers'): log.info('Helpers failed {} due to {}'.format(filename, selftest)) +def load_package_helpers(): + if not HAS_PACKAGE_HELPERS: + log.info('Unable to load MISP helpers from package.') + sys.exit() + mhandlers = {} + helpers = [] + for path, helper in sys.modules.items(): + if not path.startswith('misp_modules.helpers.'): + continue + helpername = path.replace('misp_modules.helpers.', '') + mhandlers[helpername] = helper + helpers.append(helpername) + log.info('Helper loaded {}'.format(helpername)) + return mhandlers, helpers + + def load_modules(mod_dir): sys.path.append(mod_dir) mhandlers = {} @@ -86,6 +119,23 @@ def load_modules(mod_dir): return mhandlers, modules +def load_package_modules(): + if not HAS_PACKAGE_MODULES: + log.info('Unable to load MISP modules from package.') + sys.exit() + mhandlers = {} + modules = [] + for path, module in sys.modules.items(): + r = re.findall("misp_modules[.]modules[.](\w+)[.](\w+)", path) + if r and len(r[0]) == 2: + moduletype, modulename = r[0] + mhandlers[modulename] = module + modules.append(modulename) + log.info('MISP modules {0} imported'.format(modulename)) + mhandlers['type:' + modulename] = moduletype + return mhandlers, modules + + class ListModules(tornado.web.RequestHandler): def get(self): ret = [] @@ -110,20 +160,25 @@ class QueryModule(tornado.web.RequestHandler): if __name__ == '__main__': - if os.path.dirname(__file__) is not '': - os.chdir(os.path.dirname(__file__)) + if os.path.dirname(__file__) is '.': + os.chdir('../') argParser = argparse.ArgumentParser(description='misp-modules server') argParser.add_argument('-t', default=False, action='store_true', help='Test mode') + argParser.add_argument('-s', default=False, action='store_true', help='Run a system install (package installed via pip)') argParser.add_argument('-p', default=6666, help='misp-modules TCP port (default 6666)') argParser.add_argument('-l', default='localhost', help='misp-modules listen address (default localhost)') args = argParser.parse_args() port = args.p listen = args.l - modulesdir = '../modules' - helpersdir = '../helpers' log = init_logger() - load_helpers(helpersdir=helpersdir) - mhandlers, modules = load_modules(modulesdir) + if args.s: + load_package_helpers() + mhandlers, modules = load_package_modules() + else: + modulesdir = 'misp_modules/modules' + helpersdir = 'misp_modules/helpers' + load_helpers(helpersdir=helpersdir) + mhandlers, modules = load_modules(modulesdir) service = [(r'/modules', ListModules), (r'/query', QueryModule)] application = tornado.web.Application(service) diff --git a/modules/__init__.py b/misp_modules/__init__.py similarity index 100% rename from modules/__init__.py rename to misp_modules/__init__.py diff --git a/misp_modules/helpers/__init__.py b/misp_modules/helpers/__init__.py new file mode 100644 index 0000000..cd16c88 --- /dev/null +++ b/misp_modules/helpers/__init__.py @@ -0,0 +1 @@ +__all__ = ['cache'] diff --git a/helpers/cache.py b/misp_modules/helpers/cache.py similarity index 84% rename from helpers/cache.py rename to misp_modules/helpers/cache.py index da43cd8..7d70159 100644 --- a/helpers/cache.py +++ b/misp_modules/helpers/cache.py @@ -32,7 +32,7 @@ def selftest(enable=True): return False r = redis.StrictRedis(host=hostname, port=port, db=db) try: - r.set('test', 'selftest') + r.ping() except: return 'Redis not running or not installed. Helper will be disabled.' @@ -44,16 +44,15 @@ def get(modulename=None, query=None, value=None, debug=False): h = hashlib.sha1() h.update(query.encode('UTF-8')) hv = h.hexdigest() - key = "m:"+modulename+":"+hv + key = "m:" + modulename + ":" + hv if not r.exists(key): if debug: - print ("Key {} added in cache".format(key)) - r.set(key, value) - r.expire(key, 86400) + print("Key {} added in cache".format(key)) + r.setex(key, 86400, value) else: if debug: - print ("Cache hit with Key {}".format(key)) + print("Cache hit with Key {}".format(key)) return r.get(key) @@ -68,14 +67,14 @@ if __name__ == "__main__": if selftest() is not None: sys.exit() else: - print ("Selftest ok") + print("Selftest ok") v = get(modulename="testmodule", query="abcdef", value="barfoo", debug=True) if v == b'barfoo': - print ("Cache ok") + print("Cache ok") v = get(modulename="testmodule", query="abcdef") - print (v) + print(v) v = get(modulename="testmodule") if (not v): - print ("Failed ok") + print("Failed ok") if flush(): - print ("Cache flushed ok") + print("Cache flushed ok") diff --git a/misp_modules/modules/__init__.py b/misp_modules/modules/__init__.py new file mode 100644 index 0000000..f0f4cdb --- /dev/null +++ b/misp_modules/modules/__init__.py @@ -0,0 +1 @@ +from .expansion import * diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py new file mode 100644 index 0000000..5854e40 --- /dev/null +++ b/misp_modules/modules/expansion/__init__.py @@ -0,0 +1,2 @@ +__all__ = ['asn_history', 'circl_passivedns', 'circl_passivessl', 'cve', 'dns', + 'eupi', 'ipasn', 'passivetotal', 'sourcecache'] diff --git a/modules/expansion/asn_history.py b/misp_modules/modules/expansion/asn_history.py similarity index 100% rename from modules/expansion/asn_history.py rename to misp_modules/modules/expansion/asn_history.py diff --git a/modules/expansion/circl_passivedns.py b/misp_modules/modules/expansion/circl_passivedns.py similarity index 100% rename from modules/expansion/circl_passivedns.py rename to misp_modules/modules/expansion/circl_passivedns.py diff --git a/modules/expansion/circl_passivessl.py b/misp_modules/modules/expansion/circl_passivessl.py similarity index 100% rename from modules/expansion/circl_passivessl.py rename to misp_modules/modules/expansion/circl_passivessl.py diff --git a/modules/expansion/cve.py b/misp_modules/modules/expansion/cve.py similarity index 98% rename from modules/expansion/cve.py rename to misp_modules/modules/expansion/cve.py index 48e8f02..e370116 100755 --- a/modules/expansion/cve.py +++ b/misp_modules/modules/expansion/cve.py @@ -11,7 +11,6 @@ cveapi_url = 'https://cve.circl.lu/api/cve/' def handler(q=False): if q is False: return False - print (q) request = json.loads(q) if not request.get('vulnerability'): misperrors['error'] = 'Vulnerability id missing' diff --git a/modules/expansion/dns.py b/misp_modules/modules/expansion/dns.py similarity index 100% rename from modules/expansion/dns.py rename to misp_modules/modules/expansion/dns.py diff --git a/modules/expansion/eupi.py b/misp_modules/modules/expansion/eupi.py similarity index 100% rename from modules/expansion/eupi.py rename to misp_modules/modules/expansion/eupi.py diff --git a/modules/expansion/intelmq_eventdb.py.experimental b/misp_modules/modules/expansion/intelmq_eventdb.py.experimental similarity index 100% rename from modules/expansion/intelmq_eventdb.py.experimental rename to misp_modules/modules/expansion/intelmq_eventdb.py.experimental diff --git a/modules/expansion/ipasn.py b/misp_modules/modules/expansion/ipasn.py similarity index 100% rename from modules/expansion/ipasn.py rename to misp_modules/modules/expansion/ipasn.py diff --git a/modules/expansion/passivetotal.py b/misp_modules/modules/expansion/passivetotal.py similarity index 100% rename from modules/expansion/passivetotal.py rename to misp_modules/modules/expansion/passivetotal.py diff --git a/modules/expansion/sourcecache.py b/misp_modules/modules/expansion/sourcecache.py similarity index 100% rename from modules/expansion/sourcecache.py rename to misp_modules/modules/expansion/sourcecache.py diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..1628cba --- /dev/null +++ b/setup.py @@ -0,0 +1,37 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +from setuptools import setup, find_packages + +setup( + name='misp-modules', + version='1.0', + author='Alexandre Dulaunoy', + author_email='alexandre.dulaunoy@circl.lu', + maintainer='Alexandre Dulaunoy', + url='https://github.com/MISP/misp-modules', + description='MISP modules are autonomous modules that can be used for expansion and other services in MISP', + packages=find_packages(), + scripts=['bin/misp-modules'], + test_suite="tests", + classifiers=[ + 'License :: OSI Approved :: GNU Affero General Public License v3', + 'Development Status :: 5 - Production/Stable', + 'Environment :: Console', + 'Intended Audience :: Science/Research', + 'Programming Language :: Python :: 3', + 'Topic :: Security', + ], + install_requires=[ + 'tornado', + 'dnspython3', + 'requests', + 'urlarchiver', + 'passivetotal', + 'PyPDNS', + 'pypssl', + 'redis', + 'pyeupi', + 'ipasn-redis', + 'asnhistory', + ] +) diff --git a/modules/expansion/__init__.py b/tests/__init__.py similarity index 100% rename from modules/expansion/__init__.py rename to tests/__init__.py diff --git a/tests/query-circl_passivedns.sh b/tests/query-circl_passivedns.sh old mode 100644 new mode 100755 diff --git a/tests/query-circl_passivessl.sh b/tests/query-circl_passivessl.sh old mode 100644 new mode 100755 diff --git a/tests/query-cve.sh b/tests/query-cve.sh old mode 100644 new mode 100755 diff --git a/tests/query-dns.sh b/tests/query-dns.sh old mode 100644 new mode 100755 diff --git a/tests/query-sourcecache.sh b/tests/query-sourcecache.sh old mode 100644 new mode 100755 diff --git a/tests/search-modules.sh b/tests/search-modules.sh old mode 100644 new mode 100755 diff --git a/tests/test.py b/tests/test.py new file mode 100755 index 0000000..4732358 --- /dev/null +++ b/tests/test.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import unittest +import requests + + +class TestModules(unittest.TestCase): + + def setUp(self): + self.maxDiff = None + self.headers = {'Content-Type': 'application/json'} + + def test_introspection(self): + response = requests.get('http://127.0.0.1:6666/modules') + print(response.json()) + + def test_cve(self): + with open('tests/bodycve.json', 'r') as f: + response = requests.post('http://127.0.0.1:6666/query', data=f.read()) + print(response.json()) + + def test_dns(self): + with open('tests/body.json', 'r') as f: + response = requests.post('http://127.0.0.1:6666/query', data=f.read()) + print(response.json())