mirror of https://github.com/MISP/misp-modules
commit
879928c6be
|
@ -1,3 +1,6 @@
|
||||||
*.pyc
|
*.pyc
|
||||||
*.swp
|
*.swp
|
||||||
__pycache__
|
__pycache__
|
||||||
|
build/
|
||||||
|
dist/
|
||||||
|
misp_modules.egg-info/
|
||||||
|
|
13
.travis.yml
13
.travis.yml
|
@ -17,9 +17,16 @@ python:
|
||||||
- "nightly"
|
- "nightly"
|
||||||
|
|
||||||
install:
|
install:
|
||||||
- pip install -r REQUIREMENTS
|
- python setup.py install
|
||||||
|
|
||||||
script:
|
script:
|
||||||
- pushd bin
|
- misp-modules &
|
||||||
- ./misp-modules.py -t
|
- sleep 15
|
||||||
|
- python setup.py test
|
||||||
|
- pkill misp-modules
|
||||||
|
- pushd ~/
|
||||||
|
- misp-modules -s &
|
||||||
- popd
|
- popd
|
||||||
|
- sleep 15
|
||||||
|
- python setup.py test
|
||||||
|
- pkill misp-modules
|
||||||
|
|
23
README.md
23
README.md
|
@ -13,15 +13,15 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/
|
||||||
|
|
||||||
## Existing MISP modules
|
## Existing MISP modules
|
||||||
|
|
||||||
* [ASN History](/modules/expansion/asn_history.py) - a hover and expansion module to expand an AS number with the ASN description and its history.
|
* [ASN History](misp_modules/modules/expansion/asn_history.py) - a hover and expansion module to expand an AS number with the ASN description and its history.
|
||||||
* [CIRCL Passive SSL](modules/expansion/circl_passivessl.py) - a hover and expansion module to expand IP addresses with the X.509 certificate seen.
|
* [CIRCL Passive SSL](misp_modules/modules/expansion/circl_passivessl.py) - a hover and expansion module to expand IP addresses with the X.509 certificate seen.
|
||||||
* [CIRCL Passive DNS](modules/expansion/circl_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information.
|
* [CIRCL Passive DNS](misp_modules/modules/expansion/circl_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information.
|
||||||
* [CVE](modules/expansion/cve.py) - a hover module to give more information about a vulnerability (CVE).
|
* [CVE](misp_modules/modules/expansion/cve.py) - a hover module to give more information about a vulnerability (CVE).
|
||||||
* [DNS](modules/expansion/dns.py) - a simple module to resolve MISP attributes like hostname and domain to expand IP addresses attributes.
|
* [DNS](misp_modules/modules/expansion/dns.py) - a simple module to resolve MISP attributes like hostname and domain to expand IP addresses attributes.
|
||||||
* [EUPI](modules/expansion/eupi.py) - a hover and expansion module to get information about an URL from the [Phishing Initiative project](https://phishing-initiative.eu/?lang=en).
|
* [EUPI](misp_modules/modules/expansion/eupi.py) - a hover and expansion module to get information about an URL from the [Phishing Initiative project](https://phishing-initiative.eu/?lang=en).
|
||||||
* [IPASN](modules/expansion/ipasn.py) - a hover and expansion to get the BGP ASN of an IP address.
|
* [IPASN](misp_modules/modules/expansion/ipasn.py) - a hover and expansion to get the BGP ASN of an IP address.
|
||||||
* [passivetotal](modules/expansion/passivetotal.py) - a [passivetotal](https://www.passivetotal.org/) module that queries a number of different PassiveTotal datasets.
|
* [passivetotal](misp_modules/modules/expansion/passivetotal.py) - a [passivetotal](https://www.passivetotal.org/) module that queries a number of different PassiveTotal datasets.
|
||||||
* [sourcecache](modules/expansion/sourcecache.py) - a module to cache a specific link from a MISP instance.
|
* [sourcecache](misp_modules/modules/expansion/sourcecache.py) - a module to cache a specific link from a MISP instance.
|
||||||
|
|
||||||
## How to install and start MISP modules?
|
## How to install and start MISP modules?
|
||||||
|
|
||||||
|
@ -30,13 +30,12 @@ apt-get install python3-dev python3-pip libpq5
|
||||||
git clone https://github.com/MISP/misp-modules.git
|
git clone https://github.com/MISP/misp-modules.git
|
||||||
cd misp-modules
|
cd misp-modules
|
||||||
pip3 install -r REQUIREMENTS
|
pip3 install -r REQUIREMENTS
|
||||||
cd bin
|
python3 bin/misp-modules
|
||||||
python3 misp-modules.py
|
|
||||||
~~~~
|
~~~~
|
||||||
|
|
||||||
## How to add your own MISP modules?
|
## How to add your own MISP modules?
|
||||||
|
|
||||||
Create your module in [modules/expansion/](modules/expansion/). The module should have at minimum three functions:
|
Create your module in [misp_modules/modules/expansion/](misp_modules/modules/expansion/). The module should have at minimum three functions:
|
||||||
|
|
||||||
* **introspection** function that returns a dict of the supported attributes (input and output) by your expansion module.
|
* **introspection** function that returns a dict of the supported attributes (input and output) by your expansion module.
|
||||||
* **handler** function which accepts a JSON document to expand the values and return a dictionary of the expanded values.
|
* **handler** function which accepts a JSON document to expand the values and return a dictionary of the expanded values.
|
||||||
|
|
|
@ -0,0 +1 @@
|
||||||
|
README.md
|
|
@ -29,6 +29,21 @@ import fnmatch
|
||||||
import argparse
|
import argparse
|
||||||
import re
|
import re
|
||||||
|
|
||||||
|
try:
|
||||||
|
import misp_modules.modules
|
||||||
|
HAS_PACKAGE_MODULES = True
|
||||||
|
except Exception as e:
|
||||||
|
print(e)
|
||||||
|
HAS_PACKAGE_MODULES = False
|
||||||
|
|
||||||
|
try:
|
||||||
|
from misp_modules.helpers import *
|
||||||
|
HAS_PACKAGE_HELPERS = True
|
||||||
|
except Exception as e:
|
||||||
|
print(e)
|
||||||
|
HAS_PACKAGE_HELPERS = False
|
||||||
|
|
||||||
|
|
||||||
def init_logger():
|
def init_logger():
|
||||||
log = logging.getLogger('misp-modules')
|
log = logging.getLogger('misp-modules')
|
||||||
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
||||||
|
@ -41,7 +56,7 @@ def init_logger():
|
||||||
return log
|
return log
|
||||||
|
|
||||||
|
|
||||||
def load_helpers(helpersdir='../helpers'):
|
def load_helpers(helpersdir):
|
||||||
sys.path.append(helpersdir)
|
sys.path.append(helpersdir)
|
||||||
hhandlers = {}
|
hhandlers = {}
|
||||||
helpers = []
|
helpers = []
|
||||||
|
@ -51,6 +66,8 @@ def load_helpers(helpersdir='../helpers'):
|
||||||
if re.match(r'^\.', os.path.basename(root)):
|
if re.match(r'^\.', os.path.basename(root)):
|
||||||
continue
|
continue
|
||||||
for filename in fnmatch.filter(filenames, '*.py'):
|
for filename in fnmatch.filter(filenames, '*.py'):
|
||||||
|
if filename == '__init__.py':
|
||||||
|
continue
|
||||||
helpername = filename.split(".")[0]
|
helpername = filename.split(".")[0]
|
||||||
hhandlers[helpername] = importlib.import_module(helpername)
|
hhandlers[helpername] = importlib.import_module(helpername)
|
||||||
selftest = hhandlers[helpername].selftest()
|
selftest = hhandlers[helpername].selftest()
|
||||||
|
@ -61,6 +78,22 @@ def load_helpers(helpersdir='../helpers'):
|
||||||
log.info('Helpers failed {} due to {}'.format(filename, selftest))
|
log.info('Helpers failed {} due to {}'.format(filename, selftest))
|
||||||
|
|
||||||
|
|
||||||
|
def load_package_helpers():
|
||||||
|
if not HAS_PACKAGE_HELPERS:
|
||||||
|
log.info('Unable to load MISP helpers from package.')
|
||||||
|
sys.exit()
|
||||||
|
mhandlers = {}
|
||||||
|
helpers = []
|
||||||
|
for path, helper in sys.modules.items():
|
||||||
|
if not path.startswith('misp_modules.helpers.'):
|
||||||
|
continue
|
||||||
|
helpername = path.replace('misp_modules.helpers.', '')
|
||||||
|
mhandlers[helpername] = helper
|
||||||
|
helpers.append(helpername)
|
||||||
|
log.info('Helper loaded {}'.format(helpername))
|
||||||
|
return mhandlers, helpers
|
||||||
|
|
||||||
|
|
||||||
def load_modules(mod_dir):
|
def load_modules(mod_dir):
|
||||||
sys.path.append(mod_dir)
|
sys.path.append(mod_dir)
|
||||||
mhandlers = {}
|
mhandlers = {}
|
||||||
|
@ -86,6 +119,23 @@ def load_modules(mod_dir):
|
||||||
return mhandlers, modules
|
return mhandlers, modules
|
||||||
|
|
||||||
|
|
||||||
|
def load_package_modules():
|
||||||
|
if not HAS_PACKAGE_MODULES:
|
||||||
|
log.info('Unable to load MISP modules from package.')
|
||||||
|
sys.exit()
|
||||||
|
mhandlers = {}
|
||||||
|
modules = []
|
||||||
|
for path, module in sys.modules.items():
|
||||||
|
r = re.findall("misp_modules[.]modules[.](\w+)[.](\w+)", path)
|
||||||
|
if r and len(r[0]) == 2:
|
||||||
|
moduletype, modulename = r[0]
|
||||||
|
mhandlers[modulename] = module
|
||||||
|
modules.append(modulename)
|
||||||
|
log.info('MISP modules {0} imported'.format(modulename))
|
||||||
|
mhandlers['type:' + modulename] = moduletype
|
||||||
|
return mhandlers, modules
|
||||||
|
|
||||||
|
|
||||||
class ListModules(tornado.web.RequestHandler):
|
class ListModules(tornado.web.RequestHandler):
|
||||||
def get(self):
|
def get(self):
|
||||||
ret = []
|
ret = []
|
||||||
|
@ -110,18 +160,23 @@ class QueryModule(tornado.web.RequestHandler):
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
if os.path.dirname(__file__) is not '':
|
if os.path.dirname(__file__) is '.':
|
||||||
os.chdir(os.path.dirname(__file__))
|
os.chdir('../')
|
||||||
argParser = argparse.ArgumentParser(description='misp-modules server')
|
argParser = argparse.ArgumentParser(description='misp-modules server')
|
||||||
argParser.add_argument('-t', default=False, action='store_true', help='Test mode')
|
argParser.add_argument('-t', default=False, action='store_true', help='Test mode')
|
||||||
|
argParser.add_argument('-s', default=False, action='store_true', help='Run a system install (package installed via pip)')
|
||||||
argParser.add_argument('-p', default=6666, help='misp-modules TCP port (default 6666)')
|
argParser.add_argument('-p', default=6666, help='misp-modules TCP port (default 6666)')
|
||||||
argParser.add_argument('-l', default='localhost', help='misp-modules listen address (default localhost)')
|
argParser.add_argument('-l', default='localhost', help='misp-modules listen address (default localhost)')
|
||||||
args = argParser.parse_args()
|
args = argParser.parse_args()
|
||||||
port = args.p
|
port = args.p
|
||||||
listen = args.l
|
listen = args.l
|
||||||
modulesdir = '../modules'
|
|
||||||
helpersdir = '../helpers'
|
|
||||||
log = init_logger()
|
log = init_logger()
|
||||||
|
if args.s:
|
||||||
|
load_package_helpers()
|
||||||
|
mhandlers, modules = load_package_modules()
|
||||||
|
else:
|
||||||
|
modulesdir = 'misp_modules/modules'
|
||||||
|
helpersdir = 'misp_modules/helpers'
|
||||||
load_helpers(helpersdir=helpersdir)
|
load_helpers(helpersdir=helpersdir)
|
||||||
mhandlers, modules = load_modules(modulesdir)
|
mhandlers, modules = load_modules(modulesdir)
|
||||||
service = [(r'/modules', ListModules), (r'/query', QueryModule)]
|
service = [(r'/modules', ListModules), (r'/query', QueryModule)]
|
|
@ -0,0 +1 @@
|
||||||
|
__all__ = ['cache']
|
|
@ -32,7 +32,7 @@ def selftest(enable=True):
|
||||||
return False
|
return False
|
||||||
r = redis.StrictRedis(host=hostname, port=port, db=db)
|
r = redis.StrictRedis(host=hostname, port=port, db=db)
|
||||||
try:
|
try:
|
||||||
r.set('test', 'selftest')
|
r.ping()
|
||||||
except:
|
except:
|
||||||
return 'Redis not running or not installed. Helper will be disabled.'
|
return 'Redis not running or not installed. Helper will be disabled.'
|
||||||
|
|
||||||
|
@ -49,8 +49,7 @@ def get(modulename=None, query=None, value=None, debug=False):
|
||||||
if not r.exists(key):
|
if not r.exists(key):
|
||||||
if debug:
|
if debug:
|
||||||
print("Key {} added in cache".format(key))
|
print("Key {} added in cache".format(key))
|
||||||
r.set(key, value)
|
r.setex(key, 86400, value)
|
||||||
r.expire(key, 86400)
|
|
||||||
else:
|
else:
|
||||||
if debug:
|
if debug:
|
||||||
print("Cache hit with Key {}".format(key))
|
print("Cache hit with Key {}".format(key))
|
|
@ -0,0 +1 @@
|
||||||
|
from .expansion import *
|
|
@ -0,0 +1,2 @@
|
||||||
|
__all__ = ['asn_history', 'circl_passivedns', 'circl_passivessl', 'cve', 'dns',
|
||||||
|
'eupi', 'ipasn', 'passivetotal', 'sourcecache']
|
|
@ -11,7 +11,6 @@ cveapi_url = 'https://cve.circl.lu/api/cve/'
|
||||||
def handler(q=False):
|
def handler(q=False):
|
||||||
if q is False:
|
if q is False:
|
||||||
return False
|
return False
|
||||||
print (q)
|
|
||||||
request = json.loads(q)
|
request = json.loads(q)
|
||||||
if not request.get('vulnerability'):
|
if not request.get('vulnerability'):
|
||||||
misperrors['error'] = 'Vulnerability id missing'
|
misperrors['error'] = 'Vulnerability id missing'
|
|
@ -0,0 +1,37 @@
|
||||||
|
#!/usr/bin/python
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
from setuptools import setup, find_packages
|
||||||
|
|
||||||
|
setup(
|
||||||
|
name='misp-modules',
|
||||||
|
version='1.0',
|
||||||
|
author='Alexandre Dulaunoy',
|
||||||
|
author_email='alexandre.dulaunoy@circl.lu',
|
||||||
|
maintainer='Alexandre Dulaunoy',
|
||||||
|
url='https://github.com/MISP/misp-modules',
|
||||||
|
description='MISP modules are autonomous modules that can be used for expansion and other services in MISP',
|
||||||
|
packages=find_packages(),
|
||||||
|
scripts=['bin/misp-modules'],
|
||||||
|
test_suite="tests",
|
||||||
|
classifiers=[
|
||||||
|
'License :: OSI Approved :: GNU Affero General Public License v3',
|
||||||
|
'Development Status :: 5 - Production/Stable',
|
||||||
|
'Environment :: Console',
|
||||||
|
'Intended Audience :: Science/Research',
|
||||||
|
'Programming Language :: Python :: 3',
|
||||||
|
'Topic :: Security',
|
||||||
|
],
|
||||||
|
install_requires=[
|
||||||
|
'tornado',
|
||||||
|
'dnspython3',
|
||||||
|
'requests',
|
||||||
|
'urlarchiver',
|
||||||
|
'passivetotal',
|
||||||
|
'PyPDNS',
|
||||||
|
'pypssl',
|
||||||
|
'redis',
|
||||||
|
'pyeupi',
|
||||||
|
'ipasn-redis',
|
||||||
|
'asnhistory',
|
||||||
|
]
|
||||||
|
)
|
|
@ -0,0 +1,26 @@
|
||||||
|
#!/usr/bin/env python
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
|
import unittest
|
||||||
|
import requests
|
||||||
|
|
||||||
|
|
||||||
|
class TestModules(unittest.TestCase):
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
self.maxDiff = None
|
||||||
|
self.headers = {'Content-Type': 'application/json'}
|
||||||
|
|
||||||
|
def test_introspection(self):
|
||||||
|
response = requests.get('http://127.0.0.1:6666/modules')
|
||||||
|
print(response.json())
|
||||||
|
|
||||||
|
def test_cve(self):
|
||||||
|
with open('tests/bodycve.json', 'r') as f:
|
||||||
|
response = requests.post('http://127.0.0.1:6666/query', data=f.read())
|
||||||
|
print(response.json())
|
||||||
|
|
||||||
|
def test_dns(self):
|
||||||
|
with open('tests/body.json', 'r') as f:
|
||||||
|
response = requests.post('http://127.0.0.1:6666/query', data=f.read())
|
||||||
|
print(response.json())
|
Loading…
Reference in New Issue