MISP
/
misp-modules
mirror of https://github.com/MISP/misp-modules
2
0
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'master' of https://github.com/MISP/misp-modules

pull/41/head
Hannah Ward 2016-08-12 12:35:47 +01:00
parent 38205910df 1293cb3109
commit 6f770ad0c7
No known key found for this signature in database
GPG Key ID: BA89E572EE1B4C5F
13 changed files with 230 additions and 195 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
README.md
View File

@ -30,8 +30,8 @@ sudo apt-get install python3-dev python3-pip libpq5
cd /usr/local/src/
sudo git clone https://github.com/MISP/misp-modules.git
cd misp-modules
sudo pip3 install -r REQUIREMENTS
sudo pip3 install .
sudo pip3 install --upgrade -r REQUIREMENTS
sudo pip3 install --upgrade .
sudo vi /etc/rc.local, add this line: `sudo -u www-data misp-modules -s`
~~~~

2
REQUIREMENTS
View File

@ -12,3 +12,5 @@ pyeupi
ipasn-redis
asnhistory
git+https://github.com/Rafiot/uwhoisd.git@testing#egg=uwhois&subdirectory=client
pillow
pytesseract

190
bin/misp-modules
View File

@ -1,190 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Core MISP expansion modules loader and web service
#
# Copyright (C) 2016 Alexandre Dulaunoy
# Copyright (C) 2016 CIRCL - Computer Incident Response Center Luxembourg
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import os
import sys
import tornado.web
import importlib
import json
import logging
import fnmatch
import argparse
import re
try:
import misp_modules.modules
HAS_PACKAGE_MODULES = True
except Exception as e:
print(e)
HAS_PACKAGE_MODULES = False
try:
from misp_modules.helpers import *
HAS_PACKAGE_HELPERS = True
except Exception as e:
print(e)
HAS_PACKAGE_HELPERS = False
def init_logger():
log = logging.getLogger('misp-modules')
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler = logging.StreamHandler(stream=sys.stdout)
handler.setFormatter(formatter)
handler.setLevel(logging.INFO)
log.addHandler(handler)
log.setLevel(logging.INFO)
return log
def load_helpers(helpersdir):
sys.path.append(helpersdir)
hhandlers = {}
helpers = []
for root, dirnames, filenames in os.walk(helpersdir):
if os.path.basename(root) == '__pycache__':
continue
if re.match(r'^\.', os.path.basename(root)):
continue
for filename in fnmatch.filter(filenames, '*.py'):
if filename == '__init__.py':
continue
helpername = filename.split(".")[0]
hhandlers[helpername] = importlib.import_module(helpername)
selftest = hhandlers[helpername].selftest()
if selftest is None:
helpers.append(helpername)
log.info('Helpers loaded {} '.format(filename))
else:
log.info('Helpers failed {} due to {}'.format(filename, selftest))
def load_package_helpers():
if not HAS_PACKAGE_HELPERS:
log.info('Unable to load MISP helpers from package.')
sys.exit()
mhandlers = {}
helpers = []
for path, helper in sys.modules.items():
if not path.startswith('misp_modules.helpers.'):
continue
helpername = path.replace('misp_modules.helpers.', '')
mhandlers[helpername] = helper
helpers.append(helpername)
log.info('Helper loaded {}'.format(helpername))
return mhandlers, helpers
def load_modules(mod_dir):
sys.path.append(mod_dir)
mhandlers = {}
modules = []
for root, dirnames, filenames in os.walk(mod_dir):
if os.path.basename(root) == '__pycache__':
continue
if os.path.basename(root).startswith("."):
continue
for filename in fnmatch.filter(filenames, '*.py'):
if filename == '__init__.py':
continue
modulename = filename.split(".")[0]
moduletype = os.path.split(modulesdir)[1]
try:
mhandlers[modulename] = importlib.import_module(os.path.basename(root) + '.' + modulename)
except Exception as e:
log.warning('MISP modules {0} failed due to {1}'.format(modulename, e))
continue
modules.append(modulename)
log.info('MISP modules {0} imported'.format(modulename))
mhandlers['type:' + modulename] = moduletype
return mhandlers, modules
def load_package_modules():
if not HAS_PACKAGE_MODULES:
log.info('Unable to load MISP modules from package.')
sys.exit()
mhandlers = {}
modules = []
for path, module in sys.modules.items():
r = re.findall("misp_modules[.]modules[.](\w+)[.](\w+)", path)
if r and len(r[0]) == 2:
moduletype, modulename = r[0]
mhandlers[modulename] = module
modules.append(modulename)
log.info('MISP modules {0} imported'.format(modulename))
mhandlers['type:' + modulename] = moduletype
return mhandlers, modules
class ListModules(tornado.web.RequestHandler):
def get(self):
ret = []
for module in modules:
x = {}
x['name'] = module
x['type'] = mhandlers['type:' + module]
x['mispattributes'] = mhandlers[module].introspection()
x['meta'] = mhandlers[module].version()
ret.append(x)
log.debug('MISP ListModules request')
self.write(json.dumps(ret))
class QueryModule(tornado.web.RequestHandler):
def post(self):
jsonpayload = self.request.body.decode('utf-8')
x = json.loads(jsonpayload)
log.debug('MISP QueryModule request {0}'.format(jsonpayload))
ret = mhandlers[x['module']].handler(q=jsonpayload)
self.write(json.dumps(ret))
if __name__ == '__main__':
if os.path.dirname(__file__) in ['.', '']:
os.chdir('../')
argParser = argparse.ArgumentParser(description='misp-modules server')
argParser.add_argument('-t', default=False, action='store_true', help='Test mode')
argParser.add_argument('-s', default=False, action='store_true', help='Run a system install (package installed via pip)')
argParser.add_argument('-p', default=6666, help='misp-modules TCP port (default 6666)')
argParser.add_argument('-l', default='localhost', help='misp-modules listen address (default localhost)')
args = argParser.parse_args()
port = args.p
listen = args.l
log = init_logger()
if args.s:
load_package_helpers()
mhandlers, modules = load_package_modules()
else:
modulesdir = 'misp_modules/modules'
helpersdir = 'misp_modules/helpers'
load_helpers(helpersdir=helpersdir)
mhandlers, modules = load_modules(modulesdir)
service = [(r'/modules', ListModules), (r'/query', QueryModule)]
application = tornado.web.Application(service)
application.listen(port, address=listen)
log.info('MISP modules server started on {0} port {1}'.format(listen, port))
if args.t:
log.info('MISP modules started in test-mode, quitting immediately.')
sys.exit()
tornado.ioloop.IOLoop.instance().start()

213
misp_modules/__init__.py
View File

@ -0,0 +1,213 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Core MISP expansion modules loader and web service
#
# Copyright (C) 2016 Alexandre Dulaunoy
# Copyright (C) 2016 CIRCL - Computer Incident Response Center Luxembourg
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import os
import signal
import sys
from tornado.ioloop import IOLoop
import tornado.web
import importlib
import json
import logging
import fnmatch
import argparse
import re
try:
from .modules import *
HAS_PACKAGE_MODULES = True
except Exception as e:
print(e)
HAS_PACKAGE_MODULES = False
try:
from .helpers import *
HAS_PACKAGE_HELPERS = True
except Exception as e:
print(e)
HAS_PACKAGE_HELPERS = False
log = logging.getLogger('misp-modules')
def handle_signal(sig, frame):
IOLoop.instance().add_callback(IOLoop.instance().stop)
def init_logger():
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler = logging.StreamHandler(stream=sys.stdout)
handler.setFormatter(formatter)
handler.setLevel(logging.INFO)
log.addHandler(handler)
log.setLevel(logging.INFO)
return log
def load_helpers(helpersdir):
sys.path.append(helpersdir)
hhandlers = {}
helpers = []
for root, dirnames, filenames in os.walk(helpersdir):
if os.path.basename(root) == '__pycache__':
continue
if re.match(r'^\.', os.path.basename(root)):
continue
for filename in fnmatch.filter(filenames, '*.py'):
if filename == '__init__.py':
continue
helpername = filename.split(".")[0]
hhandlers[helpername] = importlib.import_module(helpername)
selftest = hhandlers[helpername].selftest()
if selftest is None:
helpers.append(helpername)
log.info('Helpers loaded {} '.format(filename))
else:
log.info('Helpers failed {} due to {}'.format(filename, selftest))
def load_package_helpers():
if not HAS_PACKAGE_HELPERS:
log.info('Unable to load MISP helpers from package.')
sys.exit()
mhandlers = {}
helpers = []
for path, helper in sys.modules.items():
if not path.startswith('misp_modules.helpers.'):
continue
helpername = path.replace('misp_modules.helpers.', '')
mhandlers[helpername] = helper
selftest = mhandlers[helpername].selftest()
if selftest is None:
helpers.append(helpername)
log.info('Helper loaded {}'.format(helpername))
else:
log.info('Helpers failed {} due to {}'.format(helpername, selftest))
return mhandlers, helpers
def load_modules(mod_dir):
sys.path.append(mod_dir)
mhandlers = {}
modules = []
for root, dirnames, filenames in os.walk(mod_dir):
if os.path.basename(root) == '__pycache__':
continue
if os.path.basename(root).startswith("."):
continue
for filename in fnmatch.filter(filenames, '*.py'):
if filename == '__init__.py':
continue
modulename = filename.split(".")[0]
moduletype = os.path.split(mod_dir)[1]
try:
mhandlers[modulename] = importlib.import_module(os.path.basename(root) + '.' + modulename)
except Exception as e:
log.warning('MISP modules {0} failed due to {1}'.format(modulename, e))
continue
modules.append(modulename)
log.info('MISP modules {0} imported'.format(modulename))
mhandlers['type:' + modulename] = moduletype
return mhandlers, modules
def load_package_modules():
if not HAS_PACKAGE_MODULES:
log.info('Unable to load MISP modules from package.')
sys.exit()
mhandlers = {}
modules = []
for path, module in sys.modules.items():
r = re.findall("misp_modules[.]modules[.](\w+)[.](\w+)", path)
if r and len(r[0]) == 2:
moduletype, modulename = r[0]
mhandlers[modulename] = module
modules.append(modulename)
log.info('MISP modules {0} imported'.format(modulename))
mhandlers['type:' + modulename] = moduletype
return mhandlers, modules
class ListModules(tornado.web.RequestHandler):
def get(self):
global mhandlers
global loaded_modules
ret = []
for module in loaded_modules:
x = {}
x['name'] = module
x['type'] = mhandlers['type:' + module]
x['mispattributes'] = mhandlers[module].introspection()
x['meta'] = mhandlers[module].version()
ret.append(x)
log.debug('MISP ListModules request')
self.write(json.dumps(ret))
class QueryModule(tornado.web.RequestHandler):
def post(self):
global mhandlers
jsonpayload = self.request.body.decode('utf-8')
x = json.loads(jsonpayload)
log.debug('MISP QueryModule request {0}'.format(jsonpayload))
ret = mhandlers[x['module']].handler(q=jsonpayload)
self.write(json.dumps(ret))
def main():
global mhandlers
global loaded_modules
signal.signal(signal.SIGINT, handle_signal)
signal.signal(signal.SIGTERM, handle_signal)
argParser = argparse.ArgumentParser(description='misp-modules server')
argParser.add_argument('-t', default=False, action='store_true', help='Test mode')
argParser.add_argument('-s', default=False, action='store_true', help='Run a system install (package installed via pip)')
argParser.add_argument('-p', default=6666, help='misp-modules TCP port (default 6666)')
argParser.add_argument('-l', default='localhost', help='misp-modules listen address (default localhost)')
args = argParser.parse_args()
port = args.p
listen = args.l
log = init_logger()
if args.s:
load_package_helpers()
mhandlers, loaded_modules = load_package_modules()
else:
os.chdir(os.path.dirname(__file__))
modulesdir = 'modules'
helpersdir = 'helpers'
load_helpers(helpersdir=helpersdir)
mhandlers, loaded_modules = load_modules(modulesdir)
service = [(r'/modules', ListModules), (r'/query', QueryModule)]
application = tornado.web.Application(service)
application.listen(port, address=listen)
log.info('MISP modules server started on {0} port {1}'.format(listen, port))
if args.t:
log.info('MISP modules started in test-mode, quitting immediately.')
sys.exit()
IOLoop.instance().start()
IOLoop.instance().stop()
return 0
if __name__ == '__main__':
sys.exit(main())

2
misp_modules/modules/__init__.py
View File

@ -1 +1,3 @@
from .expansion import *
from .import_mod import *
from .export_mod import *

2
misp_modules/modules/expansion/__init__.py
View File

@ -1,2 +1,2 @@
__all__ = ['asn_history', 'circl_passivedns', 'circl_passivessl', 'cve', 'dns',
'eupi', 'ipasn', 'passivetotal', 'sourcecache']
'eupi', 'ipasn', 'passivetotal', 'sourcecache', 'whois']

5
misp_modules/modules/expansion/whois.py
View File

@ -1,7 +1,10 @@
# -*- coding: utf-8 -*-
import json
from uwhois import Uwhois
try:
from uwhois import Uwhois
except ImportError:
print("uwhois module not installed.")
misperrors = {'error': 'Error'}
mispattributes = {'input': ['domain', 'ip-src', 'ip-dst'], 'output': ['freetext']}

1
misp_modules/modules/export_mod/__init__.py Normal file
View File

@ -0,0 +1 @@
__all__ = ['testexport']

1
misp_modules/modules/export/testexport.py → misp_modules/modules/export_mod/testexport.py
View File

@ -9,6 +9,7 @@ userConfig = {
};
moduleconfig = []
# fixed for now, options in the future:
# event, attribute, event-collection, attribute-collection

1
misp_modules/modules/import_mod/__init__.py Normal file
View File

@ -0,0 +1 @@
__all__ = ['testimport', 'ocr']

0
misp_modules/modules/import/ocr.py → misp_modules/modules/import_mod/ocr.py
View File

0
misp_modules/modules/import/testimport.py → misp_modules/modules/import_mod/testimport.py
View File

4
setup.py
View File

@ -11,7 +11,7 @@ setup(
url='https://github.com/MISP/misp-modules',
description='MISP modules are autonomous modules that can be used for expansion and other services in MISP',
packages=find_packages(),
scripts=['bin/misp-modules'],
entry_points = {'console_scripts': ['misp-modules = misp_modules:main']},
test_suite="tests",
classifiers=[
'License :: OSI Approved :: GNU Affero General Public License v3',
@ -35,5 +35,7 @@ setup(
'asnhistory',
'stix',
'cybox'
'pillow',
'pytesseract',
]
)