diff --git a/README.md b/README.md
index 15511b8..655e060 100644
--- a/README.md
+++ b/README.md
@@ -30,8 +30,8 @@ sudo apt-get install python3-dev python3-pip libpq5
cd /usr/local/src/
sudo git clone https://github.com/MISP/misp-modules.git
cd misp-modules
-sudo pip3 install -r REQUIREMENTS
-sudo pip3 install .
+sudo pip3 install --upgrade -r REQUIREMENTS
+sudo pip3 install --upgrade .
sudo vi /etc/rc.local, add this line: `sudo -u www-data misp-modules -s`
~~~~
diff --git a/REQUIREMENTS b/REQUIREMENTS
index 063978b..f138191 100644
--- a/REQUIREMENTS
+++ b/REQUIREMENTS
@@ -12,3 +12,5 @@ pyeupi
ipasn-redis
asnhistory
git+https://github.com/Rafiot/uwhoisd.git@testing#egg=uwhois&subdirectory=client
+pillow
+pytesseract
diff --git a/bin/misp-modules b/bin/misp-modules
deleted file mode 100755
index 011d767..0000000
--- a/bin/misp-modules
+++ /dev/null
@@ -1,190 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-#
-# Core MISP expansion modules loader and web service
-#
-# Copyright (C) 2016 Alexandre Dulaunoy
-# Copyright (C) 2016 CIRCL - Computer Incident Response Center Luxembourg
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see .
-
-import os
-import sys
-import tornado.web
-import importlib
-import json
-import logging
-import fnmatch
-import argparse
-import re
-
-try:
- import misp_modules.modules
- HAS_PACKAGE_MODULES = True
-except Exception as e:
- print(e)
- HAS_PACKAGE_MODULES = False
-
-try:
- from misp_modules.helpers import *
- HAS_PACKAGE_HELPERS = True
-except Exception as e:
- print(e)
- HAS_PACKAGE_HELPERS = False
-
-
-def init_logger():
- log = logging.getLogger('misp-modules')
- formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
- handler = logging.StreamHandler(stream=sys.stdout)
- handler.setFormatter(formatter)
- handler.setLevel(logging.INFO)
-
- log.addHandler(handler)
- log.setLevel(logging.INFO)
- return log
-
-
-def load_helpers(helpersdir):
- sys.path.append(helpersdir)
- hhandlers = {}
- helpers = []
- for root, dirnames, filenames in os.walk(helpersdir):
- if os.path.basename(root) == '__pycache__':
- continue
- if re.match(r'^\.', os.path.basename(root)):
- continue
- for filename in fnmatch.filter(filenames, '*.py'):
- if filename == '__init__.py':
- continue
- helpername = filename.split(".")[0]
- hhandlers[helpername] = importlib.import_module(helpername)
- selftest = hhandlers[helpername].selftest()
- if selftest is None:
- helpers.append(helpername)
- log.info('Helpers loaded {} '.format(filename))
- else:
- log.info('Helpers failed {} due to {}'.format(filename, selftest))
-
-
-def load_package_helpers():
- if not HAS_PACKAGE_HELPERS:
- log.info('Unable to load MISP helpers from package.')
- sys.exit()
- mhandlers = {}
- helpers = []
- for path, helper in sys.modules.items():
- if not path.startswith('misp_modules.helpers.'):
- continue
- helpername = path.replace('misp_modules.helpers.', '')
- mhandlers[helpername] = helper
- helpers.append(helpername)
- log.info('Helper loaded {}'.format(helpername))
- return mhandlers, helpers
-
-
-def load_modules(mod_dir):
- sys.path.append(mod_dir)
- mhandlers = {}
- modules = []
- for root, dirnames, filenames in os.walk(mod_dir):
- if os.path.basename(root) == '__pycache__':
- continue
- if os.path.basename(root).startswith("."):
- continue
- for filename in fnmatch.filter(filenames, '*.py'):
- if filename == '__init__.py':
- continue
- modulename = filename.split(".")[0]
- moduletype = os.path.split(modulesdir)[1]
- try:
- mhandlers[modulename] = importlib.import_module(os.path.basename(root) + '.' + modulename)
- except Exception as e:
- log.warning('MISP modules {0} failed due to {1}'.format(modulename, e))
- continue
- modules.append(modulename)
- log.info('MISP modules {0} imported'.format(modulename))
- mhandlers['type:' + modulename] = moduletype
- return mhandlers, modules
-
-
-def load_package_modules():
- if not HAS_PACKAGE_MODULES:
- log.info('Unable to load MISP modules from package.')
- sys.exit()
- mhandlers = {}
- modules = []
- for path, module in sys.modules.items():
- r = re.findall("misp_modules[.]modules[.](\w+)[.](\w+)", path)
- if r and len(r[0]) == 2:
- moduletype, modulename = r[0]
- mhandlers[modulename] = module
- modules.append(modulename)
- log.info('MISP modules {0} imported'.format(modulename))
- mhandlers['type:' + modulename] = moduletype
- return mhandlers, modules
-
-
-class ListModules(tornado.web.RequestHandler):
- def get(self):
- ret = []
- for module in modules:
- x = {}
- x['name'] = module
- x['type'] = mhandlers['type:' + module]
- x['mispattributes'] = mhandlers[module].introspection()
- x['meta'] = mhandlers[module].version()
- ret.append(x)
- log.debug('MISP ListModules request')
- self.write(json.dumps(ret))
-
-
-class QueryModule(tornado.web.RequestHandler):
- def post(self):
- jsonpayload = self.request.body.decode('utf-8')
- x = json.loads(jsonpayload)
- log.debug('MISP QueryModule request {0}'.format(jsonpayload))
- ret = mhandlers[x['module']].handler(q=jsonpayload)
- self.write(json.dumps(ret))
-
-
-if __name__ == '__main__':
- if os.path.dirname(__file__) in ['.', '']:
- os.chdir('../')
- argParser = argparse.ArgumentParser(description='misp-modules server')
- argParser.add_argument('-t', default=False, action='store_true', help='Test mode')
- argParser.add_argument('-s', default=False, action='store_true', help='Run a system install (package installed via pip)')
- argParser.add_argument('-p', default=6666, help='misp-modules TCP port (default 6666)')
- argParser.add_argument('-l', default='localhost', help='misp-modules listen address (default localhost)')
- args = argParser.parse_args()
- port = args.p
- listen = args.l
- log = init_logger()
- if args.s:
- load_package_helpers()
- mhandlers, modules = load_package_modules()
- else:
- modulesdir = 'misp_modules/modules'
- helpersdir = 'misp_modules/helpers'
- load_helpers(helpersdir=helpersdir)
- mhandlers, modules = load_modules(modulesdir)
- service = [(r'/modules', ListModules), (r'/query', QueryModule)]
-
- application = tornado.web.Application(service)
- application.listen(port, address=listen)
- log.info('MISP modules server started on {0} port {1}'.format(listen, port))
- if args.t:
- log.info('MISP modules started in test-mode, quitting immediately.')
- sys.exit()
- tornado.ioloop.IOLoop.instance().start()
diff --git a/misp_modules/__init__.py b/misp_modules/__init__.py
index e69de29..2617fae 100644
--- a/misp_modules/__init__.py
+++ b/misp_modules/__init__.py
@@ -0,0 +1,213 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Core MISP expansion modules loader and web service
+#
+# Copyright (C) 2016 Alexandre Dulaunoy
+# Copyright (C) 2016 CIRCL - Computer Incident Response Center Luxembourg
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see .
+
+import os
+import signal
+import sys
+from tornado.ioloop import IOLoop
+import tornado.web
+import importlib
+import json
+import logging
+import fnmatch
+import argparse
+import re
+
+try:
+ from .modules import *
+ HAS_PACKAGE_MODULES = True
+except Exception as e:
+ print(e)
+ HAS_PACKAGE_MODULES = False
+
+try:
+ from .helpers import *
+ HAS_PACKAGE_HELPERS = True
+except Exception as e:
+ print(e)
+ HAS_PACKAGE_HELPERS = False
+
+log = logging.getLogger('misp-modules')
+
+
+def handle_signal(sig, frame):
+ IOLoop.instance().add_callback(IOLoop.instance().stop)
+
+
+def init_logger():
+ formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+ handler = logging.StreamHandler(stream=sys.stdout)
+ handler.setFormatter(formatter)
+ handler.setLevel(logging.INFO)
+
+ log.addHandler(handler)
+ log.setLevel(logging.INFO)
+ return log
+
+
+def load_helpers(helpersdir):
+ sys.path.append(helpersdir)
+ hhandlers = {}
+ helpers = []
+ for root, dirnames, filenames in os.walk(helpersdir):
+ if os.path.basename(root) == '__pycache__':
+ continue
+ if re.match(r'^\.', os.path.basename(root)):
+ continue
+ for filename in fnmatch.filter(filenames, '*.py'):
+ if filename == '__init__.py':
+ continue
+ helpername = filename.split(".")[0]
+ hhandlers[helpername] = importlib.import_module(helpername)
+ selftest = hhandlers[helpername].selftest()
+ if selftest is None:
+ helpers.append(helpername)
+ log.info('Helpers loaded {} '.format(filename))
+ else:
+ log.info('Helpers failed {} due to {}'.format(filename, selftest))
+
+
+def load_package_helpers():
+ if not HAS_PACKAGE_HELPERS:
+ log.info('Unable to load MISP helpers from package.')
+ sys.exit()
+ mhandlers = {}
+ helpers = []
+ for path, helper in sys.modules.items():
+ if not path.startswith('misp_modules.helpers.'):
+ continue
+ helpername = path.replace('misp_modules.helpers.', '')
+ mhandlers[helpername] = helper
+ selftest = mhandlers[helpername].selftest()
+ if selftest is None:
+ helpers.append(helpername)
+ log.info('Helper loaded {}'.format(helpername))
+ else:
+ log.info('Helpers failed {} due to {}'.format(helpername, selftest))
+ return mhandlers, helpers
+
+
+def load_modules(mod_dir):
+ sys.path.append(mod_dir)
+ mhandlers = {}
+ modules = []
+ for root, dirnames, filenames in os.walk(mod_dir):
+ if os.path.basename(root) == '__pycache__':
+ continue
+ if os.path.basename(root).startswith("."):
+ continue
+ for filename in fnmatch.filter(filenames, '*.py'):
+ if filename == '__init__.py':
+ continue
+ modulename = filename.split(".")[0]
+ moduletype = os.path.split(mod_dir)[1]
+ try:
+ mhandlers[modulename] = importlib.import_module(os.path.basename(root) + '.' + modulename)
+ except Exception as e:
+ log.warning('MISP modules {0} failed due to {1}'.format(modulename, e))
+ continue
+ modules.append(modulename)
+ log.info('MISP modules {0} imported'.format(modulename))
+ mhandlers['type:' + modulename] = moduletype
+ return mhandlers, modules
+
+
+def load_package_modules():
+ if not HAS_PACKAGE_MODULES:
+ log.info('Unable to load MISP modules from package.')
+ sys.exit()
+ mhandlers = {}
+ modules = []
+ for path, module in sys.modules.items():
+ r = re.findall("misp_modules[.]modules[.](\w+)[.](\w+)", path)
+ if r and len(r[0]) == 2:
+ moduletype, modulename = r[0]
+ mhandlers[modulename] = module
+ modules.append(modulename)
+ log.info('MISP modules {0} imported'.format(modulename))
+ mhandlers['type:' + modulename] = moduletype
+ return mhandlers, modules
+
+
+class ListModules(tornado.web.RequestHandler):
+ def get(self):
+ global mhandlers
+ global loaded_modules
+ ret = []
+ for module in loaded_modules:
+ x = {}
+ x['name'] = module
+ x['type'] = mhandlers['type:' + module]
+ x['mispattributes'] = mhandlers[module].introspection()
+ x['meta'] = mhandlers[module].version()
+ ret.append(x)
+ log.debug('MISP ListModules request')
+ self.write(json.dumps(ret))
+
+
+class QueryModule(tornado.web.RequestHandler):
+ def post(self):
+ global mhandlers
+ jsonpayload = self.request.body.decode('utf-8')
+ x = json.loads(jsonpayload)
+ log.debug('MISP QueryModule request {0}'.format(jsonpayload))
+ ret = mhandlers[x['module']].handler(q=jsonpayload)
+ self.write(json.dumps(ret))
+
+
+def main():
+ global mhandlers
+ global loaded_modules
+ signal.signal(signal.SIGINT, handle_signal)
+ signal.signal(signal.SIGTERM, handle_signal)
+ argParser = argparse.ArgumentParser(description='misp-modules server')
+ argParser.add_argument('-t', default=False, action='store_true', help='Test mode')
+ argParser.add_argument('-s', default=False, action='store_true', help='Run a system install (package installed via pip)')
+ argParser.add_argument('-p', default=6666, help='misp-modules TCP port (default 6666)')
+ argParser.add_argument('-l', default='localhost', help='misp-modules listen address (default localhost)')
+ args = argParser.parse_args()
+ port = args.p
+ listen = args.l
+ log = init_logger()
+ if args.s:
+ load_package_helpers()
+ mhandlers, loaded_modules = load_package_modules()
+ else:
+ os.chdir(os.path.dirname(__file__))
+ modulesdir = 'modules'
+ helpersdir = 'helpers'
+ load_helpers(helpersdir=helpersdir)
+ mhandlers, loaded_modules = load_modules(modulesdir)
+ service = [(r'/modules', ListModules), (r'/query', QueryModule)]
+
+ application = tornado.web.Application(service)
+ application.listen(port, address=listen)
+ log.info('MISP modules server started on {0} port {1}'.format(listen, port))
+ if args.t:
+ log.info('MISP modules started in test-mode, quitting immediately.')
+ sys.exit()
+ IOLoop.instance().start()
+ IOLoop.instance().stop()
+ return 0
+
+
+if __name__ == '__main__':
+ sys.exit(main())
diff --git a/misp_modules/modules/__init__.py b/misp_modules/modules/__init__.py
index f0f4cdb..65ce6b2 100644
--- a/misp_modules/modules/__init__.py
+++ b/misp_modules/modules/__init__.py
@@ -1 +1,3 @@
from .expansion import *
+from .import_mod import *
+from .export_mod import *
diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py
index 5854e40..79fabff 100644
--- a/misp_modules/modules/expansion/__init__.py
+++ b/misp_modules/modules/expansion/__init__.py
@@ -1,2 +1,2 @@
__all__ = ['asn_history', 'circl_passivedns', 'circl_passivessl', 'cve', 'dns',
- 'eupi', 'ipasn', 'passivetotal', 'sourcecache']
+ 'eupi', 'ipasn', 'passivetotal', 'sourcecache', 'whois']
diff --git a/misp_modules/modules/expansion/whois.py b/misp_modules/modules/expansion/whois.py
index 5f3602e..4aec40c 100755
--- a/misp_modules/modules/expansion/whois.py
+++ b/misp_modules/modules/expansion/whois.py
@@ -1,7 +1,10 @@
# -*- coding: utf-8 -*-
import json
-from uwhois import Uwhois
+try:
+ from uwhois import Uwhois
+except ImportError:
+ print("uwhois module not installed.")
misperrors = {'error': 'Error'}
mispattributes = {'input': ['domain', 'ip-src', 'ip-dst'], 'output': ['freetext']}
diff --git a/misp_modules/modules/export_mod/__init__.py b/misp_modules/modules/export_mod/__init__.py
new file mode 100644
index 0000000..35cc7cb
--- /dev/null
+++ b/misp_modules/modules/export_mod/__init__.py
@@ -0,0 +1 @@
+__all__ = ['testexport']
diff --git a/misp_modules/modules/export/testexport.py b/misp_modules/modules/export_mod/testexport.py
similarity index 98%
rename from misp_modules/modules/export/testexport.py
rename to misp_modules/modules/export_mod/testexport.py
index 14829cf..ed93228 100755
--- a/misp_modules/modules/export/testexport.py
+++ b/misp_modules/modules/export_mod/testexport.py
@@ -9,6 +9,7 @@ userConfig = {
};
+moduleconfig = []
# fixed for now, options in the future:
# event, attribute, event-collection, attribute-collection
diff --git a/misp_modules/modules/import_mod/__init__.py b/misp_modules/modules/import_mod/__init__.py
new file mode 100644
index 0000000..5716751
--- /dev/null
+++ b/misp_modules/modules/import_mod/__init__.py
@@ -0,0 +1 @@
+__all__ = ['testimport', 'ocr']
diff --git a/misp_modules/modules/import/ocr.py b/misp_modules/modules/import_mod/ocr.py
similarity index 100%
rename from misp_modules/modules/import/ocr.py
rename to misp_modules/modules/import_mod/ocr.py
diff --git a/misp_modules/modules/import/testimport.py b/misp_modules/modules/import_mod/testimport.py
similarity index 100%
rename from misp_modules/modules/import/testimport.py
rename to misp_modules/modules/import_mod/testimport.py
diff --git a/setup.py b/setup.py
index 241f689..ec30f92 100644
--- a/setup.py
+++ b/setup.py
@@ -11,7 +11,7 @@ setup(
url='https://github.com/MISP/misp-modules',
description='MISP modules are autonomous modules that can be used for expansion and other services in MISP',
packages=find_packages(),
- scripts=['bin/misp-modules'],
+ entry_points = {'console_scripts': ['misp-modules = misp_modules:main']},
test_suite="tests",
classifiers=[
'License :: OSI Approved :: GNU Affero General Public License v3',
@@ -35,5 +35,7 @@ setup(
'asnhistory',
'stix',
'cybox'
+ 'pillow',
+ 'pytesseract',
]
)