Merge branch 'master' of https://github.com/MISP/misp-modules

README.md

@@ -0,0 +1,52 @@
+# MISP modules
+
+MISP modules are autonomous modules that can be used for expansion and other services in [MISP](https://github.com/MISP/MISP).
+
+The modules are written in Python 3 following a simple API interface. The objective is to ease the extensions of MISP functionalities
+without modifying core components. The API is available via a simple REST API which is independent from MISP installation or configuration. 
+
+MISP modules support is included in MISP starting from version 2.4.X.
+
+## Existing MISP modules
+
+* [DNS](modules/expansion/dns.py) - a simple module to resolve MISP attributes like hostname and domain to expand IP addresses attributes.
+
+## How to add your own MISP modules?
+
+Create your module in [modules/expansion/](modules/expansion/). The module should have at minimum two functions:
+
+* **introspection** function that returns an array of the supported attributes by your expansion module.
+* **handler** function which accepts a JSON document to expand the values and return a dictionary of the expanded values.
+
+## Testing your modules?
+
+MISP uses the **modules** function to discover the available MISP modules and their supported MISP attributes:
+
+~~~
+% curl -s http://127.0.0.1:6666/modules | jq .
+[
+  {
+    "name": "dns",
+    "mispattributes": [
+      "hostname",
+      "domain"
+    ]
+  }
+]
+
+~~~
+
+The MISP module service returns the available modules in a JSON array containing each module name along with their supported input attributes.
+
+Based on this information, a query can be built in a JSON format and saved as body.json:
+
+~~~json
+{"module": "dns", "hostname": "www.github.com"}
+~~~
+
+Then you can POST this JSON format query towards the MISP object server:
+
+~~~
+curl -s http://127.0.0.1:6666/query -H "Content-Type: application/json" --data @body.json -X POST
+~~~
+

bin/misp-modules.py

@@ -24,9 +24,20 @@ import sys
 import tornado.web
 import importlib
 import json
+import logging
 
 runPath = os.path.dirname(os.path.realpath(__file__))
 sys.path.append(os.path.join(runPath, '..'))
+port = 6666
+
+log = logging.getLogger('misp-modules')
+formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+handler = logging.StreamHandler(stream=sys.stdout)
+handler.setFormatter(formatter)
+handler.setLevel(logging.INFO)
+
+log.addHandler(handler)
+log.setLevel(logging.INFO)
 
 modulesdir = '../modules/expansion'
 
@@ -37,8 +48,8 @@ for module in os.listdir(modulesdir):
         continue
     modulename = module.split(".")[0]
     modules.append(modulename)
+    log.info('MISP modules {0} imported'.format(modulename))
     mhandlers[modulename] = importlib.import_module('modules.expansion.'+modulename)
-    print (module)
 
 class ListModules(tornado.web.RequestHandler):
     def get(self):
@@ -47,13 +58,14 @@ class ListModules(tornado.web.RequestHandler):
             x = {}
             x['name'] = module
             x['mispattributes'] = mhandlers[module].introspection()
-            print (x['mispattributes'])
             ret.append(x)
+        log.debug('MISP ListModules request')
         self.write(json.dumps(ret))
 class QueryModule(tornado.web.RequestHandler):
     def post(self):
         jsonpayload = self.request.body.decode('utf-8')
         x=json.loads(jsonpayload)
+        log.debug('MISP QueryModule request {0}'.format(jsonpayload))
         ret = mhandlers[x['module']].handler(q=jsonpayload)
         self.write(json.dumps(ret))
 
@@ -61,5 +73,6 @@ class QueryModule(tornado.web.RequestHandler):
 service = [(r'/modules',ListModules), (r'/query',QueryModule)]
 
 application = tornado.web.Application(service)
-application.listen(6666)
+log.info('MISP modules server started on TCP port {0}'.format(port))
+application.listen(port)
 tornado.ioloop.IOLoop.instance().start()

tests/query-dns.sh

@@ -1 +1 @@
-curl http://127.0.0.1:6666/query -H "Content-Type: application/json" --data @body.json -X POST
+curl -s http://127.0.0.1:6666/query -H "Content-Type: application/json" --data @body.json -X POST

tests/search-modules.sh

@@ -1 +1 @@
-curl http://127.0.0.1:6666/modules
+curl -s http://127.0.0.1:6666/modules