initial hashstore commit for misp-module

misp_modules/modules/expansion/_hastore_helpers/hashstore_init.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*
+#pip install mysql-connector==2.1.6
+
+import mysql.connector
+import redis
+
+
+sql = """SELECT sha2((CASE WHEN attributes.value2 = '' THEN attributes.value1 ELSE CONCAT(attributes.value1, '|', attributes.value2) END), 256) AS hashvalue, attributes.uuid FROM attributes;"""
+
+# check misp conf
+paramMysql = {
+    'host': 'localhost',
+    'user': 'misp_user',
+    'password': 'misp_password',
+    'database': 'misp_database'
+}
+# configure a new database to store hashstore data
+paramRedis = {
+    'host': '127.0.0.1',
+    'port': 6379,
+    'db': 7,
+    'decode_responses': True,
+    'charset': 'utf-8'
+}
+
+
+conn = mysql.connector.connect(**paramMysql)
+hashStore = redis.Redis(**paramRedis)
+cursor = conn.cursor()
+cursor.execute(sql)
+rows = cursor.fetchall()
+for row in rows:
+    hashStore.sadd(row[0], row[1].decode("utf-8"))
+
+print('done')
+conn.close()

misp_modules/modules/expansion/_hastore_helpers/hashstore_sub.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*
+import zmq, json, time, hashlib
+import redis #redis-cli INFO | grep ^db
+
+# you need to execute this script to update your hashstore database from each add/edit/delet attributes
+# You can add this to /var/www/MISP/tools/misp_zmq
+
+# zmq configuration from misp configuration
+zmq_host = "127.0.0.1"
+zmq_port = "50000"
+zmq_protocol = "tcp"
+
+# redis configuration where is store your hashstore
+redis_conf = {
+    'host': '127.0.0.1',
+    'port': 6379,
+    'db': 7,
+}
+
+
+context = zmq.Context()
+socket = context.socket(zmq.SUB)
+socket.connect("{}://{}:{}".format(zmq_protocol, zmq_host, zmq_port))
+socket.setsockopt(zmq.SUBSCRIBE, b'')
+
+poller = zmq.Poller()
+poller.register(socket, zmq.POLLIN)
+
+hashStore = redis.Redis(**redis_conf)
+
+while True:
+    socks = dict(poller.poll(timeout=None))
+    if socket in socks and socks[socket] == zmq.POLLIN:
+        message = socket.recv()
+        topic, s, m = message.decode('utf-8').partition(" ")
+        jsonevent = json.loads(m)
+        jsonattr = None
+        
+       
+               
+        if 'Attribute' in jsonevent:
+
+            jsonattr = jsonevent['Attribute']
+            value = jsonattr["value"].encode('utf-8')
+            hash_object = hashlib.sha256(value).hexdigest()
+            if 'action' in jsonevent:
+                # print(jsonevent['action'])
+                if 'delete' in jsonevent['action']:
+                    hashStore.srem(hash_object, jsonattr["uuid"])
+                elif 'add' in jsonevent['action']:
+                    hashStore.sadd(hash_object, jsonattr["uuid"])
+                else:
+                    # Edit case 
+                    # add new value and remove the last
+                    hashStore.sadd(hash_object, jsonattr["uuid"])
+                    # remove old value
+                    if jsonevent.get('attribute_diff'):
+                        if jsonevent['attribute_diff'].get('value'):
+                            old_value = jsonevent['attribute_diff']['value'].encode('utf-8')
+                            
+                            old_hash_object = hashlib.sha256(old_value).hexdigest()
+                            hashStore.srem(old_hash_object, jsonattr["uuid"])
+                            
+                            # print(old_value)
+                            # print(hashStore.smembers(old_hash_object))
+                            # if the last member is deleted, the set is deleted
+                            # and exist return False
+                            # print(hashStore.exists(old_hash_object))
+
+            # print(value)
+            # print(hashStore.smembers(hash_object))
+        time.sleep(2)

misp_modules/modules/expansion/_hastore_helpers/hashstore_test.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*
+import requests
+mispUrlLocal = 'http://misp.local/modules/queryEnrichment/'
+mispKeyLocal = 'xxxxxxxx'
+
+def printResult(p):
+    session = requests.Session()
+    session.headers.update({
+        'Authorization': mispKeyLocal,
+        'Accept': 'application/json',
+        'content-type': 'application/json'})
+    r = session.post(mispUrlLocal, json=p)
+    print(r.text)
+
+
+def testSimpleHash():
+    p = {'module':'hashstore',
+        'hashs': '0002969c86f26bf044714999910ca15c1397365ff4db9b752078a425b5dac8b5'
+    }
+    printResult(p)
+
+def testListHash():
+    p = {'module':'hashstore',
+        'hashs':[
+            '0002969c86f26bf044714999910ca15c1397365ff4db9b752078a425b5dac8b5',
+            '003681a69aecd31b1d8ffd870fc0b91c2d68c46694d98d8c13bdd7ab0620d46e'
+        ]}
+    printResult(p)
+
+
+def testSimpleQuick():
+    p = {'module':'hashstore',
+        'hashs': '0002969c86f26bf044714999910ca15c1397365ff4db9b752078a425b5dac8b5',
+        'quick_search' : True}
+    printResult(p)
+
+def testListQuick():
+    p = {'module':'hashstore',
+        'hashs':[
+            '0002969c86f26bf044714999910ca15c1397365ff4db9b752078a425b5dac8b5',
+            '003681a69aecd31b1d8ffd870fc0b91c2d68c46694d98d8c13bdd7ab0620d46e'
+        ],
+        'quick_search' : True}
+    printResult(p)
+
+def testSimpleUuid():
+    p = {
+        'module':'hashstore',
+        'hashs': '0002969c86f26bf044714999910ca15c1397365ff4db9b752078a425b5dac8b5',
+        'return_uuid' : True
+    }
+    printResult(p)
+
+
+def testListUuid():
+    p = {
+        'module':'hashstore',
+        'hashs':[
+            '0002969c86f26bf044714999910ca15c1397365ff4db9b752078a425b5dac8b5',
+            '003681a69aecd31b1d8ffd870fc0b91c2d68c46694d98d8c13bdd7ab0620d46e'
+        ],
+        'return_uuid' : True
+    }
+    printResult(p)
+
+def testSimpleValue():
+    p = {
+        'module':'hashstore',
+        'values': '31.210.111.154'
+    }
+    printResult(p)
+
+def testMutlipleValue():
+    p = {
+        'module':'hashstore',
+        'values': [
+            '31.210.111.154',
+            'globaldefencetalk.com'
+            ]
+    }
+    printResult(p)
+
+if __name__ == "__main__":
+    testSimpleHash()
+    testListHash()
+    testSimpleQuick()
+    testListQuick()
+    testSimpleUuid()
+    testListUuid()
+    testSimpleValue()
+    testMutlipleValue()

misp_modules/modules/expansion/hashstore.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*
+import json, redis, hashlib
+# TODO : test sha256 len
+
+misperrors = {'error': 'Error'}
+mispattributes = {'input': ['hashs', 'values'], 'output': ['bool', 'uuid']}
+moduleinfo = {'version': '0.1', 'author': 'Tristan Métayer',
+              'description': 'Is this hashs (sha256) or stric values in misp database ?',
+              'module-type': ['expansion', 'hover']}
+
+moduleconfig = ['hashstore_host', 'hashstore_port', 'hashstore_db']
+
+def exists(store, keys, return_uuid=False, quick_search=False):
+    if quick_search is False:
+        if return_uuid is False:
+            if isinstance(keys, str):
+                return {keys: store.exists(keys)}
+            return [{key: store.exists(key)} for key in keys]
+        else:
+            if isinstance(keys, str):
+                return {keys: list(store.smembers(keys))}
+            return [{key: list(store.smembers(key))} for key in keys]
+    else:
+        with store.pipeline() as pipe:
+            if isinstance(keys, str):
+                pipe.exists(keys)
+            else :
+                for key in keys:
+                    pipe.exists(key)
+            return pipe.execute()
+
+
+def handler(q=False):
+    if q is False:
+        return False
+    q = json.loads(q)
+
+    quick_search = False
+    return_uuid = False
+    results = {}
+
+    # Test if there are some hashs or values in payload
+    if q.get('hashs') is False and q.get('values') is False:
+        misperrors['error'] = "No hashs of values in post data"
+        return misperrors
+
+    if q.get('quick_search'):
+        quick_search = q['quick_search']
+
+    if q.get('return_uuid'):
+        return_uuid = q['return_uuid']
+
+    # default redis value
+    paramRedis = {
+        'host': '127.0.0.1',
+        'port': 6379,
+        'db': 7,
+        'decode_responses': True,
+        'encoding' : 'utf-8'
+    }
+    if q.get('config'):
+        if q['config'].get('hashstore_host'):
+            paramRedis['host'] = q['config'].get('hashstore_host')
+        if q['config'].get('hashstore_port'):
+            paramRedis['port'] = q['config'].get('hashstore_port')
+        if q['config'].get('hashstore_db'):
+            paramRedis['db'] = q['config'].get('hashstore_db')
+
+    # Connect to redis
+    store = redis.Redis(**paramRedis)
+    
+
+    if q.get('hashs'):
+        results['hashs'] = exists(store, q['hashs'], return_uuid, quick_search)
+
+    if q.get('values'):
+        hashFromValue = None
+        if isinstance(q['values'], str):
+            hashFromValue = hashlib.sha256(q['values'].encode('utf-8')).hexdigest()
+        else:
+            hashFromValue = [hashlib.sha256(x.encode('utf-8')).hexdigest() for x in q['values']]
+        results['hash_values'] = exists(store, hashFromValue, return_uuid, quick_search)
+
+    return {'results':results}
+
+
+
+def introspection():
+    return mispattributes
+
+
+def version():
+    moduleinfo['config'] = moduleconfig
+    return moduleinfo