chg: Use New version of PyMISP in the feed generator

examples/feed-generator/generate.py

@@ -4,83 +4,11 @@
 import sys
 import json
 import os
-import hashlib
+from pymisp import ExpandedPyMISP
-from pymisp import PyMISP
 from settings import url, key, ssl, outputdir, filters, valid_attribute_distribution_levels
 
-objectsFields = {
-    'Attribute': {
-        'uuid',
-        'value',
-        'category',
-        'type',
-        'comment',
-        'data',
-        'timestamp',
-        'to_ids',
-        'object_relation',
-        'disable_correlation'
-    },
-    'Event': {
-        'uuid',
-        'info',
-        'threat_level_id',
-        'analysis',
-        'timestamp',
-        'publish_timestamp',
-        'published',
-        'date',
-        'extends_uuid'
-    },
-    'Object': {
-        'name',
-        'meta-category',
-        'description',
-        'template_uuid',
-        'template_version',
-        'uuid',
-        'timestamp',
-        'distribution',
-        'sharing_group_id',
-        'comment'
-    },
-    'ObjectReference': {
-        'uuid',
-        'timestamp',
-        'relationship_type',
-        'comment',
-        'object_uuid',
-        'referenced_uuid'
-    },
-    'Orgc': {
-        'name',
-        'uuid'
-    },
-    'Tag': {
-        'name',
-        'colour',
-        'exportable'
-    }
-}
-
-objectsToSave = {
-    'Orgc': {},
-    'Tag': {},
-    'Attribute': {
-        'Tag': {}
-    },
-    'Object': {
-        'Attribute': {
-            'Tag': {}
-        },
-        'ObjectReference': {}
-    }
-}
-
 valid_attribute_distributions = []
 
-attributeHashes = []
-
 
 def init():
     # If we have an old settings.py file then this variable won't exist
@@ -89,66 +17,23 @@ def init():
         valid_attribute_distributions = valid_attribute_distribution_levels
     except Exception:
         valid_attribute_distributions = ['0', '1', '2', '3', '4', '5']
-    return PyMISP(url, key, ssl)
+    return ExpandedPyMISP(url, key, ssl)
 
 
-def recursiveExtract(container, containerType, leaf, eventUuid):
+def saveEvent(event):
-    temp = {}
-    if containerType in ['Attribute', 'Object']:
-        if (__blockByDistribution(container)):
-            return False
-    for field in objectsFields[containerType]:
-        if field in container:
-            temp[field] = container[field]
-    if (containerType == 'Attribute'):
-        global attributeHashes
-        if ('|' in container['type'] or container['type'] == 'malware-sample'):
-            split = container['value'].split('|')
-            attributeHashes.append([hashlib.md5(split[0].encode("utf-8")).hexdigest(), eventUuid])
-            attributeHashes.append([hashlib.md5(split[1].encode("utf-8")).hexdigest(), eventUuid])
-        else:
-            attributeHashes.append([hashlib.md5(container['value'].encode("utf-8")).hexdigest(), eventUuid])
-    children = leaf.keys()
-    for childType in children:
-        childContainer = container.get(childType)
-        if (childContainer):
-            if (type(childContainer) is dict):
-                temp[childType] = recursiveExtract(childContainer, childType, leaf[childType], eventUuid)
-            else:
-                temp[childType] = []
-                for element in childContainer:
-                    processed = recursiveExtract(element, childType, leaf[childType], eventUuid)
-                    if (processed):
-                        temp[childType].append(processed)
-    return temp
-
-
-def saveEvent(misp, uuid):
-    event = misp.get_event(uuid)
-    if not event.get('Event'):
-        print('Error while fetching event: {}'.format(event['message']))
-        sys.exit('Could not create file for event ' + uuid + '.')
-    event['Event'] = recursiveExtract(event['Event'], 'Event', objectsToSave, event['Event']['uuid'])
-    event = json.dumps(event)
-    eventFile = open(os.path.join(outputdir, uuid + '.json'), 'w')
-    eventFile.write(event)
-    eventFile.close()
-
-
-def __blockByDistribution(element):
-    if element['distribution'] not in valid_attribute_distributions:
-        return True
-    return False
-
-
-def saveHashes():
-    if not attributeHashes:
-        return False
     try:
-        hashFile = open(os.path.join(outputdir, 'hashes.csv'), 'w')
+        with open(os.path.join(outputdir, f'{event["uuid"]}.json'), 'w') as f:
-        for element in attributeHashes:
+            json.dump(event, f, indent=2)
+    except Exception as e:
+        print(e)
+        sys.exit('Could not create the event dump.')
+
+
+def saveHashes(hashes):
+    try:
+        with open(os.path.join(outputdir, 'hashes.csv'), 'w') as hashFile:
+            for element in hashes:
                 hashFile.write('{},{}\n'.format(element[0], element[1]))
-        hashFile.close()
     except Exception as e:
         print(e)
         sys.exit('Could not create the quick hash lookup file.')
@@ -164,41 +49,28 @@ def saveManifest(manifest):
         sys.exit('Could not create the manifest file.')
 
 
-def __addEventToManifest(event):
-    tags = []
-    for eventTag in event['EventTag']:
-        tags.append({'name': eventTag['Tag']['name'],
-                     'colour': eventTag['Tag']['colour']})
-    return {'Orgc': event['Orgc'],
-            'Tag': tags,
-            'info': event['info'],
-            'date': event['date'],
-            'analysis': event['analysis'],
-            'threat_level_id': event['threat_level_id'],
-            'timestamp': event['timestamp']
-            }
-
-
 if __name__ == '__main__':
     misp = init()
     try:
-        r = misp.get_index(filters)
+        events = misp.search(metadata=True, limit=200, **filters, pythonify=True)
-        events = r['response']
-        print(events[0])
     except Exception as e:
         print(e)
         sys.exit("Invalid response received from MISP.")
     if len(events) == 0:
         sys.exit("No events returned.")
     manifest = {}
+    hashes = []
     counter = 1
     total = len(events)
     for event in events:
-        saveEvent(misp, event['uuid'])
+        e = misp.get_event(event.uuid, pythonify=True)
-        manifest[event['uuid']] = __addEventToManifest(event)
+        e_feed = e.to_feed()
+        hashes += [[h, e.uuid] for h in e_feed.pop('_hashes')]
+        manifest.update(e_feed.pop('_manifest'))
+        saveEvent(e_feed)
         print("Event " + str(counter) + "/" + str(total) + " exported.")
         counter += 1
     saveManifest(manifest)
     print('Manifest saved.')
-    saveHashes()
+    saveHashes(hashes)
     print('Hashes saved. Feed creation completed.')