diff --git a/examples/feed-generator/generate.py b/examples/feed-generator/generate.py index 1400813..991b2da 100755 --- a/examples/feed-generator/generate.py +++ b/examples/feed-generator/generate.py @@ -4,83 +4,11 @@ import sys import json import os -import hashlib -from pymisp import PyMISP +from pymisp import ExpandedPyMISP from settings import url, key, ssl, outputdir, filters, valid_attribute_distribution_levels -objectsFields = { - 'Attribute': { - 'uuid', - 'value', - 'category', - 'type', - 'comment', - 'data', - 'timestamp', - 'to_ids', - 'object_relation', - 'disable_correlation' - }, - 'Event': { - 'uuid', - 'info', - 'threat_level_id', - 'analysis', - 'timestamp', - 'publish_timestamp', - 'published', - 'date', - 'extends_uuid' - }, - 'Object': { - 'name', - 'meta-category', - 'description', - 'template_uuid', - 'template_version', - 'uuid', - 'timestamp', - 'distribution', - 'sharing_group_id', - 'comment' - }, - 'ObjectReference': { - 'uuid', - 'timestamp', - 'relationship_type', - 'comment', - 'object_uuid', - 'referenced_uuid' - }, - 'Orgc': { - 'name', - 'uuid' - }, - 'Tag': { - 'name', - 'colour', - 'exportable' - } -} - -objectsToSave = { - 'Orgc': {}, - 'Tag': {}, - 'Attribute': { - 'Tag': {} - }, - 'Object': { - 'Attribute': { - 'Tag': {} - }, - 'ObjectReference': {} - } -} - valid_attribute_distributions = [] -attributeHashes = [] - def init(): # If we have an old settings.py file then this variable won't exist @@ -89,66 +17,23 @@ def init(): valid_attribute_distributions = valid_attribute_distribution_levels except Exception: valid_attribute_distributions = ['0', '1', '2', '3', '4', '5'] - return PyMISP(url, key, ssl) + return ExpandedPyMISP(url, key, ssl) -def recursiveExtract(container, containerType, leaf, eventUuid): - temp = {} - if containerType in ['Attribute', 'Object']: - if (__blockByDistribution(container)): - return False - for field in objectsFields[containerType]: - if field in container: - temp[field] = container[field] - if (containerType == 'Attribute'): - global attributeHashes - if ('|' in container['type'] or container['type'] == 'malware-sample'): - split = container['value'].split('|') - attributeHashes.append([hashlib.md5(split[0].encode("utf-8")).hexdigest(), eventUuid]) - attributeHashes.append([hashlib.md5(split[1].encode("utf-8")).hexdigest(), eventUuid]) - else: - attributeHashes.append([hashlib.md5(container['value'].encode("utf-8")).hexdigest(), eventUuid]) - children = leaf.keys() - for childType in children: - childContainer = container.get(childType) - if (childContainer): - if (type(childContainer) is dict): - temp[childType] = recursiveExtract(childContainer, childType, leaf[childType], eventUuid) - else: - temp[childType] = [] - for element in childContainer: - processed = recursiveExtract(element, childType, leaf[childType], eventUuid) - if (processed): - temp[childType].append(processed) - return temp - - -def saveEvent(misp, uuid): - event = misp.get_event(uuid) - if not event.get('Event'): - print('Error while fetching event: {}'.format(event['message'])) - sys.exit('Could not create file for event ' + uuid + '.') - event['Event'] = recursiveExtract(event['Event'], 'Event', objectsToSave, event['Event']['uuid']) - event = json.dumps(event) - eventFile = open(os.path.join(outputdir, uuid + '.json'), 'w') - eventFile.write(event) - eventFile.close() - - -def __blockByDistribution(element): - if element['distribution'] not in valid_attribute_distributions: - return True - return False - - -def saveHashes(): - if not attributeHashes: - return False +def saveEvent(event): try: - hashFile = open(os.path.join(outputdir, 'hashes.csv'), 'w') - for element in attributeHashes: - hashFile.write('{},{}\n'.format(element[0], element[1])) - hashFile.close() + with open(os.path.join(outputdir, f'{event["uuid"]}.json'), 'w') as f: + json.dump(event, f, indent=2) + except Exception as e: + print(e) + sys.exit('Could not create the event dump.') + + +def saveHashes(hashes): + try: + with open(os.path.join(outputdir, 'hashes.csv'), 'w') as hashFile: + for element in hashes: + hashFile.write('{},{}\n'.format(element[0], element[1])) except Exception as e: print(e) sys.exit('Could not create the quick hash lookup file.') @@ -164,41 +49,28 @@ def saveManifest(manifest): sys.exit('Could not create the manifest file.') -def __addEventToManifest(event): - tags = [] - for eventTag in event['EventTag']: - tags.append({'name': eventTag['Tag']['name'], - 'colour': eventTag['Tag']['colour']}) - return {'Orgc': event['Orgc'], - 'Tag': tags, - 'info': event['info'], - 'date': event['date'], - 'analysis': event['analysis'], - 'threat_level_id': event['threat_level_id'], - 'timestamp': event['timestamp'] - } - - if __name__ == '__main__': misp = init() try: - r = misp.get_index(filters) - events = r['response'] - print(events[0]) + events = misp.search(metadata=True, limit=200, **filters, pythonify=True) except Exception as e: print(e) sys.exit("Invalid response received from MISP.") if len(events) == 0: sys.exit("No events returned.") manifest = {} + hashes = [] counter = 1 total = len(events) for event in events: - saveEvent(misp, event['uuid']) - manifest[event['uuid']] = __addEventToManifest(event) + e = misp.get_event(event.uuid, pythonify=True) + e_feed = e.to_feed() + hashes += [[h, e.uuid] for h in e_feed.pop('_hashes')] + manifest.update(e_feed.pop('_manifest')) + saveEvent(e_feed) print("Event " + str(counter) + "/" + str(total) + " exported.") counter += 1 saveManifest(manifest) print('Manifest saved.') - saveHashes() + saveHashes(hashes) print('Hashes saved. Feed creation completed.')