mirror of https://github.com/MISP/PyMISP
chg: Use New version of PyMISP in the feed generator
Raphaël Vinot
parent
61867a8257
commit
5ebaca3b52
|
|
@ -4,83 +4,11 @@
|
|||
import sys
|
||||
import json
|
||||
import os
|
||||
import hashlib
|
||||
from pymisp import PyMISP
|
||||
from pymisp import ExpandedPyMISP
|
||||
from settings import url, key, ssl, outputdir, filters, valid_attribute_distribution_levels
|
||||
|
||||
objectsFields = {
|
||||
'Attribute': {
|
||||
'uuid',
|
||||
'value',
|
||||
'category',
|
||||
'type',
|
||||
'comment',
|
||||
'data',
|
||||
'timestamp',
|
||||
'to_ids',
|
||||
'object_relation',
|
||||
'disable_correlation'
|
||||
},
|
||||
'Event': {
|
||||
'uuid',
|
||||
'info',
|
||||
'threat_level_id',
|
||||
'analysis',
|
||||
'timestamp',
|
||||
'publish_timestamp',
|
||||
'published',
|
||||
'date',
|
||||
'extends_uuid'
|
||||
},
|
||||
'Object': {
|
||||
'name',
|
||||
'meta-category',
|
||||
'description',
|
||||
'template_uuid',
|
||||
'template_version',
|
||||
'uuid',
|
||||
'timestamp',
|
||||
'distribution',
|
||||
'sharing_group_id',
|
||||
'comment'
|
||||
},
|
||||
'ObjectReference': {
|
||||
'uuid',
|
||||
'timestamp',
|
||||
'relationship_type',
|
||||
'comment',
|
||||
'object_uuid',
|
||||
'referenced_uuid'
|
||||
},
|
||||
'Orgc': {
|
||||
'name',
|
||||
'uuid'
|
||||
},
|
||||
'Tag': {
|
||||
'name',
|
||||
'colour',
|
||||
'exportable'
|
||||
}
|
||||
}
|
||||
|
||||
objectsToSave = {
|
||||
'Orgc': {},
|
||||
'Tag': {},
|
||||
'Attribute': {
|
||||
'Tag': {}
|
||||
},
|
||||
'Object': {
|
||||
'Attribute': {
|
||||
'Tag': {}
|
||||
},
|
||||
'ObjectReference': {}
|
||||
}
|
||||
}
|
||||
|
||||
valid_attribute_distributions = []
|
||||
|
||||
attributeHashes = []
|
||||
|
||||
|
||||
def init():
|
||||
# If we have an old settings.py file then this variable won't exist
|
||||
|
|
@ -89,66 +17,23 @@ def init():
|
|||
valid_attribute_distributions = valid_attribute_distribution_levels
|
||||
except Exception:
|
||||
valid_attribute_distributions = ['0', '1', '2', '3', '4', '5']
|
||||
return PyMISP(url, key, ssl)
|
||||
return ExpandedPyMISP(url, key, ssl)
|
||||
|
||||
|
||||
def recursiveExtract(container, containerType, leaf, eventUuid):
|
||||
temp = {}
|
||||
if containerType in ['Attribute', 'Object']:
|
||||
if (__blockByDistribution(container)):
|
||||
return False
|
||||
for field in objectsFields[containerType]:
|
||||
if field in container:
|
||||
temp[field] = container[field]
|
||||
if (containerType == 'Attribute'):
|
||||
global attributeHashes
|
||||
if ('|' in container['type'] or container['type'] == 'malware-sample'):
|
||||
split = container['value'].split('|')
|
||||
attributeHashes.append([hashlib.md5(split[0].encode("utf-8")).hexdigest(), eventUuid])
|
||||
attributeHashes.append([hashlib.md5(split[1].encode("utf-8")).hexdigest(), eventUuid])
|
||||
else:
|
||||
attributeHashes.append([hashlib.md5(container['value'].encode("utf-8")).hexdigest(), eventUuid])
|
||||
children = leaf.keys()
|
||||
for childType in children:
|
||||
childContainer = container.get(childType)
|
||||
if (childContainer):
|
||||
if (type(childContainer) is dict):
|
||||
temp[childType] = recursiveExtract(childContainer, childType, leaf[childType], eventUuid)
|
||||
else:
|
||||
temp[childType] = []
|
||||
for element in childContainer:
|
||||
processed = recursiveExtract(element, childType, leaf[childType], eventUuid)
|
||||
if (processed):
|
||||
temp[childType].append(processed)
|
||||
return temp
|
||||
|
||||
|
||||
def saveEvent(misp, uuid):
|
||||
event = misp.get_event(uuid)
|
||||
if not event.get('Event'):
|
||||
print('Error while fetching event: {}'.format(event['message']))
|
||||
sys.exit('Could not create file for event ' + uuid + '.')
|
||||
event['Event'] = recursiveExtract(event['Event'], 'Event', objectsToSave, event['Event']['uuid'])
|
||||
event = json.dumps(event)
|
||||
eventFile = open(os.path.join(outputdir, uuid + '.json'), 'w')
|
||||
eventFile.write(event)
|
||||
eventFile.close()
|
||||
|
||||
|
||||
def __blockByDistribution(element):
|
||||
if element['distribution'] not in valid_attribute_distributions:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def saveHashes():
|
||||
if not attributeHashes:
|
||||
return False
|
||||
def saveEvent(event):
|
||||
try:
|
||||
hashFile = open(os.path.join(outputdir, 'hashes.csv'), 'w')
|
||||
for element in attributeHashes:
|
||||
hashFile.write('{},{}\n'.format(element[0], element[1]))
|
||||
hashFile.close()
|
||||
with open(os.path.join(outputdir, f'{event["uuid"]}.json'), 'w') as f:
|
||||
json.dump(event, f, indent=2)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
sys.exit('Could not create the event dump.')
|
||||
|
||||
|
||||
def saveHashes(hashes):
|
||||
try:
|
||||
with open(os.path.join(outputdir, 'hashes.csv'), 'w') as hashFile:
|
||||
for element in hashes:
|
||||
hashFile.write('{},{}\n'.format(element[0], element[1]))
|
||||
except Exception as e:
|
||||
print(e)
|
||||
sys.exit('Could not create the quick hash lookup file.')
|
||||
|
|
@ -164,41 +49,28 @@ def saveManifest(manifest):
|
|||
sys.exit('Could not create the manifest file.')
|
||||
|
||||
|
||||
def __addEventToManifest(event):
|
||||
tags = []
|
||||
for eventTag in event['EventTag']:
|
||||
tags.append({'name': eventTag['Tag']['name'],
|
||||
'colour': eventTag['Tag']['colour']})
|
||||
return {'Orgc': event['Orgc'],
|
||||
'Tag': tags,
|
||||
'info': event['info'],
|
||||
'date': event['date'],
|
||||
'analysis': event['analysis'],
|
||||
'threat_level_id': event['threat_level_id'],
|
||||
'timestamp': event['timestamp']
|
||||
}
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
misp = init()
|
||||
try:
|
||||
r = misp.get_index(filters)
|
||||
events = r['response']
|
||||
print(events[0])
|
||||
events = misp.search(metadata=True, limit=200, **filters, pythonify=True)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
sys.exit("Invalid response received from MISP.")
|
||||
if len(events) == 0:
|
||||
sys.exit("No events returned.")
|
||||
manifest = {}
|
||||
hashes = []
|
||||
counter = 1
|
||||
total = len(events)
|
||||
for event in events:
|
||||
saveEvent(misp, event['uuid'])
|
||||
manifest[event['uuid']] = __addEventToManifest(event)
|
||||
e = misp.get_event(event.uuid, pythonify=True)
|
||||
e_feed = e.to_feed()
|
||||
hashes += [[h, e.uuid] for h in e_feed.pop('_hashes')]
|
||||
manifest.update(e_feed.pop('_manifest'))
|
||||
saveEvent(e_feed)
|
||||
print("Event " + str(counter) + "/" + str(total) + " exported.")
|
||||
counter += 1
|
||||
saveManifest(manifest)
|
||||
print('Manifest saved.')
|
||||
saveHashes()
|
||||
saveHashes(hashes)
|
||||
print('Hashes saved. Feed creation completed.')
|
||||
|
|
|
|||
Loading…
Reference in New Issue