MISP
/
PyMISP
mirror of https://github.com/MISP/PyMISP
2
0
Fork 0
Code Issues Releases Wiki Activity

Merge remote-tracking branch 'upstream/master'

pull/142/head
c-goes 2017-11-20 15:10:24 +01:00
parent dee33507e7 635d9df8c0
commit 7409f77b52
10 changed files with 277 additions and 150 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
CHANGELOG.txt
View File

@ -2,6 +2,78 @@ Changelog
=========
v2.4.82 (2017-11-09)
--------------------
New
~~~
- Proper debug system. [Raphaël Vinot]
Make it easy to investigate the json blobs sent to the server.
Changes
~~~~~~~
- Bump misp-objects. [Raphaël Vinot]
- Update readme for new logging system. [Raphaël Vinot]
- Small improvments in the logging system. [Raphaël Vinot]
- Properly use python logging module. [Raphaël Vinot]
- Update asciidoctor generator. [Raphaël Vinot]
- Remove warning if PyMISP is too new. [Raphaël Vinot]
- Add simple asciidoc generator for MISP event. [Raphaël Vinot]
- Update changelog. [Raphaël Vinot]
Fix
~~~
- Typo loger -> logger. [Raphaël Vinot]
- Let load unknown object relations in known templates. [Raphaël Vinot]
This isn't recommended, but happens very often.
- Allow to load non-malware ZIP files in MISP Event. [Raphaël Vinot]
Prior to his patch, any zip file loaded by MISP Event was unpacked and
processed as an excrypted malware from MISP.
- Properly pass the distribution when uploading a sample. [Raphaël
Vinot]
- Properly upload a sample in an existing event. [Raphaël Vinot]
Fix https://github.com/MISP/PyMISP/issues/123
- Properly set the distribution at event level. [Raphaël Vinot]
fix #120
- Properly pop the distribution key. [Raphaël Vinot]
- Update dependencies for VT generator. [Raphaël Vinot]
Other
~~~~~
- Merge pull request #126 from CenturyLinkCIRT/master. [Raphaël Vinot]
Added vt_to_misp.py example and VTReportObject
- Merge branch 'master' of https://github.com/MISP/PyMISP. [Thomas
Gardner]
- Fix test suite. [Raphaël Vinot]
- Merge branch 'master' of github.com:MISP/PyMISP. [Raphaël Vinot]
- Merge pull request #122 from LDO-CERT/master. [Raphaël Vinot]
Created add_generic_object.py
- Created add_generic_object.py. [garanews]
usage: add_generic_object.py [-h] -e EVENT -t TYPE -d DICT
Examples:
python3 add_generic_object.py -e 1683 -t email -d '{"subject":"The Pink Letter", "to":"jon@snow.org"}'
python3 add_generic_object.py -e 2343 -t person -d '{"first-name":"Daenerys", "last-name":"Targaryen", "place-of-birth":"Dragonstone"}'
python3 add_generic_object.py -e 3596 -t "domain|ip" -d '{"domain":"stormborn.org", "ip":"50.63.202.33"}'
- Added vtreportobject and vt_to_misp example. [Thomas Gardner]
- Created add_generic_object.py. [garanews]
usage: add_generic_object.py [-h] -e EVENT -t TYPE -d DICT
Examples:
python3 add_generic_object.py -e 1683 -t email -d '{"subject":"The Pink Letter", "to":"jon@snow.org"}'
python3 add_generic_object.py -e 2343 -t person -d '{"first-name":"Daenerys", "last-name":"Targaryen", "place-of-birth":"Dragonstone"}'
python3 add_generic_object.py -e 3596 -t "domain|ip" -d '{"domain":"stormborn.org", "ip":"50.63.202.33"}'
v2.4.81.2 (2017-10-24)
----------------------

14
README.md
View File

@ -24,7 +24,7 @@ pip3 install pymisp
## Install the latest version from repo
```
git clone https://github.com/CIRCL/PyMISP.git && cd PyMISP
git clone https://github.com/MISP/PyMISP.git && cd PyMISP
pip3 install -I .
```
@ -67,6 +67,16 @@ logger = logging.getLogger('pymisp')
logger.setLevel(logging.DEBUG)
```
Or if you want to write the debug output to a file instead of stderr:
```python
import pymisp
import logging
logger = logging.getLogger('pymisp')
logging.basicConfig(level=logging.DEBUG, filename="debug.log", filemode='w', format=pymisp.FORMAT)
```
## Documentation
[PyMISP API documentation is available](https://media.readthedocs.org/pdf/pymisp/master/pymisp.pdf).
@ -74,7 +84,7 @@ logger.setLevel(logging.DEBUG)
Documentation can be generated with epydoc:
```
epydoc --url https://github.com/CIRCL/PyMISP --graph all --name PyMISP --pdf pymisp -o doc
epydoc --url https://github.com/MISP/PyMISP --graph all --name PyMISP --pdf pymisp -o doc
```
## Everything is a Mutable Mapping

23
examples/add_generic_object.py Normal file → Executable file
View File

@ -1,24 +1,22 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import json
from pymisp import PyMISP
from pymisp.tools.abstractgenerator import AbstractMISPObjectGenerator
from pymisp.tools import GenericObjectGenerator
from keys import misp_url, misp_key, misp_verifycert
import argparse
class GenericObject(AbstractMISPObjectGenerator):
def __init__(self, type, data_dict):
super(GenericObject, self).__init__(type)
self.__data = data_dict
self.generate_attributes()
def generate_attributes(self):
for key, value in self.__data.items():
self.add_attribute(key, value=value)
"""
Sample usage:
./add_generic_object.py -e 5065 -t email -l '[{"to": "undisclosed@ppp.com"}, {"to": "second.to@mail.com"}]'
"""
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Create a MISP Object selectable by type starting from a dictionary')
parser.add_argument("-e", "--event", required=True, help="Event ID to update")
parser.add_argument("-t", "--type", required=True, help="Type of the generic object")
parser.add_argument("-d", "--dict", required=True, help="Dict ")
parser.add_argument("-l", "--attr_list", required=True, help="List of attributes")
args = parser.parse_args()
pymisp = PyMISP(misp_url, misp_key, misp_verifycert)
@ -29,5 +27,6 @@ if __name__ == '__main__':
print ("Template for type %s not found! Valid types are: %s" % (args.type, valid_types))
exit()
misp_object = GenericObject(args.type.replace("|", "-"), json.loads(args.dict))
misp_object = GenericObjectGenerator(args.type.replace("|", "-"))
misp_object.generate_attributes(json.loads(args.attr_list))
r = pymisp.add_object(args.event, template_id, misp_object)

169
examples/feed-generator/generate.py
View File

@ -4,28 +4,79 @@
import sys
import json
import os
import hashlib
from pymisp import PyMISP
from settings import url, key, ssl, outputdir, filters, valid_attribute_distribution_levels
objectsFields = {
'Attribute': {
'uuid',
'value',
'category',
'type',
'comment',
'data',
'timestamp',
'to_ids'
},
'Event': {
'uuid',
'info',
'threat_level_id',
'analysis',
'timestamp',
'publish_timestamp',
'published',
'date'
},
'Object': {
'name',
'meta-category',
'description',
'template_uuid',
'template_version',
'uuid',
'timestamp',
'distribution',
'sharing_group_id',
'comment'
},
'ObjectReference': {
'uuid',
'timestamp',
'relationship_type',
'comment',
'object_uuid',
'referenced_uuid'
},
'Orgc': {
'name',
'uuid'
},
'Tag': {
'name',
'colour',
'exportable'
}
}
objectsToSave = {'Orgc': {'fields': ['name', 'uuid'],
'multiple': False,
},
'Tag': {'fields': ['name', 'colour', 'exportable'],
'multiple': True,
},
'Attribute': {'fields': ['uuid', 'value', 'category', 'type',
'comment', 'data', 'timestamp', 'to_ids'],
'multiple': True,
},
}
fieldsToSave = ['uuid', 'info', 'threat_level_id', 'analysis',
'timestamp', 'publish_timestamp', 'published',
'date']
objectsToSave = {
'Orgc': {},
'Tag': {},
'Attribute': {
'Tag': {}
},
'Object': {
'Attribute': {
'Tag': {}
},
'ObjectReference': {}
}
}
valid_attribute_distributions = []
attributeHashes = []
def init():
# If we have an old settings.py file then this variable won't exist
@ -36,61 +87,65 @@ def init():
valid_attribute_distributions = ['0', '1', '2', '3', '4', '5']
return PyMISP(url, key, ssl)
def recursiveExtract(container, containerType, leaf, eventUuid):
temp = {}
if containerType in ['Attribute', 'Object']:
if (__blockByDistribution(container)):
return False
for field in objectsFields[containerType]:
if field in container:
temp[field] = container[field]
if (containerType == 'Attribute'):
global attributeHashes
if ('|' in container['type'] or container['type'] == 'malware-sample'):
split = container['value'].split('|')
attributeHashes.append([hashlib.md5(split[0].encode("utf-8")).hexdigest(), eventUuid])
attributeHashes.append([hashlib.md5(split[1].encode("utf-8")).hexdigest(), eventUuid])
else:
attributeHashes.append([hashlib.md5(container['value'].encode("utf-8")).hexdigest(), eventUuid])
children = leaf.keys()
for childType in children:
childContainer = container.get(childType)
if (childContainer):
if (type(childContainer) is dict):
temp[childType] = recursiveExtract(childContainer, childType, leaf[childType], eventUuid)
else:
temp[childType] = []
for element in childContainer:
processed = recursiveExtract(element, childType, leaf[childType], eventUuid)
if (processed):
temp[childType].append(processed)
return temp
def saveEvent(misp, uuid):
result = {}
event = misp.get_event(uuid)
if not event.get('Event'):
print('Error while fetching event: {}'.format(event['message']))
sys.exit('Could not create file for event ' + uuid + '.')
event = __cleanUpEvent(event)
event['Event'] = recursiveExtract(event['Event'], 'Event', objectsToSave, event['Event']['uuid'])
event = json.dumps(event)
eventFile = open(os.path.join(outputdir, uuid + '.json'), 'w')
eventFile.write(event)
eventFile.close()
def __cleanUpEvent(event):
temp = event
event = {'Event': {}}
__cleanupEventFields(event, temp)
__cleanupEventObjects(event, temp)
return event
def __cleanupEventFields(event, temp):
for field in fieldsToSave:
if field in temp['Event'].keys():
event['Event'][field] = temp['Event'][field]
return event
def __blockAttributeByDistribution(attribute):
if attribute['distribution'] not in valid_attribute_distributions:
def __blockByDistribution(element):
if element['distribution'] not in valid_attribute_distributions:
return True
return False
def saveHashes():
if not attributeHashes:
return False
try:
hashFile = open(os.path.join(outputdir, 'hashes.csv'), 'w')
for element in attributeHashes:
hashFile.write('{},{}\n'.format(element[0], element[1]))
hashFile.close()
except Exception as e:
print(e)
sys.exit('Could not create the quick hash lookup file.')
def __cleanupEventObjects(event, temp):
for objectType in objectsToSave.keys():
if objectsToSave[objectType]['multiple'] is True:
if objectType in temp['Event']:
for objectInstance in temp['Event'][objectType]:
if objectType is 'Attribute':
if __blockAttributeByDistribution(objectInstance):
continue
tempObject = {}
for field in objectsToSave[objectType]['fields']:
if field in objectInstance.keys():
tempObject[field] = objectInstance[field]
if objectType not in event['Event']:
event['Event'][objectType] = []
event['Event'][objectType].append(tempObject)
else:
tempObject = {}
for field in objectsToSave[objectType]['fields']:
tempObject[field] = temp['Event'][objectType][field]
event['Event'][objectType] = tempObject
return event
def saveManifest(manifest):
@ -138,4 +193,6 @@ if __name__ == '__main__':
print("Event " + str(counter) + "/" + str(total) + " exported.")
counter += 1
saveManifest(manifest)
print('Manifest saved. Feed creation completed.')
print('Manifest saved.')
saveHashes()
print('Hashes saved. Feed creation completed.')

2
pymisp/__init__.py
View File

@ -1,4 +1,4 @@
__version__ = '2.4.81.2'
__version__ = '2.4.82'
import sys
import logging
logger = logging.getLogger(__name__)

107
pymisp/api.py
View File

@ -17,7 +17,7 @@ import zipfile
from . import __version__
from .exceptions import PyMISPError, SearchError, NoURL, NoKey
from .mispevent import MISPEvent, MISPAttribute
from .mispevent import MISPEvent, MISPAttribute, MISPUser, MISPOrganisation
from .abstract import MISPEncode
logger = logging.getLogger('pymisp')
@ -68,11 +68,11 @@ class PyMISP(object):
:param url: URL of the MISP instance you want to connect to
:param key: API key of the user you want to use
:param ssl: can be True or False (to check ot not the validity
of the certificate. Or a CA_BUNDLE in case of self
signed certiifcate (the concatenation of all the
*.crt of the chain)
of the certificate. Or a CA_BUNDLE in case of self
signed certiifcate (the concatenation of all the
*.crt of the chain)
:param out_type: Type of object (json) NOTE: XML output isn't supported anymore, keeping the flag for compatibility reasons.
:param debug: deprecated, configure logging in api client instead
:param debug: Write all the debug information to stderr
:param proxies: Proxy dict as describes here: http://docs.python-requests.org/en/master/user/advanced/#proxies
:param cert: Client certificate, as described there: http://docs.python-requests.org/en/master/user/advanced/#ssl-cert-verification
:param asynch: Use asynchronous processing where possible
@ -385,6 +385,18 @@ class PyMISP(object):
eid = e.id
return self.update_event(eid, e)
def fast_publish(self, event_id, alert=False):
"""Does the same as the publish method, but just try to publish the event
even with one single HTTP GET.
The default is to not send a mail as it is assumed this method is called on update.
"""
if not alert:
url = urljoin(self.root_url, 'events/publish/{}'.format(event_id))
else:
url = urljoin(self.root_url, 'events/alert/{}'.format(event_id))
response = self.__prepare_request('POST', url)
return self._check_response(response)
def publish(self, event, alert=True):
"""Publish event (with or without alert email)
:param event: pass event or event id (as string or int) to publish
@ -398,12 +410,7 @@ class PyMISP(object):
event_id = full_event.id
if full_event.published:
return {'error': 'Already published'}
if not alert:
url = urljoin(self.root_url, 'events/publish/{}'.format(event_id))
else:
url = urljoin(self.root_url, 'events/alert/{}'.format(event_id))
response = self.__prepare_request('POST', url)
return self._check_response(response)
return self.fast_publish(event_id, alert)
def change_threat_level(self, event, threat_level_id):
"""Change the threat level of an event"""
@ -895,8 +902,8 @@ class PyMISP(object):
analysis=None, attribute=None, org=None, async_callback=None, normalize=False):
"""Search only at the index level. Use ! infront of value as NOT, default OR
If using async, give a callback that takes 2 args, session and response:
basic usage is
pymisp.search_index(..., async_callback=lambda ses,resp: print(resp.json()))
basic usage is
pymisp.search_index(..., async_callback=lambda ses,resp: print(resp.json()))
:param published: Published (0,1)
:param eventid: Evend ID(s) | str or list
@ -1251,46 +1258,6 @@ class PyMISP(object):
# ############## Users ##################
def _set_user_parameters(self, **kwargs):
user = {}
if kwargs.get('email'):
user['email'] = kwargs.get('email')
if kwargs.get('org_id'):
user['org_id'] = kwargs.get('org_id')
if kwargs.get('role_id'):
user['role_id'] = kwargs.get('role_id')
if kwargs.get('password'):
user['password'] = kwargs.get('password')
if kwargs.get('external_auth_required') is not None:
user['external_auth_required'] = kwargs.get('external_auth_required')
if kwargs.get('external_auth_key'):
user['external_auth_key'] = kwargs.get('external_auth_key')
if kwargs.get('enable_password') is not None:
user['enable_password'] = kwargs.get('enable_password')
if kwargs.get('nids_sid'):
user['nids_sid'] = kwargs.get('nids_sid')
if kwargs.get('server_id') is not None:
user['server_id'] = kwargs.get('server_id')
if kwargs.get('gpgkey'):
user['gpgkey'] = kwargs.get('gpgkey')
if kwargs.get('certif_public'):
user['certif_public'] = kwargs.get('certif_public')
if kwargs.get('autoalert') is not None:
user['autoalert'] = kwargs.get('autoalert')
if kwargs.get('contactalert') is not None:
user['contactalert'] = kwargs.get('contactalert')
if kwargs.get('disabled') is not None:
user['disabled'] = kwargs.get('disabled')
if kwargs.get('change_pw') is not None:
user['change_pw'] = kwargs.get('change_pw')
if kwargs.get('termsaccepted') is not None:
user['termsaccepted'] = kwargs.get('termsaccepted')
if kwargs.get('newsread') is not None:
user['newsread'] = kwargs.get('newsread')
if kwargs.get('authkey'):
user['authkey'] = kwargs.get('authkey')
return user
def get_users_list(self):
url = urljoin(self.root_url, 'admin/users')
response = self.__prepare_request('GET', url)
@ -1303,8 +1270,9 @@ class PyMISP(object):
def add_user(self, email, org_id, role_id, **kwargs):
url = urljoin(self.root_url, 'admin/users/add/')
new_user = self._set_user_parameters(**dict(email=email, org_id=org_id, role_id=role_id, **kwargs))
response = self.__prepare_request('POST', url, json.dumps(new_user))
new_user = MISPUser()
new_user.from_dict(email=email, org_id=org_id, role_id=role_id, **kwargs)
response = self.__prepare_request('POST', url, new_user.to_json())
return self._check_response(response)
def add_user_json(self, json_file):
@ -1320,7 +1288,8 @@ class PyMISP(object):
return self._check_response(response)
def edit_user(self, user_id, **kwargs):
edit_user = self._set_user_parameters(**kwargs)
edit_user = MISPUser()
edit_user.from_dict(**kwargs)
url = urljoin(self.root_url, 'admin/users/edit/{}'.format(user_id))
response = self.__prepare_request('POST', url, json.dumps(edit_user))
return self._check_response(response)
@ -1339,26 +1308,6 @@ class PyMISP(object):
# ############## Organisations ##################
def _set_organisation_parameters(self, **kwargs):
organisation = {}
if kwargs.get('name'):
organisation['name'] = kwargs.get('name')
if kwargs.get('description'):
organisation['description'] = kwargs.get('description')
if kwargs.get('type'):
organisation['type'] = kwargs.get('type')
if kwargs.get('nationality'):
organisation['nationality'] = kwargs.get('nationality')
if kwargs.get('sector'):
organisation['sector'] = kwargs.get('sector')
if kwargs.get('uuid'):
organisation['uuid'] = kwargs.get('uuid')
if kwargs.get('contacts'):
organisation['contacts'] = kwargs.get('contacts')
if kwargs.get('local') is not None:
organisation['local'] = kwargs.get('local')
return organisation
def get_organisations_list(self, scope="local"):
scope = scope.lower()
if scope not in ["local", "external", "all"]:
@ -1373,7 +1322,8 @@ class PyMISP(object):
return self._check_response(response)
def add_organisation(self, name, **kwargs):
new_org = self._set_organisation_parameters(**dict(name=name, **kwargs))
new_org = MISPOrganisation()
new_org.from_dict(name=name, **kwargs)
if 'local' in new_org:
if new_org.get('local') is False:
if 'uuid' not in new_org:
@ -1395,7 +1345,8 @@ class PyMISP(object):
return self._check_response(response)
def edit_organisation(self, org_id, **kwargs):
edit_org = self._set_organisation_parameters(**kwargs)
edit_org = MISPOrganisation()
edit_org.from_dict(**kwargs)
url = urljoin(self.root_url, 'admin/organisations/edit/{}'.format(org_id))
response = self.__prepare_request('POST', url, json.dumps(edit_org))
return self._check_response(response)

2
pymisp/data/misp-objects

@ -1 +1 @@
Subproject commit 6b43b68651a350a26891080ef0feda364b74727a
Subproject commit 66c4578f08efc6b92737f1667cbaf237149a8e46

21
pymisp/mispevent.py
View File

@ -595,6 +595,26 @@ class MISPObjectReference(AbstractMISP):
setattr(self, k, v)
class MISPUser(AbstractMISP):
def __init__(self):
super(MISPUser, self).__init__()
def from_dict(self, **kwargs):
for k, v in kwargs.items():
setattr(self, k, v)
class MISPOrganisation(AbstractMISP):
def __init__(self):
super(MISPOrganisation, self).__init__()
def from_dict(self, **kwargs):
for k, v in kwargs.items():
setattr(self, k, v)
class MISPObjectAttribute(MISPAttribute):
def __init__(self, definition):
@ -727,6 +747,7 @@ class MISPObject(AbstractMISP):
attribute = MISPObjectAttribute(self.__definition['attributes'][object_relation])
else:
# Woopsie, this object_relation is unknown, no sane defaults for you.
logger.warning("The template ({}) doesn't have the object_relation ({}) you're trying to add.".format(self.name, object_relation))
attribute = MISPObjectAttribute({})
else:
attribute = MISPObjectAttribute({})

1
pymisp/tools/__init__.py
View File

@ -6,3 +6,4 @@ from .elfobject import ELFObject, ELFSectionObject # noqa
from .machoobject import MachOObject, MachOSectionObject # noqa
from .create_misp_object import make_binary_objects # noqa
from .abstractgenerator import AbstractMISPObjectGenerator # noqa
from .genericgenerator import GenericObjectGenerator # noqa

16
pymisp/tools/genericgenerator.py Normal file
View File

@ -0,0 +1,16 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from .abstractgenerator import AbstractMISPObjectGenerator
class GenericObjectGenerator(AbstractMISPObjectGenerator):
def generate_attributes(self, attributes):
for attribute in attributes:
for object_relation, value in attribute.items():
if isinstance(value, dict):
self.add_attribute(object_relation, **value)
else:
# In this case, we need a valid template, as all the other parameters will be pre-set.
self.add_attribute(object_relation, value=value)