Merge remote-tracking branch 'upstream/master'

CHANGELOG.txt

@@ -2,6 +2,78 @@ Changelog
 =========
 
 
+v2.4.82 (2017-11-09)
+--------------------
+
+New
+~~~
+- Proper debug system. [Raphaël Vinot]
+
+  Make it easy to investigate the json blobs sent to the server.
+
+Changes
+~~~~~~~
+- Bump misp-objects. [Raphaël Vinot]
+- Update readme for new logging system. [Raphaël Vinot]
+- Small improvments in the logging system. [Raphaël Vinot]
+- Properly use python logging module. [Raphaël Vinot]
+- Update asciidoctor generator. [Raphaël Vinot]
+- Remove warning if PyMISP is too new. [Raphaël Vinot]
+- Add simple asciidoc generator for MISP event. [Raphaël Vinot]
+- Update changelog. [Raphaël Vinot]
+
+Fix
+~~~
+- Typo loger -> logger. [Raphaël Vinot]
+- Let load unknown object relations in known templates. [Raphaël Vinot]
+
+  This isn't recommended, but happens very often.
+- Allow to load non-malware ZIP files in MISP Event. [Raphaël Vinot]
+
+  Prior to his patch, any zip file loaded by MISP Event was unpacked and
+  processed as an excrypted malware from MISP.
+- Properly pass the distribution when uploading a sample. [Raphaël
+  Vinot]
+- Properly upload a sample in an existing event. [Raphaël Vinot]
+
+  Fix https://github.com/MISP/PyMISP/issues/123
+- Properly set the distribution at event level. [Raphaël Vinot]
+
+  fix #120
+- Properly pop the distribution key. [Raphaël Vinot]
+- Update dependencies for VT generator. [Raphaël Vinot]
+
+Other
+~~~~~
+- Merge pull request #126 from CenturyLinkCIRT/master. [Raphaël Vinot]
+
+  Added vt_to_misp.py example and VTReportObject
+- Merge branch 'master' of https://github.com/MISP/PyMISP. [Thomas
+  Gardner]
+- Fix test suite. [Raphaël Vinot]
+- Merge branch 'master' of github.com:MISP/PyMISP. [Raphaël Vinot]
+- Merge pull request #122 from LDO-CERT/master. [Raphaël Vinot]
+
+  Created add_generic_object.py
+- Created add_generic_object.py. [garanews]
+
+  usage: add_generic_object.py [-h] -e EVENT -t TYPE -d DICT
+
+  Examples:
+  python3 add_generic_object.py -e 1683 -t email -d '{"subject":"The Pink Letter", "to":"jon@snow.org"}'
+  python3 add_generic_object.py -e 2343 -t person -d '{"first-name":"Daenerys", "last-name":"Targaryen", "place-of-birth":"Dragonstone"}'
+  python3 add_generic_object.py -e 3596 -t "domain|ip" -d '{"domain":"stormborn.org", "ip":"50.63.202.33"}'
+- Added vtreportobject and vt_to_misp example. [Thomas Gardner]
+- Created add_generic_object.py. [garanews]
+
+  usage: add_generic_object.py [-h] -e EVENT -t TYPE -d DICT
+
+  Examples:
+  python3 add_generic_object.py -e 1683 -t email -d '{"subject":"The Pink Letter", "to":"jon@snow.org"}'
+  python3 add_generic_object.py -e 2343 -t person -d '{"first-name":"Daenerys", "last-name":"Targaryen", "place-of-birth":"Dragonstone"}'
+  python3 add_generic_object.py -e 3596 -t "domain|ip" -d '{"domain":"stormborn.org", "ip":"50.63.202.33"}'
+
+
 v2.4.81.2 (2017-10-24)
 ----------------------
 

README.md

@@ -24,7 +24,7 @@ pip3 install pymisp
 ## Install the latest version from repo
 
 ```
-git clone https://github.com/CIRCL/PyMISP.git && cd PyMISP
+git clone https://github.com/MISP/PyMISP.git && cd PyMISP
 pip3 install -I .
 ```
 
@@ -67,6 +67,16 @@ logger = logging.getLogger('pymisp')
 logger.setLevel(logging.DEBUG)
 ```
 
+Or if you want to write the debug output to a file instead of stderr:
+
+```python
+import pymisp
+import logging
+
+logger = logging.getLogger('pymisp')
+logging.basicConfig(level=logging.DEBUG, filename="debug.log", filemode='w', format=pymisp.FORMAT)
+```
+
 ## Documentation
 
 [PyMISP API documentation is available](https://media.readthedocs.org/pdf/pymisp/master/pymisp.pdf).
@@ -74,7 +84,7 @@ logger.setLevel(logging.DEBUG)
 Documentation can be generated with epydoc:
 
 ```
-epydoc --url https://github.com/CIRCL/PyMISP --graph all --name PyMISP --pdf pymisp -o doc
+epydoc --url https://github.com/MISP/PyMISP --graph all --name PyMISP --pdf pymisp -o doc
 ```
 
 ## Everything is a Mutable Mapping

examples/add_generic_object.py

@@ -1,24 +1,22 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
 import json
 from pymisp import PyMISP
-from pymisp.tools.abstractgenerator import AbstractMISPObjectGenerator
+from pymisp.tools import GenericObjectGenerator
 from keys import misp_url, misp_key, misp_verifycert
 import argparse
 
-class GenericObject(AbstractMISPObjectGenerator):
+"""
-    def __init__(self, type, data_dict):
+Sample usage:
-        super(GenericObject, self).__init__(type)
+./add_generic_object.py -e 5065 -t email -l '[{"to": "undisclosed@ppp.com"}, {"to": "second.to@mail.com"}]'
-        self.__data = data_dict
+"""
-        self.generate_attributes()
-
-    def generate_attributes(self):
-        for key, value in self.__data.items():
-            self.add_attribute(key, value=value)
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='Create a MISP Object selectable by type starting from a dictionary')
     parser.add_argument("-e", "--event", required=True, help="Event ID to update")
     parser.add_argument("-t", "--type", required=True, help="Type of the generic object")
-    parser.add_argument("-d", "--dict", required=True, help="Dict ")
+    parser.add_argument("-l", "--attr_list", required=True, help="List of attributes")
     args = parser.parse_args()
 
     pymisp = PyMISP(misp_url, misp_key, misp_verifycert)
@@ -29,5 +27,6 @@ if __name__ == '__main__':
         print ("Template for type %s not found! Valid types are: %s" % (args.type, valid_types))
         exit()
 
-    misp_object = GenericObject(args.type.replace("|", "-"), json.loads(args.dict))
+    misp_object = GenericObjectGenerator(args.type.replace("|", "-"))
+    misp_object.generate_attributes(json.loads(args.attr_list))
     r = pymisp.add_object(args.event, template_id, misp_object)

examples/feed-generator/generate.py

@@ -4,28 +4,79 @@
 import sys
 import json
 import os
+import hashlib
 from pymisp import PyMISP
 from settings import url, key, ssl, outputdir, filters, valid_attribute_distribution_levels
 
+objectsFields = {
+    'Attribute': {
+        'uuid',
+        'value',
+        'category',
+        'type',
+        'comment',
+        'data',
+        'timestamp',
+        'to_ids'
+    },
+    'Event': {
+        'uuid',
+        'info',
+        'threat_level_id',
+        'analysis',
+        'timestamp',
+        'publish_timestamp',
+        'published',
+        'date'
+    },
+    'Object': {
+        'name',
+        'meta-category',
+        'description',
+        'template_uuid',
+        'template_version',
+        'uuid',
+        'timestamp',
+        'distribution',
+        'sharing_group_id',
+        'comment'
+    },
+    'ObjectReference': {
+        'uuid',
+        'timestamp',
+        'relationship_type',
+        'comment',
+        'object_uuid',
+        'referenced_uuid'
+    },
+    'Orgc': {
+        'name',
+        'uuid'
+    },
+    'Tag': {
+        'name',
+        'colour',
+        'exportable'
+    }
+}
 
-objectsToSave = {'Orgc': {'fields': ['name', 'uuid'],
+objectsToSave = {
-                          'multiple': False,
+    'Orgc': {},
-                          },
+    'Tag': {},
-                 'Tag': {'fields': ['name', 'colour', 'exportable'],
+    'Attribute': {
-                         'multiple': True,
+        'Tag': {}
-                         },
+    },
-                 'Attribute': {'fields': ['uuid', 'value', 'category', 'type',
+    'Object': {
-                                          'comment', 'data', 'timestamp', 'to_ids'],
+        'Attribute': {
-                               'multiple': True,
+            'Tag': {}
-                               },
+        },
-                 }
+        'ObjectReference': {}
-
+    }
-fieldsToSave = ['uuid', 'info', 'threat_level_id', 'analysis',
+}
-                'timestamp', 'publish_timestamp', 'published',
-                'date']
 
 valid_attribute_distributions = []
 
+attributeHashes = []
 
 def init():
     # If we have an old settings.py file then this variable won't exist
@@ -36,61 +87,65 @@ def init():
         valid_attribute_distributions = ['0', '1', '2', '3', '4', '5']
     return PyMISP(url, key, ssl)
 
+def recursiveExtract(container, containerType, leaf, eventUuid):
+    temp = {}
+    if containerType in ['Attribute', 'Object']:
+        if (__blockByDistribution(container)):
+            return False
+    for field in objectsFields[containerType]:
+        if field in container:
+            temp[field] = container[field]
+    if (containerType == 'Attribute'):
+        global attributeHashes
+        if ('|' in container['type'] or container['type'] == 'malware-sample'):
+            split = container['value'].split('|')
+            attributeHashes.append([hashlib.md5(split[0].encode("utf-8")).hexdigest(), eventUuid])
+            attributeHashes.append([hashlib.md5(split[1].encode("utf-8")).hexdigest(), eventUuid])
+        else:
+            attributeHashes.append([hashlib.md5(container['value'].encode("utf-8")).hexdigest(), eventUuid])
+    children = leaf.keys()
+    for childType in children:
+        childContainer = container.get(childType)
+        if (childContainer):
+            if (type(childContainer) is dict):
+                temp[childType] = recursiveExtract(childContainer, childType, leaf[childType], eventUuid)
+            else:
+                temp[childType] = []
+                for element in childContainer:
+                    processed = recursiveExtract(element, childType, leaf[childType], eventUuid)
+                    if (processed):
+                        temp[childType].append(processed)
+    return temp
 
 def saveEvent(misp, uuid):
+    result = {}
     event = misp.get_event(uuid)
     if not event.get('Event'):
         print('Error while fetching event: {}'.format(event['message']))
         sys.exit('Could not create file for event ' + uuid + '.')
-    event = __cleanUpEvent(event)
+    event['Event'] = recursiveExtract(event['Event'], 'Event', objectsToSave, event['Event']['uuid'])
     event = json.dumps(event)
     eventFile = open(os.path.join(outputdir, uuid + '.json'), 'w')
     eventFile.write(event)
     eventFile.close()
 
-
+def __blockByDistribution(element):
-def __cleanUpEvent(event):
+    if element['distribution'] not in valid_attribute_distributions:
-        temp = event
-        event = {'Event': {}}
-        __cleanupEventFields(event, temp)
-        __cleanupEventObjects(event, temp)
-        return event
-
-
-def __cleanupEventFields(event, temp):
-    for field in fieldsToSave:
-        if field in temp['Event'].keys():
-            event['Event'][field] = temp['Event'][field]
-    return event
-
-
-def __blockAttributeByDistribution(attribute):
-    if attribute['distribution'] not in valid_attribute_distributions:
         return True
     return False
 
+def saveHashes():
+    if not attributeHashes:
+        return False
+    try:
+        hashFile = open(os.path.join(outputdir, 'hashes.csv'), 'w')
+        for element in attributeHashes:
+            hashFile.write('{},{}\n'.format(element[0], element[1]))
+        hashFile.close()
+    except Exception as e:
+        print(e)
+        sys.exit('Could not create the quick hash lookup file.')
 
-def __cleanupEventObjects(event, temp):
-    for objectType in objectsToSave.keys():
-        if objectsToSave[objectType]['multiple'] is True:
-            if objectType in temp['Event']:
-                for objectInstance in temp['Event'][objectType]:
-                    if objectType is 'Attribute':
-                        if __blockAttributeByDistribution(objectInstance):
-                            continue
-                    tempObject = {}
-                    for field in objectsToSave[objectType]['fields']:
-                        if field in objectInstance.keys():
-                            tempObject[field] = objectInstance[field]
-                    if objectType not in event['Event']:
-                        event['Event'][objectType] = []
-                    event['Event'][objectType].append(tempObject)
-        else:
-            tempObject = {}
-            for field in objectsToSave[objectType]['fields']:
-                tempObject[field] = temp['Event'][objectType][field]
-            event['Event'][objectType] = tempObject
-    return event
 
 
 def saveManifest(manifest):
@@ -138,4 +193,6 @@ if __name__ == '__main__':
         print("Event " + str(counter) + "/" + str(total) + " exported.")
         counter += 1
     saveManifest(manifest)
-    print('Manifest saved. Feed creation completed.')
+    print('Manifest saved.')
+    saveHashes()
+    print('Hashes saved. Feed creation completed.')

pymisp/__init__.py

@@ -1,4 +1,4 @@
-__version__ = '2.4.81.2'
+__version__ = '2.4.82'
 import sys
 import logging
 logger = logging.getLogger(__name__)

pymisp/api.py

@@ -17,7 +17,7 @@ import zipfile
 
 from . import __version__
 from .exceptions import PyMISPError, SearchError, NoURL, NoKey
-from .mispevent import MISPEvent, MISPAttribute
+from .mispevent import MISPEvent, MISPAttribute, MISPUser, MISPOrganisation
 from .abstract import MISPEncode
 
 logger = logging.getLogger('pymisp')
@@ -68,11 +68,11 @@ class PyMISP(object):
     :param url: URL of the MISP instance you want to connect to
     :param key: API key of the user you want to use
     :param ssl: can be True or False (to check ot not the validity
-                of the certificate. Or a CA_BUNDLE in case of self
+    of the certificate. Or a CA_BUNDLE in case of self
-                signed certiifcate (the concatenation of all the
+    signed certiifcate (the concatenation of all the
-                *.crt of the chain)
+    *.crt of the chain)
     :param out_type: Type of object (json) NOTE: XML output isn't supported anymore, keeping the flag for compatibility reasons.
-    :param debug: deprecated, configure logging in api client instead
+    :param debug: Write all the debug information to stderr
     :param proxies: Proxy dict as describes here: http://docs.python-requests.org/en/master/user/advanced/#proxies
     :param cert: Client certificate, as described there: http://docs.python-requests.org/en/master/user/advanced/#ssl-cert-verification
     :param asynch: Use asynchronous processing where possible
@@ -385,6 +385,18 @@ class PyMISP(object):
             eid = e.id
         return self.update_event(eid, e)
 
+    def fast_publish(self, event_id, alert=False):
+        """Does the same as the publish method, but just try to publish the event
+        even with one single HTTP GET.
+        The default is to not send a mail as it is assumed this method is called on update.
+        """
+        if not alert:
+            url = urljoin(self.root_url, 'events/publish/{}'.format(event_id))
+        else:
+            url = urljoin(self.root_url, 'events/alert/{}'.format(event_id))
+        response = self.__prepare_request('POST', url)
+        return self._check_response(response)
+
     def publish(self, event, alert=True):
         """Publish event (with or without alert email)
         :param event: pass event or event id (as string or int) to publish
@@ -398,12 +410,7 @@ class PyMISP(object):
         event_id = full_event.id
         if full_event.published:
             return {'error': 'Already published'}
-        if not alert:
+        return self.fast_publish(event_id, alert)
-            url = urljoin(self.root_url, 'events/publish/{}'.format(event_id))
-        else:
-            url = urljoin(self.root_url, 'events/alert/{}'.format(event_id))
-        response = self.__prepare_request('POST', url)
-        return self._check_response(response)
 
     def change_threat_level(self, event, threat_level_id):
         """Change the threat level of an event"""
@@ -895,8 +902,8 @@ class PyMISP(object):
                      analysis=None, attribute=None, org=None, async_callback=None, normalize=False):
         """Search only at the index level. Use ! infront of value as NOT, default OR
         If using async, give a callback that takes 2 args, session and response:
-            basic usage is
+        basic usage is
-            pymisp.search_index(..., async_callback=lambda ses,resp: print(resp.json()))
+        pymisp.search_index(..., async_callback=lambda ses,resp: print(resp.json()))
 
         :param published: Published (0,1)
         :param eventid: Evend ID(s) | str or list
@@ -1251,46 +1258,6 @@ class PyMISP(object):
 
     # ############## Users ##################
 
-    def _set_user_parameters(self, **kwargs):
-        user = {}
-        if kwargs.get('email'):
-            user['email'] = kwargs.get('email')
-        if kwargs.get('org_id'):
-            user['org_id'] = kwargs.get('org_id')
-        if kwargs.get('role_id'):
-            user['role_id'] = kwargs.get('role_id')
-        if kwargs.get('password'):
-            user['password'] = kwargs.get('password')
-        if kwargs.get('external_auth_required') is not None:
-            user['external_auth_required'] = kwargs.get('external_auth_required')
-        if kwargs.get('external_auth_key'):
-            user['external_auth_key'] = kwargs.get('external_auth_key')
-        if kwargs.get('enable_password') is not None:
-            user['enable_password'] = kwargs.get('enable_password')
-        if kwargs.get('nids_sid'):
-            user['nids_sid'] = kwargs.get('nids_sid')
-        if kwargs.get('server_id') is not None:
-            user['server_id'] = kwargs.get('server_id')
-        if kwargs.get('gpgkey'):
-            user['gpgkey'] = kwargs.get('gpgkey')
-        if kwargs.get('certif_public'):
-            user['certif_public'] = kwargs.get('certif_public')
-        if kwargs.get('autoalert') is not None:
-            user['autoalert'] = kwargs.get('autoalert')
-        if kwargs.get('contactalert') is not None:
-            user['contactalert'] = kwargs.get('contactalert')
-        if kwargs.get('disabled') is not None:
-            user['disabled'] = kwargs.get('disabled')
-        if kwargs.get('change_pw') is not None:
-            user['change_pw'] = kwargs.get('change_pw')
-        if kwargs.get('termsaccepted') is not None:
-            user['termsaccepted'] = kwargs.get('termsaccepted')
-        if kwargs.get('newsread') is not None:
-            user['newsread'] = kwargs.get('newsread')
-        if kwargs.get('authkey'):
-            user['authkey'] = kwargs.get('authkey')
-        return user
-
     def get_users_list(self):
         url = urljoin(self.root_url, 'admin/users')
         response = self.__prepare_request('GET', url)
@@ -1303,8 +1270,9 @@ class PyMISP(object):
 
     def add_user(self, email, org_id, role_id, **kwargs):
         url = urljoin(self.root_url, 'admin/users/add/')
-        new_user = self._set_user_parameters(**dict(email=email, org_id=org_id, role_id=role_id, **kwargs))
+        new_user = MISPUser()
-        response = self.__prepare_request('POST', url, json.dumps(new_user))
+        new_user.from_dict(email=email, org_id=org_id, role_id=role_id, **kwargs)
+        response = self.__prepare_request('POST', url, new_user.to_json())
         return self._check_response(response)
 
     def add_user_json(self, json_file):
@@ -1320,7 +1288,8 @@ class PyMISP(object):
         return self._check_response(response)
 
     def edit_user(self, user_id, **kwargs):
-        edit_user = self._set_user_parameters(**kwargs)
+        edit_user = MISPUser()
+        edit_user.from_dict(**kwargs)
         url = urljoin(self.root_url, 'admin/users/edit/{}'.format(user_id))
         response = self.__prepare_request('POST', url, json.dumps(edit_user))
         return self._check_response(response)
@@ -1339,26 +1308,6 @@ class PyMISP(object):
 
     # ############## Organisations ##################
 
-    def _set_organisation_parameters(self, **kwargs):
-        organisation = {}
-        if kwargs.get('name'):
-            organisation['name'] = kwargs.get('name')
-        if kwargs.get('description'):
-            organisation['description'] = kwargs.get('description')
-        if kwargs.get('type'):
-            organisation['type'] = kwargs.get('type')
-        if kwargs.get('nationality'):
-            organisation['nationality'] = kwargs.get('nationality')
-        if kwargs.get('sector'):
-            organisation['sector'] = kwargs.get('sector')
-        if kwargs.get('uuid'):
-            organisation['uuid'] = kwargs.get('uuid')
-        if kwargs.get('contacts'):
-            organisation['contacts'] = kwargs.get('contacts')
-        if kwargs.get('local') is not None:
-            organisation['local'] = kwargs.get('local')
-        return organisation
-
     def get_organisations_list(self, scope="local"):
         scope = scope.lower()
         if scope not in ["local", "external", "all"]:
@@ -1373,7 +1322,8 @@ class PyMISP(object):
         return self._check_response(response)
 
     def add_organisation(self, name, **kwargs):
-        new_org = self._set_organisation_parameters(**dict(name=name, **kwargs))
+        new_org = MISPOrganisation()
+        new_org.from_dict(name=name, **kwargs)
         if 'local' in new_org:
             if new_org.get('local') is False:
                 if 'uuid' not in new_org:
@@ -1395,7 +1345,8 @@ class PyMISP(object):
         return self._check_response(response)
 
     def edit_organisation(self, org_id, **kwargs):
-        edit_org = self._set_organisation_parameters(**kwargs)
+        edit_org = MISPOrganisation()
+        edit_org.from_dict(**kwargs)
         url = urljoin(self.root_url, 'admin/organisations/edit/{}'.format(org_id))
         response = self.__prepare_request('POST', url, json.dumps(edit_org))
         return self._check_response(response)

pymisp/data/misp-objects

@@ -1 +1 @@
-Subproject commit 6b43b68651a350a26891080ef0feda364b74727a
+Subproject commit 66c4578f08efc6b92737f1667cbaf237149a8e46

pymisp/mispevent.py

@@ -595,6 +595,26 @@ class MISPObjectReference(AbstractMISP):
             setattr(self, k, v)
 
 
+class MISPUser(AbstractMISP):
+
+    def __init__(self):
+        super(MISPUser, self).__init__()
+
+    def from_dict(self, **kwargs):
+        for k, v in kwargs.items():
+            setattr(self, k, v)
+
+
+class MISPOrganisation(AbstractMISP):
+
+    def __init__(self):
+        super(MISPOrganisation, self).__init__()
+
+    def from_dict(self, **kwargs):
+        for k, v in kwargs.items():
+            setattr(self, k, v)
+
+
 class MISPObjectAttribute(MISPAttribute):
 
     def __init__(self, definition):
@@ -727,6 +747,7 @@ class MISPObject(AbstractMISP):
                 attribute = MISPObjectAttribute(self.__definition['attributes'][object_relation])
             else:
                 # Woopsie, this object_relation is unknown, no sane defaults for you.
+                logger.warning("The template ({}) doesn't have the object_relation ({}) you're trying to add.".format(self.name, object_relation))
                 attribute = MISPObjectAttribute({})
         else:
             attribute = MISPObjectAttribute({})

pymisp/tools/__init__.py

@@ -6,3 +6,4 @@ from .elfobject import ELFObject, ELFSectionObject  # noqa
 from .machoobject import MachOObject, MachOSectionObject  # noqa
 from .create_misp_object import make_binary_objects  # noqa
 from .abstractgenerator import AbstractMISPObjectGenerator  # noqa
+from .genericgenerator import GenericObjectGenerator  # noqa

pymisp/tools/genericgenerator.py

@@ -0,0 +1,16 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from .abstractgenerator import AbstractMISPObjectGenerator
+
+
+class GenericObjectGenerator(AbstractMISPObjectGenerator):
+
+    def generate_attributes(self, attributes):
+        for attribute in attributes:
+            for object_relation, value in attribute.items():
+                if isinstance(value, dict):
+                    self.add_attribute(object_relation, **value)
+                else:
+                    # In this case, we need a valid template, as all the other parameters will be pre-set.
+                    self.add_attribute(object_relation, value=value)