diff --git a/examples/add_file_object.py b/examples/add_file_object.py index 7780ab9..e61d809 100755 --- a/examples/add_file_object.py +++ b/examples/add_file_object.py @@ -7,7 +7,6 @@ import traceback from keys import misp_url, misp_key, misp_verifycert import glob import argparse -import json if __name__ == '__main__': parser = argparse.ArgumentParser(description='Extract indicators out of binaries and add MISP objects to a MISP instance.') @@ -25,20 +24,17 @@ if __name__ == '__main__': if seos: for s in seos: - obj, refs = s - template_id = pymisp.get_object_template_id(obj['name']) - r = pymisp.add_object(args.event, template_id, obj) + template_id = pymisp.get_object_template_id(s['name']) + r = pymisp.add_object(args.event, template_id, s) if peo: - obj, refs = peo - template_id = pymisp.get_object_template_id(obj['name']) - r = pymisp.add_object(args.event, template_id, obj) - for ref in refs: - r = pymisp.add_object_reference(obj['uuid'], ref) + template_id = pymisp.get_object_template_id(peo['name']) + r = pymisp.add_object(args.event, template_id, peo) + for ref in peo.references: + r = pymisp.add_object_reference(ref) if fo: - obj, refs = fo - template_id = pymisp.get_object_template_id(obj['name']) - response = pymisp.add_object(args.event, template_id, obj) - for ref in refs: - r = pymisp.add_object_reference(obj['uuid'], ref) + template_id = pymisp.get_object_template_id(fo['name']) + response = pymisp.add_object(args.event, template_id, fo) + for ref in fo.references: + r = pymisp.add_object_reference(ref) diff --git a/examples/generate_file_objects.py b/examples/generate_file_objects.py new file mode 100755 index 0000000..5b9ad33 --- /dev/null +++ b/examples/generate_file_objects.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +from pymisp import MISPEncode +from pymisp.tools import make_binary_objects +import argparse +import json + + +def check(): + missing_dependencies = {'pydeep': False, 'lief': False, 'magic': False, 'pymisp': False} + try: + import pymisp # noqa + except ImportError: + missing_dependencies['pymisp'] = 'Please install pydeep: pip install pymisp' + try: + import pydeep # noqa + except ImportError: + missing_dependencies['pydeep'] = 'Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git' + try: + import lief # noqa + except ImportError: + missing_dependencies['lief'] = 'Please install lief, documentation here: https://github.com/lief-project/LIEF' + try: + import magic # noqa + except ImportError: + missing_dependencies['magic'] = 'Please install python-magic: pip install python-magic.' + return json.dumps(missing_dependencies) + + +def make_objects(path): + to_return = {'objects': [], 'references': []} + fo, peo, seos = make_binary_objects(path) + + if seos: + for s in seos: + to_return['objects'].append(s) + if s.references: + to_return['references'] += s.references + + if peo: + to_return['objects'].append(peo) + if peo.references: + to_return['references'] += peo.references + + if fo: + to_return['objects'].append(fo) + if fo.references: + to_return['references'] += fo.references + return json.dumps(to_return, cls=MISPEncode) + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Extract indicators out of binaries and returns MISP objects.') + group = parser.add_mutually_exclusive_group() + group.add_argument("-p", "--path", help="Path to process.") + group.add_argument("-c", "--check", action='store_true', help="Check the dependencies.") + args = parser.parse_args() + + if args.check: + print(check()) + if args.path: + obj = make_objects(args.path) + print(obj) diff --git a/pymisp/__init__.py b/pymisp/__init__.py index 27967ec..9d3e949 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -2,7 +2,7 @@ __version__ = '2.4.77' from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey from .api import PyMISP -from .abstract import AbstractMISP +from .abstract import AbstractMISP, MISPEncode from .mispevent import MISPEvent, MISPAttribute, EncodeUpdate, EncodeFull from .tools import Neo4j from .tools import stix diff --git a/pymisp/abstract.py b/pymisp/abstract.py index ceea59b..e2bf76e 100644 --- a/pymisp/abstract.py +++ b/pymisp/abstract.py @@ -4,9 +4,18 @@ import six # Remove that import when discarding python2 support. import abc import json +from json import JSONEncoder import collections +class MISPEncode(JSONEncoder): + + def default(self, obj): + if isinstance(obj, AbstractMISP): + return obj.jsonable() + return JSONEncoder.default(self, obj) + + @six.add_metaclass(abc.ABCMeta) # Remove that line when discarding python2 support. # Python3 way: class MISPObjectGenerator(metaclass=abc.ABCMeta): class AbstractMISP(collections.MutableMapping): @@ -48,8 +57,11 @@ class AbstractMISP(collections.MutableMapping): to_return[attribute] = val return to_return + def jsonable(self): + return self.to_dict() + def to_json(self): - return json.dumps(self.to_dict(), indent=4, sort_keys=True) + return json.dumps(self.to_dict(), cls=MISPEncode) def __getitem__(self, key): if self.__check_dict_key(key): diff --git a/pymisp/api.py b/pymisp/api.py index 8f898d5..4638726 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -1585,13 +1585,13 @@ class PyMISP(object): def add_object(self, event_id, template_id, misp_object): session = self.__prepare_session() url = urljoin(self.root_url, 'objects/add/{}/{}'.format(event_id, template_id)) - response = session.post(url, data=json.dumps(misp_object)) + response = session.post(url, data=misp_object.to_json()) return self._check_response(response) - def add_object_reference(self, parent_uuid, misp_object_reference): + def add_object_reference(self, misp_object_reference): session = self.__prepare_session() - url = urljoin(self.root_url, 'object_references/add/{}'.format(parent_uuid)) - response = session.post(url, data=json.dumps(misp_object_reference)) + url = urljoin(self.root_url, 'object_references/add') + response = session.post(url, data=misp_object_reference.to_json()) return self._check_response(response) def get_object_templates_list(self): diff --git a/pymisp/tools/create_misp_object.py b/pymisp/tools/create_misp_object.py index 4d4df6e..a69bfdb 100644 --- a/pymisp/tools/create_misp_object.py +++ b/pymisp/tools/create_misp_object.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python # -*- coding: utf-8 -*- from pymisp.tools import FileObject, PEObject, MISPObjectException @@ -17,11 +17,11 @@ class FileTypeNotImplemented(MISPObjectException): def make_pe_objects(lief_parsed, misp_file): misp_pe = PEObject(parsed=lief_parsed) misp_file.add_reference(misp_pe.uuid, 'included-in', 'PE indicators') - file_object = misp_file.dump() - pe_object = misp_pe.dump() + file_object = misp_file + pe_object = misp_pe pe_sections = [] for s in misp_pe.sections: - pe_sections.append(s.dump()) + pe_sections.append(s) return file_object, pe_object, pe_sections @@ -45,5 +45,5 @@ def make_binary_objects(filepath): print('\tParser error: ', e) except FileTypeNotImplemented as e: print(e) - file_object = misp_file.dump() + file_object = misp_file.to_json() return file_object, None, None diff --git a/pymisp/tools/fileobject.py b/pymisp/tools/fileobject.py index f21cd2f..770307c 100644 --- a/pymisp/tools/fileobject.py +++ b/pymisp/tools/fileobject.py @@ -7,6 +7,7 @@ from io import BytesIO from hashlib import md5, sha1, sha256, sha512 import math from collections import Counter +import warnings try: import pydeep @@ -25,9 +26,9 @@ class FileObject(MISPObjectGenerator): def __init__(self, filepath=None, pseudofile=None, filename=None): if not HAS_PYDEEP: - raise ImportError("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git") + warnings.warn("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git") if not HAS_MAGIC: - raise ImportError("Please install python-magic: pip install python-magic.") + warnings.warn("Please install python-magic: pip install python-magic.") if filepath: self.filepath = filepath self.filename = os.path.basename(self.filepath) @@ -46,16 +47,18 @@ class FileObject(MISPObjectGenerator): def generate_attributes(self): self._create_attribute('filename', value=self.filename) - self._create_attribute('size-in-bytes', value=len(self.data)) - if getattr(self, 'size-in-bytes').value > 0: + size = self._create_attribute('size-in-bytes', value=len(self.data)) + if int(size.value) > 0: self._create_attribute('entropy', value=self.__entropy_H(self.data)) self._create_attribute('md5', value=md5(self.data).hexdigest()) self._create_attribute('sha1', value=sha1(self.data).hexdigest()) self._create_attribute('sha256', value=sha256(self.data).hexdigest()) self._create_attribute('sha512', value=sha512(self.data).hexdigest()) - self._create_attribute('mimetype', value=magic.from_buffer(self.data)) - self._create_attribute('ssdeep', value=pydeep.hash_buf(self.data).decode()) - self._create_attribute('malware-sample', value=self.filename.value, data=self.pseudofile) + self._create_attribute('malware-sample', value=self.filename, data=self.pseudofile) + if HAS_MAGIC: + self._create_attribute('mimetype', value=magic.from_buffer(self.data)) + if HAS_PYDEEP: + self._create_attribute('ssdeep', value=pydeep.hash_buf(self.data).decode()) def __entropy_H(self, data): """Calculate the entropy of a chunk of data.""" diff --git a/pymisp/tools/objectgenerator.py b/pymisp/tools/objectgenerator.py index 72b5ba0..1617580 100644 --- a/pymisp/tools/objectgenerator.py +++ b/pymisp/tools/objectgenerator.py @@ -26,16 +26,53 @@ if six.PY2: class MISPObjectReference(AbstractMISP): - attributes = ['uuid', 'relationship_type', 'comment'] + attributes = ['source_uuid', 'destination_uuid', 'relationship_type', 'comment'] - def __init__(self, uuid, relationship_type, comment=None): - self['uuid'] = uuid - self['relationship_type'] = relationship_type - self['comment'] = comment + def __init__(self, source_uuid, destination_uuid, relationship_type, comment=None): + self.source_uuid = source_uuid + self.destination_uuid = destination_uuid + self.relationship_type = relationship_type + self.comment = comment + + +class MISPObjectAttribute(AbstractMISP): + + # This list is very limited and hardcoded to fit the current needs (file/pe/pesection creation): MISPAttriute will follow the + # same spec and just add one attribute: object_relation + attributes = ['object_relation', 'value', 'type', 'category', 'disable_correlation', 'to_ids', + 'data', 'encrypt', 'distribution', 'comment'] + + def __init__(self, definition, object_relation, value, **kwargs): + self.object_relation = object_relation + self.value = value + # Initialize the new MISPAttribute + # Get the misp attribute type from the definition + self.type = kwargs.pop('type', None) + if self.type is None: + self.type = definition['misp-attribute'] + self.disable_correlation = kwargs.pop('disable_correlation', None) + if self.disable_correlation is None: + # The correlation can be disabled by default in the object definition. + # Use this value if it isn't overloaded by the object + self.disable_correlation = definition.get('disable_correlation') + self.to_ids = kwargs.pop('to_ids', None) + if self.to_ids is None: + # Same for the to_ids flag + self.to_ids = definition.get('to_ids') + # Initialise rest of the values + for k, v in kwargs.items(): + self[k] = v + # FIXME: dirty hack until all the classes are ported to the new format but we get the default values + temp_attribute = MISPAttribute() + temp_attribute.set_all_values(**self) + # Update default values + self.from_dict(**temp_attribute.to_dict()) class MISPObjectGenerator(AbstractMISP): + attributes = ['name', 'meta-category', 'uuid', 'description', 'version', 'Attribute'] + def __init__(self, template_dir): """This class is used to fill a new MISP object with the default values defined in the object template * template is the path to the template within the misp-object repository @@ -46,53 +83,38 @@ class MISPObjectGenerator(AbstractMISP): 'data', 'misp-objects', 'objects') with open(os.path.join(self.misp_objects_path, template_dir, 'definition.json'), 'r') as f: self.definition = json.load(f) - self.attributes = self.definition['attributes'].keys() + self.object_attributes = self.definition['attributes'].keys() self.misp_event = MISPEvent() + self.name = self.definition['name'] + setattr(self, 'meta-category', self.definition['meta-category']) self.uuid = str(uuid.uuid4()) + self.description = self.definition['description'] + self.version = self.definition['version'] + self.Attribute = [] self.references = [] def _create_attribute(self, object_type, **value): if value.get('value') is None: return None - # Initialize the new MISPAttribute - # Get the misp attribute type from the definition - value['type'] = self.definition['attributes'][object_type]['misp-attribute'] - if value.get('disable_correlation') is None: - # The correlation can be disabled by default in the object definition. - # Use this value if it isn't overloaded by the object - value['disable_correlation'] = self.definition['attributes'][object_type].get('disable_correlation') - if value.get('to_ids') is None: - # Same for the to_ids flag - value['to_ids'] = self.definition['attributes'][object_type].get('to_ids') - # Set all the values in the MISP attribute - attribute = MISPAttribute(self.misp_event.describe_types) - attribute.set_all_values(**value) - self[object_type] = attribute + attribute = MISPObjectAttribute(self.definition['attributes'][object_type], object_type, **value) + self.Attribute.append(attribute) + return attribute - def dump(self, strict=True): - """Create a new object with the values gathered by the sub-class, use the default values from the template if needed""" + def to_dict(self, strict=True): if strict: self._validate() - # Create an empty object based om the object definition - new_object = self.__new_empty_object(self.definition) - for object_type, attribute in self.items(): - # Add all the values as MISPAttributes to the current object - if attribute.value is None: - continue - # Finalize the actual MISP Object - # FIXME: This only works on python >= 3.5 - # new_object['Attribute'].append({'object_relation': object_type, **attribute._json()}) - # ### BEGIN #### - # Because we still need to support old python. - temp_attribute = {'object_relation': object_type} - temp_attribute.update(attribute._json()) - new_object['Attribute'].append(temp_attribute) - # ### END #### - return new_object, [r.to_dict() for r in self.references] + return super(MISPObjectGenerator, self).to_dict() + + def to_json(self, strict=True): + if strict: + self._validate() + return super(MISPObjectGenerator, self).to_json() def _validate(self): """Make sure the object we're creating has the required fields""" - all_attribute_names = set(self.keys()) + all_attribute_names = set() + for a in self.Attribute: + all_attribute_names.add(a.object_relation) if self.definition.get('requiredOneOf'): if not set(self.definition['requiredOneOf']) & all_attribute_names: raise InvalidMISPObject('At least one of the following attributes is required: {}'.format(', '.join(self.definition['requiredOneOf']))) @@ -102,15 +124,9 @@ class MISPObjectGenerator(AbstractMISP): raise InvalidMISPObject('{} is required is required'.format(r)) return True - def add_reference(self, uuid, relationship_type, comment=None): + def add_reference(self, destination_uuid, relationship_type, comment=None): """Add a link (uuid) to an other object""" - self.references.append(MISPObjectReference(uuid, relationship_type, comment)) - - def __new_empty_object(self, object_definiton): - """Create a new empty object out of the template""" - return {'name': object_definiton['name'], 'meta-category': object_definiton['meta-category'], - 'uuid': self.uuid, 'description': object_definiton['description'], - 'version': object_definiton['version'], 'Attribute': []} + self.references.append(MISPObjectReference(self.uuid, destination_uuid, relationship_type, comment)) @abc.abstractmethod def generate_attributes(self): diff --git a/pymisp/tools/peobject.py b/pymisp/tools/peobject.py index 3a86924..e7dc206 100644 --- a/pymisp/tools/peobject.py +++ b/pymisp/tools/peobject.py @@ -5,6 +5,7 @@ from pymisp.tools import MISPObjectGenerator from io import BytesIO from hashlib import md5, sha1, sha256, sha512 from datetime import datetime +import warnings try: @@ -24,7 +25,7 @@ class PEObject(MISPObjectGenerator): def __init__(self, parsed=None, filepath=None, pseudofile=None): if not HAS_PYDEEP: - raise ImportError("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git") + warnings.warn("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git") if not HAS_LIEF: raise ImportError('Please install lief, documentation here: https://github.com/lief-project/LIEF') if pseudofile: @@ -120,8 +121,8 @@ class PESectionObject(MISPObjectGenerator): def generate_attributes(self): self._create_attribute('name', value=self.section.name) - self._create_attribute('size-in-bytes', value=self.section.size) - if getattr(self, 'size-in-bytes').value > 0: + size = self._create_attribute('size-in-bytes', value=self.section.size) + if int(size.value) > 0: self._create_attribute('entropy', value=self.section.entropy) self._create_attribute('md5', value=md5(self.data).hexdigest()) self._create_attribute('sha1', value=sha1(self.data).hexdigest())