From 77845bd813ea6bb647ab6e6a62dd65cfa861c21b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 23 Aug 2017 15:36:13 +0200 Subject: [PATCH] Update file/pe/pe-sections objects creator. --- examples/add_file_object.py | 42 +++++------ pymisp/__init__.py | 1 + pymisp/abstract.py | 70 +++++++++++++++++++ pymisp/api.py | 6 ++ pymisp/data/misp-objects | 2 +- pymisp/mispevent.py | 6 +- pymisp/tools/create_misp_object.py | 2 +- pymisp/tools/fileobject.py | 40 +++-------- pymisp/tools/objectgenerator.py | 83 +++++++++++----------- pymisp/tools/peobject.py | 107 +++++++++-------------------- setup.py | 6 +- 11 files changed, 191 insertions(+), 174 deletions(-) create mode 100644 pymisp/abstract.py diff --git a/examples/add_file_object.py b/examples/add_file_object.py index 91568ef..7780ab9 100755 --- a/examples/add_file_object.py +++ b/examples/add_file_object.py @@ -7,7 +7,7 @@ import traceback from keys import misp_url, misp_key, misp_verifycert import glob import argparse - +import json if __name__ == '__main__': parser = argparse.ArgumentParser(description='Extract indicators out of binaries and add MISP objects to a MISP instance.') @@ -18,31 +18,27 @@ if __name__ == '__main__': pymisp = PyMISP(misp_url, misp_key, misp_verifycert) for f in glob.glob(args.path): - print('\n', f) try: fo, peo, seos = make_binary_objects(f) except Exception as e: traceback.print_exc() - continue - if fo: - template_id = pymisp.get_object_template_id(fo['name']) - try: - response = pymisp.add_object(args.event, template_id, fo) - print(response) - except Exception as e: - traceback.print_exc() - continue - continue - if peo: - template_id = pymisp.get_object_template_id(peo['name']) - print(template_id) - r = pymisp.add_object(args.event, template_id, peo) - print(r) - continue + if seos: for s in seos: - print(s) - template_id = pymisp.get_object_template_id(s['name']) - r = pymisp.add_object(args.event, template_id, s) - print(r) - break + obj, refs = s + template_id = pymisp.get_object_template_id(obj['name']) + r = pymisp.add_object(args.event, template_id, obj) + + if peo: + obj, refs = peo + template_id = pymisp.get_object_template_id(obj['name']) + r = pymisp.add_object(args.event, template_id, obj) + for ref in refs: + r = pymisp.add_object_reference(obj['uuid'], ref) + + if fo: + obj, refs = fo + template_id = pymisp.get_object_template_id(obj['name']) + response = pymisp.add_object(args.event, template_id, obj) + for ref in refs: + r = pymisp.add_object_reference(obj['uuid'], ref) diff --git a/pymisp/__init__.py b/pymisp/__init__.py index 8bf84d1..27967ec 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -2,6 +2,7 @@ __version__ = '2.4.77' from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey from .api import PyMISP +from .abstract import AbstractMISP from .mispevent import MISPEvent, MISPAttribute, EncodeUpdate, EncodeFull from .tools import Neo4j from .tools import stix diff --git a/pymisp/abstract.py b/pymisp/abstract.py new file mode 100644 index 0000000..ceea59b --- /dev/null +++ b/pymisp/abstract.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import six # Remove that import when discarding python2 support. +import abc +import json +import collections + + +@six.add_metaclass(abc.ABCMeta) # Remove that line when discarding python2 support. +# Python3 way: class MISPObjectGenerator(metaclass=abc.ABCMeta): +class AbstractMISP(collections.MutableMapping): + + attributes = None + + def __init__(self): + """Initialize the list of class-level attributes to set in the JSON dump""" + # The attribute names will be set automatically by the schemas when we will have them. + if self.attributes is None: + raise NotImplementedError('{} must define attributes'.format(type(self).__name__)) + self.attributes = sorted(self.attributes) + + def __check_dict_key(self, key): + if key not in self.attributes: + raise Exception('{} not a valid key in {}. Alowed keys: {}'.format( + key, type(self).__name__, ', '.join(self.attributes))) + return True + + def from_dict(self, **kwargs): + for attribute in self.attributes: + val = kwargs.pop(attribute, None) + if val is None: + continue + setattr(self, attribute, val) + if kwargs: + raise Exception('Unused parameter(s): {}'.format(', '.join(kwargs.keys()))) + + def from_json(self, json_string): + """Load a JSON string""" + self.from_dict(json.loads(json_string)) + + def to_dict(self): + to_return = {} + for attribute in self.attributes: + val = getattr(self, attribute, None) + if val is None: + continue + to_return[attribute] = val + return to_return + + def to_json(self): + return json.dumps(self.to_dict(), indent=4, sort_keys=True) + + def __getitem__(self, key): + if self.__check_dict_key(key): + return getattr(self, key) + + def __setitem__(self, key, value): + if self.__check_dict_key(key): + setattr(self, key, value) + + def __delitem__(self, key): + if self.__check_dict_key(key): + delattr(self, key) + + def __iter__(self): + return iter(self.to_dict()) + + def __len__(self): + return len(self.to_dict()) diff --git a/pymisp/api.py b/pymisp/api.py index 777926e..ecbd62d 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -1588,6 +1588,12 @@ class PyMISP(object): response = session.post(url, data=json.dumps(misp_object)) return self._check_response(response) + def add_object_reference(self, parent_uuid, misp_object_reference): + session = self.__prepare_session() + url = urljoin(self.root_url, 'object_references/add/{}'.format(parent_uuid)) + response = session.post(url, data=json.dumps(misp_object_reference)) + return self._check_response(response) + def get_object_templates_list(self): session = self.__prepare_session() url = urljoin(self.root_url, 'objectTemplates') diff --git a/pymisp/data/misp-objects b/pymisp/data/misp-objects index ca24684..96d7aeb 160000 --- a/pymisp/data/misp-objects +++ b/pymisp/data/misp-objects @@ -1 +1 @@ -Subproject commit ca24684e2f49bcfbd886212ff003472716c26de9 +Subproject commit 96d7aeb0729428a43f38f45b6c00a60b9fdba2b6 diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 03bcebf..2bbbe3f 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -11,6 +11,7 @@ import base64 from io import BytesIO from zipfile import ZipFile import hashlib +from .abstract import AbstractMISP try: from dateutil.parser import parse @@ -198,8 +199,7 @@ class MISPAttribute(object): self.malware_filename = self.value m = hashlib.md5() m.update(self.data.getvalue()) - md5 = m.hexdigest() - self.value = '{}|{}'.format(self.malware_filename, md5) + self.value = self.malware_filename self.malware_binary = self.data self.encrypt = True @@ -280,7 +280,7 @@ def _int_to_str(d): return d -class MISPEvent(object): +class MISPEvent(AbstractMISP): def __init__(self, describe_types=None, strict_validation=False): self.ressources_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data') diff --git a/pymisp/tools/create_misp_object.py b/pymisp/tools/create_misp_object.py index 3c2a619..4d4df6e 100644 --- a/pymisp/tools/create_misp_object.py +++ b/pymisp/tools/create_misp_object.py @@ -16,7 +16,7 @@ class FileTypeNotImplemented(MISPObjectException): def make_pe_objects(lief_parsed, misp_file): misp_pe = PEObject(parsed=lief_parsed) - misp_file.add_link(misp_pe.uuid, 'PE indicators') + misp_file.add_reference(misp_pe.uuid, 'included-in', 'PE indicators') file_object = misp_file.dump() pe_object = misp_pe.dump() pe_sections = [] diff --git a/pymisp/tools/fileobject.py b/pymisp/tools/fileobject.py index 6035b43..f21cd2f 100644 --- a/pymisp/tools/fileobject.py +++ b/pymisp/tools/fileobject.py @@ -45,15 +45,17 @@ class FileObject(MISPObjectGenerator): self.generate_attributes() def generate_attributes(self): - self.size = len(self.data) - if self.size > 0: - self.entropy = self.__entropy_H(self.data) - self.md5 = md5(self.data).hexdigest() - self.sha1 = sha1(self.data).hexdigest() - self.sha256 = sha256(self.data).hexdigest() - self.sha512 = sha512(self.data).hexdigest() - self.filetype = magic.from_buffer(self.data) - self.ssdeep = pydeep.hash_buf(self.data).decode() + self._create_attribute('filename', value=self.filename) + self._create_attribute('size-in-bytes', value=len(self.data)) + if getattr(self, 'size-in-bytes').value > 0: + self._create_attribute('entropy', value=self.__entropy_H(self.data)) + self._create_attribute('md5', value=md5(self.data).hexdigest()) + self._create_attribute('sha1', value=sha1(self.data).hexdigest()) + self._create_attribute('sha256', value=sha256(self.data).hexdigest()) + self._create_attribute('sha512', value=sha512(self.data).hexdigest()) + self._create_attribute('mimetype', value=magic.from_buffer(self.data)) + self._create_attribute('ssdeep', value=pydeep.hash_buf(self.data).decode()) + self._create_attribute('malware-sample', value=self.filename.value, data=self.pseudofile) def __entropy_H(self, data): """Calculate the entropy of a chunk of data.""" @@ -70,23 +72,3 @@ class FileObject(MISPObjectGenerator): entropy -= p_x * math.log(p_x, 2) return entropy - - def dump(self): - file_object = {} - file_object['filename'] = {'value': self.filename} - file_object['size-in-bytes'] = {'value': self.size} - if self.size > 0: - file_object['entropy'] = {'value': self.entropy} - file_object['ssdeep'] = {'value': self.ssdeep} - file_object['sha512'] = {'value': self.sha512} - file_object['md5'] = {'value': self.md5} - file_object['sha1'] = {'value': self.sha1} - file_object['sha256'] = {'value': self.sha256} - file_object['malware-sample'] = {'value': '{}|{}'.format(self.filename, self.md5), 'data': self.pseudofile} - # file_object['authentihash'] = self. - # file_object['sha-224'] = self. - # file_object['sha-384'] = self. - # file_object['sha512/224'] = self. - # file_object['sha512/256'] = self. - # file_object['tlsh'] = self. - return self._fill_object(file_object) diff --git a/pymisp/tools/objectgenerator.py b/pymisp/tools/objectgenerator.py index 086b583..970c877 100644 --- a/pymisp/tools/objectgenerator.py +++ b/pymisp/tools/objectgenerator.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from pymisp import MISPEvent, MISPAttribute +from pymisp import MISPEvent, MISPAttribute, AbstractMISP import os import json import uuid @@ -24,9 +24,17 @@ if six.PY2: warnings.warn("You're using python 2, it is strongly recommended to use python >=3.4") -@six.add_metaclass(abc.ABCMeta) # Remove that line when discarding python2 support. -# Python3 way: class MISPObjectGenerator(metaclass=abc.ABCMeta): -class MISPObjectGenerator(): +class MISPObjectReference(AbstractMISP): + + attributes = ['uuid', 'relationship_type', 'comment'] + + def __init__(self, uuid, relationship_type, comment=None): + self['uuid'] = uuid + self['relationship_type'] = relationship_type + self['comment'] = comment + + +class MISPObjectGenerator(AbstractMISP): def __init__(self, template_dir): """This class is used to fill a new MISP object with the default values defined in the object template @@ -38,46 +46,46 @@ class MISPObjectGenerator(): 'data', 'misp-objects', 'objects') with open(os.path.join(self.misp_objects_path, template_dir, 'definition.json'), 'r') as f: self.definition = json.load(f) + self.attributes = self.definition['attributes'].keys() self.misp_event = MISPEvent() self.uuid = str(uuid.uuid4()) - self.links = [] + self.references = [] - def _fill_object(self, values, strict=True): + def _create_attribute(self, object_type, **value): + if value.get('value') is None: + return None + # Initialize the new MISPAttribute + # Get the misp attribute type from the definition + value['type'] = self.definition['attributes'][object_type]['misp-attribute'] + if value.get('disable_correlation') is None: + # The correlation can be disabled by default in the object definition. + # Use this value if it isn't overloaded by the object + value['disable_correlation'] = self.definition['attributes'][object_type].get('disable_correlation') + if value.get('to_ids') is None: + # Same for the to_ids flag + value['to_ids'] = self.definition['attributes'][object_type].get('to_ids') + # Set all the values in the MISP attribute + attribute = MISPAttribute(self.misp_event.describe_types) + attribute.set_all_values(**value) + self[object_type] = attribute + + def dump(self, strict=True): """Create a new object with the values gathered by the sub-class, use the default values from the template if needed""" if strict: - self._validate(values) + self._validate() # Create an empty object based om the object definition new_object = self.__new_empty_object(self.definition) - if self.links: - # Set the links to other objects - new_object["ObjectReference"] = [] - for link in self.links: - uuid, comment = link - new_object['ObjectReference'].append({'referenced_object_uuid': uuid, 'comment': comment}) - for object_type, value in values.items(): + for object_type, attribute in self.items(): # Add all the values as MISPAttributes to the current object - if value.get('value') is None: + if attribute.value is None: continue - # Initialize the new MISPAttribute - attribute = MISPAttribute(self.misp_event.describe_types) - # Get the misp attribute type from the definition - value['type'] = self.definition['attributes'][object_type]['misp-attribute'] - if value.get('disable_correlation') is None: - # The correlation can be disabled by default in the object definition. - # Use this value if it isn't overloaded by the object - value['disable_correlation'] = self.definition['attributes'][object_type].get('disable_correlation') - if value.get('to_ids') is None: - # Same for the to_ids flag - value['to_ids'] = self.definition['attributes'][object_type].get('to_ids') - # Set all the values in the MISP attribute - attribute.set_all_values(**value) # Finalize the actual MISP Object - new_object['Attribute'].append({'type': object_type, 'Attribute': attribute._json()}) - return new_object + new_object['Attribute'].append({'object_relation': object_type, **attribute._json()}) + return new_object, [r.to_dict() for r in self.references] - def _validate(self, dump): + def _validate(self): """Make sure the object we're creating has the required fields""" - all_attribute_names = set(dump.keys()) + all_attribute_names = set(self.keys()) if self.definition.get('requiredOneOf'): if not set(self.definition['requiredOneOf']) & all_attribute_names: raise InvalidMISPObject('At least one of the following attributes is required: {}'.format(', '.join(self.definition['requiredOneOf']))) @@ -87,9 +95,9 @@ class MISPObjectGenerator(): raise InvalidMISPObject('{} is required is required'.format(r)) return True - def add_link(self, uuid, comment=None): + def add_reference(self, uuid, relationship_type, comment=None): """Add a link (uuid) to an other object""" - self.links.append((uuid, comment)) + self.references.append(MISPObjectReference(uuid, relationship_type, comment)) def __new_empty_object(self, object_definiton): """Create a new empty object out of the template""" @@ -101,10 +109,3 @@ class MISPObjectGenerator(): def generate_attributes(self): """Contains the logic where all the values of the object are gathered""" pass - - @abc.abstractmethod - def dump(self): - """This method normalize the attributes to add to the object. - It returns an python dictionary where the key is the type defined in the object, - and the value the value of the MISP Attribute""" - pass diff --git a/pymisp/tools/peobject.py b/pymisp/tools/peobject.py index a72062b..3a86924 100644 --- a/pymisp/tools/peobject.py +++ b/pymisp/tools/peobject.py @@ -60,18 +60,21 @@ class PEObject(MISPObjectGenerator): return True return False - def generate_attributes(self): + def _get_pe_type(self): if self._is_dll(): - self.pe_type = 'dll' + return 'dll' elif self._is_driver(): - self.pe_type = 'driver' + return 'driver' elif self._is_exe(): - self.pe_type = 'exe' + return 'exe' else: - self.pe_type = 'unknown' + return 'unknown' + + def generate_attributes(self): + self._create_attribute('type', value=self._get_pe_type()) # General information - self.entrypoint_address = self.pe.entrypoint - self.compilation_timestamp = datetime.utcfromtimestamp(self.pe.header.time_date_stamps).isoformat() + self._create_attribute('entrypoint-address', value=self.pe.entrypoint) + self._create_attribute('compilation-timestamp', value=datetime.utcfromtimestamp(self.pe.header.time_date_stamps).isoformat()) # self.imphash = self.pe.get_imphash() try: if (self.pe.has_resources and @@ -79,15 +82,15 @@ class PEObject(MISPObjectGenerator): self.pe.resources_manager.version.has_string_file_info and self.pe.resources_manager.version.string_file_info.langcode_items): fileinfo = dict(self.pe.resources_manager.version.string_file_info.langcode_items[0].items.items()) - self.original_filename = fileinfo.get('OriginalFilename') - self.internal_filename = fileinfo.get('InternalName') - self.file_description = fileinfo.get('FileDescription') - self.file_version = fileinfo.get('FileVersion') - self.lang_id = self.pe.resources_manager.version.string_file_info.langcode_items[0].key - self.product_name = fileinfo.get('ProductName') - self.product_version = fileinfo.get('ProductVersion') - self.company_name = fileinfo.get('CompanyName') - self.legal_copyright = fileinfo.get('LegalCopyright') + self._create_attribute('original-filename', value=fileinfo.get('OriginalFilename')) + self._create_attribute('internal-filename', value=fileinfo.get('InternalName')) + self._create_attribute('file-description', value=fileinfo.get('FileDescription')) + self._create_attribute('file-version', value=fileinfo.get('FileVersion')) + self._create_attribute('lang-id', value=self.pe.resources_manager.version.string_file_info.langcode_items[0].key) + self._create_attribute('product-name', value=fileinfo.get('ProductName')) + self._create_attribute('product-version', value=fileinfo.get('ProductVersion')) + self._create_attribute('company-name', value=fileinfo.get('CompanyName')) + self._create_attribute('legal-copyright', value=fileinfo.get('LegalCopyright')) except lief.read_out_of_bound: # The file is corrupted pass @@ -97,46 +100,15 @@ class PEObject(MISPObjectGenerator): pos = 0 for section in self.pe.sections: s = PESectionObject(section) - self.add_link(s.uuid, 'Section {} of PE'.format(pos)) - if ((self.entrypoint_address >= section.virtual_address) and - (self.entrypoint_address < (section.virtual_address + section.virtual_size))): - self.entrypoint_section = (section.name, pos) # Tuple: (section_name, position) + self.add_reference(s.uuid, 'included-in', 'Section {} of PE'.format(pos)) + if ((self.pe.entrypoint >= section.virtual_address) and + (self.pe.entrypoint < (section.virtual_address + section.virtual_size))): + self._create_attribute('entrypoint-section|position', value='{}|{}'.format(section.name, pos)) pos += 1 self.sections.append(s) - self.nb_sections = len(self.sections) + self._create_attribute('number-sections', value=len(self.sections)) # TODO: TLSSection / DIRECTORY_ENTRY_TLS - def dump(self): - pe_object = {} - pe_object['type'] = {'value': self.pe_type} - if hasattr(self, 'imphash'): - pe_object['imphash'] = {'value': self.imphash} - if hasattr(self, 'original_filename'): - pe_object['original-filename'] = {'value': self.original_filename} - if hasattr(self, 'internal_filename'): - pe_object['internal-filename'] = {'value': self.internal_filename} - if hasattr(self, 'compilation_timestamp'): - pe_object['compilation-timestamp'] = {'value': self.compilation_timestamp} - if hasattr(self, 'entrypoint_section'): - pe_object['entrypoint-section|position'] = {'value': '{}|{}'.format(*self.entrypoint_section)} - if hasattr(self, 'entrypoint_address'): - pe_object['entrypoint-address'] = {'value': self.entrypoint_address} - if hasattr(self, 'file_description'): - pe_object['file-description'] = {'value': self.file_description} - if hasattr(self, 'file_version'): - pe_object['file-version'] = {'value': self.file_version} - if hasattr(self, 'lang_id'): - pe_object['lang-id'] = {'value': self.lang_id} - if hasattr(self, 'product_name'): - pe_object['product-name'] = {'value': self.product_name} - if hasattr(self, 'product_version'): - pe_object['product-version'] = {'value': self.product_version} - if hasattr(self, 'company_name'): - pe_object['company-name'] = {'value': self.company_name} - if hasattr(self, 'nb_sections'): - pe_object['number-sections'] = {'value': self.nb_sections} - return self._fill_object(pe_object) - class PESectionObject(MISPObjectGenerator): @@ -147,26 +119,13 @@ class PESectionObject(MISPObjectGenerator): self.generate_attributes() def generate_attributes(self): - self.name = self.section.name - self.size = self.section.size - if self.size > 0: - self.entropy = self.section.entropy - self.md5 = md5(self.data).hexdigest() - self.sha1 = sha1(self.data).hexdigest() - self.sha256 = sha256(self.data).hexdigest() - self.sha512 = sha512(self.data).hexdigest() + self._create_attribute('name', value=self.section.name) + self._create_attribute('size-in-bytes', value=self.section.size) + if getattr(self, 'size-in-bytes').value > 0: + self._create_attribute('entropy', value=self.section.entropy) + self._create_attribute('md5', value=md5(self.data).hexdigest()) + self._create_attribute('sha1', value=sha1(self.data).hexdigest()) + self._create_attribute('sha256', value=sha256(self.data).hexdigest()) + self._create_attribute('sha512', value=sha512(self.data).hexdigest()) if HAS_PYDEEP: - self.ssdeep = pydeep.hash_buf(self.data).decode() - - def dump(self): - section = {} - section['name'] = {'value': self.name} - section['size-in-bytes'] = {'value': self.size} - if self.size > 0: - section['entropy'] = {'value': self.entropy} - section['md5'] = {'value': self.md5} - section['sha1'] = {'value': self.sha1} - section['sha256'] = {'value': self.sha256} - section['sha512'] = {'value': self.sha512} - section['ssdeep'] = {'value': self.ssdeep} - return self._fill_object(section) + self._create_attribute('ssdeep', value=pydeep.hash_buf(self.data).decode()) diff --git a/setup.py b/setup.py index ad1f928..67f86fd 100644 --- a/setup.py +++ b/setup.py @@ -29,6 +29,8 @@ setup( test_suite="tests", install_requires=['requests', 'python-dateutil', 'jsonschema'], include_package_data=True, - package_data={'pymisp': ['data/*.json', 'data/misp-objects/schema.json', - 'data/misp-objects/objects/*/definition.json']}, + package_data={'pymisp': ['data/*.json', 'data/misp-objects/schema_objects.json', + 'data/misp-objects/schema_relationships.json', + 'data/misp-objects/objects/*/definition.json', + 'data/misp-objects/relationships/definition.json']}, )