From 78488db7aa7b6e54fb96b570fd54244d69f26173 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 21 Jul 2017 18:47:10 +0200 Subject: [PATCH 01/29] Initial commit supporting MISP Objects --- .gitmodules | 3 + pymisp/__init__.py | 3 +- pymisp/api.py | 12 ++ pymisp/data/misp-objects | 1 + pymisp/tools/__init__.py | 5 + pymisp/tools/create_misp_object.py | 49 ++++++++ pymisp/tools/fileobject.py | 92 +++++++++++++++ pymisp/tools/objectgenerator.py | 102 +++++++++++++++++ pymisp/tools/peobject.py | 172 ++++++++++++++++++++++++++++ pymisp/tools/prepare_misp_object.py | 51 +++++++++ setup.py | 5 +- 11 files changed, 492 insertions(+), 3 deletions(-) create mode 100644 .gitmodules create mode 160000 pymisp/data/misp-objects create mode 100644 pymisp/tools/create_misp_object.py create mode 100644 pymisp/tools/fileobject.py create mode 100644 pymisp/tools/objectgenerator.py create mode 100644 pymisp/tools/peobject.py create mode 100644 pymisp/tools/prepare_misp_object.py diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..d0e9062 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "pymisp/data/misp-objects"] + path = pymisp/data/misp-objects + url = https://github.com/MISP/misp-objects diff --git a/pymisp/__init__.py b/pymisp/__init__.py index 9473142..8bf84d1 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -3,5 +3,6 @@ __version__ = '2.4.77' from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey from .api import PyMISP from .mispevent import MISPEvent, MISPAttribute, EncodeUpdate, EncodeFull -from .tools.neo4j import Neo4j +from .tools import Neo4j from .tools import stix +from .tools import MISPObjectGenerator diff --git a/pymisp/api.py b/pymisp/api.py index 3c17d18..ad9d685 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -1578,6 +1578,18 @@ class PyMISP(object): response = session.post(url) return self._check_response(response) + # ################### + # ### Objects ### + # ################### + + def add_object(self, event_id, template_id, misp_object): + session = self.__prepare_session() + url = urljoin(self.root_url, 'objectTemplates/add/{}/{}'.format(event_id, template_id)) + if not misp_object.get('object'): + misp_object = {'object': misp_object} + response = session.post(url, data=json.dumps(misp_object)) + return self._check_response(response) + # ########################### # ####### Deprecated ######## # ########################### diff --git a/pymisp/data/misp-objects b/pymisp/data/misp-objects new file mode 160000 index 0000000..ca24684 --- /dev/null +++ b/pymisp/data/misp-objects @@ -0,0 +1 @@ +Subproject commit ca24684e2f49bcfbd886212ff003472716c26de9 diff --git a/pymisp/tools/__init__.py b/pymisp/tools/__init__.py index e69de29..dc1748c 100644 --- a/pymisp/tools/__init__.py +++ b/pymisp/tools/__init__.py @@ -0,0 +1,5 @@ +from .neo4j import Neo4j +from .objectgenerator import MISPObjectGenerator, MISPObjectException, InvalidMISPObject +from .fileobject import FileObject +from .peobject import PEObject, PESectionObject +from .create_misp_object import make_binary_objects diff --git a/pymisp/tools/create_misp_object.py b/pymisp/tools/create_misp_object.py new file mode 100644 index 0000000..cb2f917 --- /dev/null +++ b/pymisp/tools/create_misp_object.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +from pymisp.tools import FileObject, PEObject, MISPObjectException + +try: + import lief + HAS_LIEF = True +except ImportError: + HAS_LIEF = False + + +class FileTypeNotImplemented(MISPObjectException): + pass + + +def make_pe_objects(lief_parsed, misp_file): + misp_pe = PEObject(parsed=lief_parsed) + misp_file.add_link(misp_pe.uuid, 'PE indicators') + file_object = misp_file.dump() + pe_object = misp_pe.dump() + pe_sections = [] + for s in misp_pe.sections: + pe_sections.append(s.dump()) + return file_object, pe_object, pe_sections + + +def make_binary_objects(filepath): + if not HAS_LIEF: + raise ImportError('Please install lief, documentation here: https://github.com/lief-project/LIEF') + misp_file = FileObject(filepath) + try: + lief_parsed = lief.parse(filepath) + if isinstance(lief_parsed, lief.PE.Binary): + make_pe_objects(lief_parsed, misp_file) + elif isinstance(lief_parsed, lief.ELF.Binary): + raise FileTypeNotImplemented('ELF not implemented yet.') + elif isinstance(lief_parsed, lief.MachO.Binary): + raise FileTypeNotImplemented('MachO not implemented yet.') + except lief.bad_format as e: + print('\tBad format: ', e) + except lief.bad_file as e: + print('\tBad file: ', e) + except lief.parser_error as e: + print('\tParser error: ', e) + except FileTypeNotImplemented as e: + print(e) + file_object = misp_file.dump() + return file_object, None, None diff --git a/pymisp/tools/fileobject.py b/pymisp/tools/fileobject.py new file mode 100644 index 0000000..6035b43 --- /dev/null +++ b/pymisp/tools/fileobject.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +from pymisp.tools import MISPObjectGenerator +import os +from io import BytesIO +from hashlib import md5, sha1, sha256, sha512 +import math +from collections import Counter + +try: + import pydeep + HAS_PYDEEP = True +except ImportError: + HAS_PYDEEP = False + +try: + import magic + HAS_MAGIC = True +except ImportError: + HAS_MAGIC = False + + +class FileObject(MISPObjectGenerator): + + def __init__(self, filepath=None, pseudofile=None, filename=None): + if not HAS_PYDEEP: + raise ImportError("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git") + if not HAS_MAGIC: + raise ImportError("Please install python-magic: pip install python-magic.") + if filepath: + self.filepath = filepath + self.filename = os.path.basename(self.filepath) + with open(filepath, 'rb') as f: + self.pseudofile = BytesIO(f.read()) + elif pseudofile and isinstance(pseudofile, BytesIO): + # WARNING: lief.parse requires a path + self.filepath = None + self.pseudofile = pseudofile + self.filename = filename + else: + raise Exception('File buffer (BytesIO) or a path is required.') + MISPObjectGenerator.__init__(self, 'file') + self.data = self.pseudofile.getvalue() + self.generate_attributes() + + def generate_attributes(self): + self.size = len(self.data) + if self.size > 0: + self.entropy = self.__entropy_H(self.data) + self.md5 = md5(self.data).hexdigest() + self.sha1 = sha1(self.data).hexdigest() + self.sha256 = sha256(self.data).hexdigest() + self.sha512 = sha512(self.data).hexdigest() + self.filetype = magic.from_buffer(self.data) + self.ssdeep = pydeep.hash_buf(self.data).decode() + + def __entropy_H(self, data): + """Calculate the entropy of a chunk of data.""" + # NOTE: copy of the entropy function from pefile + + if len(data) == 0: + return 0.0 + + occurences = Counter(bytearray(data)) + + entropy = 0 + for x in occurences.values(): + p_x = float(x) / len(data) + entropy -= p_x * math.log(p_x, 2) + + return entropy + + def dump(self): + file_object = {} + file_object['filename'] = {'value': self.filename} + file_object['size-in-bytes'] = {'value': self.size} + if self.size > 0: + file_object['entropy'] = {'value': self.entropy} + file_object['ssdeep'] = {'value': self.ssdeep} + file_object['sha512'] = {'value': self.sha512} + file_object['md5'] = {'value': self.md5} + file_object['sha1'] = {'value': self.sha1} + file_object['sha256'] = {'value': self.sha256} + file_object['malware-sample'] = {'value': '{}|{}'.format(self.filename, self.md5), 'data': self.pseudofile} + # file_object['authentihash'] = self. + # file_object['sha-224'] = self. + # file_object['sha-384'] = self. + # file_object['sha512/224'] = self. + # file_object['sha512/256'] = self. + # file_object['tlsh'] = self. + return self._fill_object(file_object) diff --git a/pymisp/tools/objectgenerator.py b/pymisp/tools/objectgenerator.py new file mode 100644 index 0000000..c0a7c63 --- /dev/null +++ b/pymisp/tools/objectgenerator.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from pymisp import MISPEvent, MISPAttribute +import os +import json +import uuid +import abc +import sys + + +class MISPObjectException(Exception): + pass + + +class InvalidMISPObject(MISPObjectException): + """Exception raised when an object doesn't contains the required field(s)""" + pass + + +class MISPObjectGenerator(metaclass=abc.ABCMeta): + + def __init__(self, template_dir): + """This class is used to fill a new MISP object with the default values defined in the object template + * template is the path to the template within the misp-object repository + * misp_objects_path is the path to the misp-object repository + """ + self.misp_objects_path = os.path.join( + os.path.abspath(os.path.dirname(sys.modules['pymisp'].__file__)), + 'data', 'misp-objects', 'objects') + with open(os.path.join(self.misp_objects_path, template_dir, 'definition.json'), 'r') as f: + self.definition = json.load(f) + self.misp_event = MISPEvent() + self.uuid = str(uuid.uuid4()) + self.links = [] + + def _fill_object(self, values, strict=True): + """Create a new object with the values gathered by the sub-class, use the default values from the template if needed""" + if strict: + self._validate(values) + # Create an empty object based om the object definition + new_object = self.__new_empty_object(self.definition) + if self.links: + # Set the links to other objects + new_object["ObjectReference"] = [] + for link in self.links: + uuid, comment = link + new_object['ObjectReference'].append({'referenced_object_uuid': uuid, 'comment': comment}) + for object_type, value in values.items(): + # Add all the values as MISPAttributes to the current object + if value.get('value') is None: + continue + # Initialize the new MISPAttribute + attribute = MISPAttribute(self.misp_event.describe_types) + # Get the misp attribute type from the definition + value['type'] = self.definition['attributes'][object_type]['misp-attribute'] + if value.get('disable_correlation') is None: + # The correlation can be disabled by default in the object definition. + # Use this value if it isn't overloaded by the object + value['disable_correlation'] = self.definition['attributes'][object_type].get('disable_correlation') + if value.get('to_ids') is None: + # Same for the to_ids flag + value['to_ids'] = self.definition['attributes'][object_type].get('to_ids') + # Set all the values in the MISP attribute + attribute.set_all_values(**value) + # Finalize the actual MISP Object + new_object['ObjectAttribute'].append({'type': object_type, 'Attribute': attribute._json()}) + return new_object + + def _validate(self, dump): + """Make sure the object we're creating has the required fields""" + all_attribute_names = set(dump.keys()) + if self.definition.get('requiredOneOf'): + if not set(self.definition['requiredOneOf']) & all_attribute_names: + raise InvalidMISPObject('At least one of the following attributes is required: {}'.format(', '.join(self.definition['requiredOneOf']))) + if self.definition.get('required'): + for r in self.definition.get('required'): + if r not in all_attribute_names: + raise InvalidMISPObject('{} is required is required'.format(r)) + return True + + def add_link(self, uuid, comment=None): + """Add a link (uuid) to an other object""" + self.links.append((uuid, comment)) + + def __new_empty_object(self, object_definiton): + """Create a new empty object out of the template""" + return {'name': object_definiton['name'], 'meta-category': object_definiton['meta-category'], + 'uuid': self.uuid, 'description': object_definiton['description'], + 'version': object_definiton['version'], 'ObjectAttribute': []} + + @abc.abstractmethod + def generate_attributes(self): + """Contains the logic where all the values of the object are gathered""" + pass + + @abc.abstractmethod + def dump(self): + """This method normalize the attributes to add to the object. + It returns an python dictionary where the key is the type defined in the object, + and the value the value of the MISP Attribute""" + pass diff --git a/pymisp/tools/peobject.py b/pymisp/tools/peobject.py new file mode 100644 index 0000000..a72062b --- /dev/null +++ b/pymisp/tools/peobject.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +from pymisp.tools import MISPObjectGenerator +from io import BytesIO +from hashlib import md5, sha1, sha256, sha512 +from datetime import datetime + + +try: + import lief + HAS_LIEF = True +except ImportError: + HAS_LIEF = False + +try: + import pydeep + HAS_PYDEEP = True +except ImportError: + HAS_PYDEEP = False + + +class PEObject(MISPObjectGenerator): + + def __init__(self, parsed=None, filepath=None, pseudofile=None): + if not HAS_PYDEEP: + raise ImportError("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git") + if not HAS_LIEF: + raise ImportError('Please install lief, documentation here: https://github.com/lief-project/LIEF') + if pseudofile: + if isinstance(pseudofile, BytesIO): + self.pe = lief.PE.parse(raw=pseudofile.getvalue()) + elif isinstance(pseudofile, bytes): + self.pe = lief.PE.parse(raw=pseudofile) + else: + raise Exception('Pseudo file can be BytesIO or bytes got {}'.format(type(pseudofile))) + elif filepath: + self.pe = lief.PE.parse(filepath) + elif parsed: + # Got an already parsed blob + if isinstance(parsed, lief.PE.Binary): + self.pe = parsed + else: + raise Exception('Not a lief.PE.Binary: {}'.format(type(parsed))) + MISPObjectGenerator.__init__(self, 'pe') + self.generate_attributes() + + def _is_exe(self): + if not self._is_dll() and not self._is_driver(): + return self.pe.header.has_characteristic(lief.PE.HEADER_CHARACTERISTICS.EXECUTABLE_IMAGE) + return False + + def _is_dll(self): + return self.pe.header.has_characteristic(lief.PE.HEADER_CHARACTERISTICS.DLL) + + def _is_driver(self): + # List from pefile + system_DLLs = set(('ntoskrnl.exe', 'hal.dll', 'ndis.sys', 'bootvid.dll', 'kdcom.dll')) + if system_DLLs.intersection([imp.lower() for imp in self.pe.libraries]): + return True + return False + + def generate_attributes(self): + if self._is_dll(): + self.pe_type = 'dll' + elif self._is_driver(): + self.pe_type = 'driver' + elif self._is_exe(): + self.pe_type = 'exe' + else: + self.pe_type = 'unknown' + # General information + self.entrypoint_address = self.pe.entrypoint + self.compilation_timestamp = datetime.utcfromtimestamp(self.pe.header.time_date_stamps).isoformat() + # self.imphash = self.pe.get_imphash() + try: + if (self.pe.has_resources and + self.pe.resources_manager.has_version and + self.pe.resources_manager.version.has_string_file_info and + self.pe.resources_manager.version.string_file_info.langcode_items): + fileinfo = dict(self.pe.resources_manager.version.string_file_info.langcode_items[0].items.items()) + self.original_filename = fileinfo.get('OriginalFilename') + self.internal_filename = fileinfo.get('InternalName') + self.file_description = fileinfo.get('FileDescription') + self.file_version = fileinfo.get('FileVersion') + self.lang_id = self.pe.resources_manager.version.string_file_info.langcode_items[0].key + self.product_name = fileinfo.get('ProductName') + self.product_version = fileinfo.get('ProductVersion') + self.company_name = fileinfo.get('CompanyName') + self.legal_copyright = fileinfo.get('LegalCopyright') + except lief.read_out_of_bound: + # The file is corrupted + pass + # Sections + self.sections = [] + if self.pe.sections: + pos = 0 + for section in self.pe.sections: + s = PESectionObject(section) + self.add_link(s.uuid, 'Section {} of PE'.format(pos)) + if ((self.entrypoint_address >= section.virtual_address) and + (self.entrypoint_address < (section.virtual_address + section.virtual_size))): + self.entrypoint_section = (section.name, pos) # Tuple: (section_name, position) + pos += 1 + self.sections.append(s) + self.nb_sections = len(self.sections) + # TODO: TLSSection / DIRECTORY_ENTRY_TLS + + def dump(self): + pe_object = {} + pe_object['type'] = {'value': self.pe_type} + if hasattr(self, 'imphash'): + pe_object['imphash'] = {'value': self.imphash} + if hasattr(self, 'original_filename'): + pe_object['original-filename'] = {'value': self.original_filename} + if hasattr(self, 'internal_filename'): + pe_object['internal-filename'] = {'value': self.internal_filename} + if hasattr(self, 'compilation_timestamp'): + pe_object['compilation-timestamp'] = {'value': self.compilation_timestamp} + if hasattr(self, 'entrypoint_section'): + pe_object['entrypoint-section|position'] = {'value': '{}|{}'.format(*self.entrypoint_section)} + if hasattr(self, 'entrypoint_address'): + pe_object['entrypoint-address'] = {'value': self.entrypoint_address} + if hasattr(self, 'file_description'): + pe_object['file-description'] = {'value': self.file_description} + if hasattr(self, 'file_version'): + pe_object['file-version'] = {'value': self.file_version} + if hasattr(self, 'lang_id'): + pe_object['lang-id'] = {'value': self.lang_id} + if hasattr(self, 'product_name'): + pe_object['product-name'] = {'value': self.product_name} + if hasattr(self, 'product_version'): + pe_object['product-version'] = {'value': self.product_version} + if hasattr(self, 'company_name'): + pe_object['company-name'] = {'value': self.company_name} + if hasattr(self, 'nb_sections'): + pe_object['number-sections'] = {'value': self.nb_sections} + return self._fill_object(pe_object) + + +class PESectionObject(MISPObjectGenerator): + + def __init__(self, section): + MISPObjectGenerator.__init__(self, 'pe-section') + self.section = section + self.data = bytes(self.section.content) + self.generate_attributes() + + def generate_attributes(self): + self.name = self.section.name + self.size = self.section.size + if self.size > 0: + self.entropy = self.section.entropy + self.md5 = md5(self.data).hexdigest() + self.sha1 = sha1(self.data).hexdigest() + self.sha256 = sha256(self.data).hexdigest() + self.sha512 = sha512(self.data).hexdigest() + if HAS_PYDEEP: + self.ssdeep = pydeep.hash_buf(self.data).decode() + + def dump(self): + section = {} + section['name'] = {'value': self.name} + section['size-in-bytes'] = {'value': self.size} + if self.size > 0: + section['entropy'] = {'value': self.entropy} + section['md5'] = {'value': self.md5} + section['sha1'] = {'value': self.sha1} + section['sha256'] = {'value': self.sha256} + section['sha512'] = {'value': self.sha512} + section['ssdeep'] = {'value': self.ssdeep} + return self._fill_object(section) diff --git a/pymisp/tools/prepare_misp_object.py b/pymisp/tools/prepare_misp_object.py new file mode 100644 index 0000000..541f6ef --- /dev/null +++ b/pymisp/tools/prepare_misp_object.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +from pymisp import PyMISP +from pymisp.tools import FileObject, PEObject +from pymisp.tools import make_binary_objects +import traceback + + +try: + import lief + HAS_LIEF = True +except ImportError: + HAS_LIEF = False + raise ImportError("Please install lief: https://github.com/lief-project/LIEF") + + +if __name__ == '__main__': + pymisp = PyMISP('https://mispbeta.circl.lu', 'et9ZEgn70YJ6URkCr6741LpJNAVUMYD1rM063od3') + + + # fo, peo, seos = make_objects('/home/raphael/.viper/projects/troopers17/vt_samples/1189/566ab945f61be016bfd9e83cc1b64f783b9b8deb891e6d504d3442bc8281b092') + import glob + for f in glob.glob('/home/raphael/.viper/projects/troopers17/vt_samples/*/*'): + #for f in glob.glob('/home/raphael/gits/pefile-tests/tests/corkami/*/*.exe'): + #for f in glob.glob('/home/raphael/gits/pefile-tests/tests/corkami/pocs/version_mini.exe'): + #for f in glob.glob('/home/raphael/gits/pefile-tests/tests/corkami/pocs/version_cust.exe'): + #for f in glob.glob('/home/raphael/gits/pefile-tests/tests/data/*.dll'): + print('\n', f) + try: + fo, peo, seos = make_binary_objects(f) + except Exception as e: + traceback.print_exc() + continue + continue + if fo: + response = pymisp.add_object(2221, 7, fo) + print(response) + if peo: + pymisp.add_object(2221, 11, peo) + if seos: + for s in seos: + pymisp.add_object(2221, 12, s) + + #with open('fileobj.json', 'w') as f: + # json.dump(fo, f) + #with open('peobj.json', 'w') as f: + # json.dump(peo, f) + #with open('seobj.json', 'w') as f: + # json.dump(seos, f) + break diff --git a/setup.py b/setup.py index 7ddac16..ad1f928 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # -*- coding: utf-8 -*- from setuptools import setup import pymisp @@ -29,5 +29,6 @@ setup( test_suite="tests", install_requires=['requests', 'python-dateutil', 'jsonschema'], include_package_data=True, - package_data={'data': ['schema.json', 'schema-lax.json', 'describeTypes.json']}, + package_data={'pymisp': ['data/*.json', 'data/misp-objects/schema.json', + 'data/misp-objects/objects/*/definition.json']}, ) From 0c66d80dd1e71fd25493c8893db951415f227d30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Sat, 22 Jul 2017 22:22:39 +0200 Subject: [PATCH 02/29] Re-enable python2 support... --- pymisp/tools/objectgenerator.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pymisp/tools/objectgenerator.py b/pymisp/tools/objectgenerator.py index c0a7c63..c113304 100644 --- a/pymisp/tools/objectgenerator.py +++ b/pymisp/tools/objectgenerator.py @@ -7,6 +7,7 @@ import json import uuid import abc import sys +import six class MISPObjectException(Exception): @@ -18,7 +19,14 @@ class InvalidMISPObject(MISPObjectException): pass -class MISPObjectGenerator(metaclass=abc.ABCMeta): +if six.PY2: + import warnings + warnings.warn("You're using python 2, it is strongly recommended to use python >=3.4") + + +@six.add_metaclass(abc.ABCMeta) # Remove that line when discarding python2 support. +# Python3 way: class MISPObjectGenerator(metaclass=abc.ABCMeta): +class MISPObjectGenerator(): def __init__(self, template_dir): """This class is used to fill a new MISP object with the default values defined in the object template From 2fd3b052020dcde4495464ae77ebe34b403e21f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 24 Jul 2017 17:16:40 +0200 Subject: [PATCH 03/29] Update accordingly to the current server implementation --- examples/add_file_object.py | 48 +++++++++++++++++++++++++++ pymisp/api.py | 17 ++++++++-- pymisp/tools/create_misp_object.py | 2 +- pymisp/tools/objectgenerator.py | 6 ++-- pymisp/tools/prepare_misp_object.py | 51 ----------------------------- 5 files changed, 66 insertions(+), 58 deletions(-) create mode 100755 examples/add_file_object.py delete mode 100644 pymisp/tools/prepare_misp_object.py diff --git a/examples/add_file_object.py b/examples/add_file_object.py new file mode 100755 index 0000000..91568ef --- /dev/null +++ b/examples/add_file_object.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +from pymisp import PyMISP +from pymisp.tools import make_binary_objects +import traceback +from keys import misp_url, misp_key, misp_verifycert +import glob +import argparse + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Extract indicators out of binaries and add MISP objects to a MISP instance.') + parser.add_argument("-e", "--event", required=True, help="Event ID to update.") + parser.add_argument("-p", "--path", required=True, help="Path to process (expanded using glob).") + args = parser.parse_args() + + pymisp = PyMISP(misp_url, misp_key, misp_verifycert) + + for f in glob.glob(args.path): + print('\n', f) + try: + fo, peo, seos = make_binary_objects(f) + except Exception as e: + traceback.print_exc() + continue + if fo: + template_id = pymisp.get_object_template_id(fo['name']) + try: + response = pymisp.add_object(args.event, template_id, fo) + print(response) + except Exception as e: + traceback.print_exc() + continue + continue + if peo: + template_id = pymisp.get_object_template_id(peo['name']) + print(template_id) + r = pymisp.add_object(args.event, template_id, peo) + print(r) + continue + if seos: + for s in seos: + print(s) + template_id = pymisp.get_object_template_id(s['name']) + r = pymisp.add_object(args.event, template_id, s) + print(r) + break diff --git a/pymisp/api.py b/pymisp/api.py index ad9d685..777926e 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -1584,12 +1584,23 @@ class PyMISP(object): def add_object(self, event_id, template_id, misp_object): session = self.__prepare_session() - url = urljoin(self.root_url, 'objectTemplates/add/{}/{}'.format(event_id, template_id)) - if not misp_object.get('object'): - misp_object = {'object': misp_object} + url = urljoin(self.root_url, 'objects/add/{}/{}'.format(event_id, template_id)) response = session.post(url, data=json.dumps(misp_object)) return self._check_response(response) + def get_object_templates_list(self): + session = self.__prepare_session() + url = urljoin(self.root_url, 'objectTemplates') + response = session.get(url) + return self._check_response(response)['response'] + + def get_object_template_id(self, object_name): + templates = self.get_object_templates_list() + for t in templates: + if t['ObjectTemplate']['name'] == object_name: + return t['ObjectTemplate']['id'] + raise Exception('Unable to find template name {} on the MISP instance'.format(object_name)) + # ########################### # ####### Deprecated ######## # ########################### diff --git a/pymisp/tools/create_misp_object.py b/pymisp/tools/create_misp_object.py index cb2f917..3c2a619 100644 --- a/pymisp/tools/create_misp_object.py +++ b/pymisp/tools/create_misp_object.py @@ -32,7 +32,7 @@ def make_binary_objects(filepath): try: lief_parsed = lief.parse(filepath) if isinstance(lief_parsed, lief.PE.Binary): - make_pe_objects(lief_parsed, misp_file) + return make_pe_objects(lief_parsed, misp_file) elif isinstance(lief_parsed, lief.ELF.Binary): raise FileTypeNotImplemented('ELF not implemented yet.') elif isinstance(lief_parsed, lief.MachO.Binary): diff --git a/pymisp/tools/objectgenerator.py b/pymisp/tools/objectgenerator.py index c113304..086b583 100644 --- a/pymisp/tools/objectgenerator.py +++ b/pymisp/tools/objectgenerator.py @@ -7,7 +7,7 @@ import json import uuid import abc import sys -import six +import six # Remove that import when discarding python2 support. class MISPObjectException(Exception): @@ -72,7 +72,7 @@ class MISPObjectGenerator(): # Set all the values in the MISP attribute attribute.set_all_values(**value) # Finalize the actual MISP Object - new_object['ObjectAttribute'].append({'type': object_type, 'Attribute': attribute._json()}) + new_object['Attribute'].append({'type': object_type, 'Attribute': attribute._json()}) return new_object def _validate(self, dump): @@ -95,7 +95,7 @@ class MISPObjectGenerator(): """Create a new empty object out of the template""" return {'name': object_definiton['name'], 'meta-category': object_definiton['meta-category'], 'uuid': self.uuid, 'description': object_definiton['description'], - 'version': object_definiton['version'], 'ObjectAttribute': []} + 'version': object_definiton['version'], 'Attribute': []} @abc.abstractmethod def generate_attributes(self): diff --git a/pymisp/tools/prepare_misp_object.py b/pymisp/tools/prepare_misp_object.py deleted file mode 100644 index 541f6ef..0000000 --- a/pymisp/tools/prepare_misp_object.py +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -from pymisp import PyMISP -from pymisp.tools import FileObject, PEObject -from pymisp.tools import make_binary_objects -import traceback - - -try: - import lief - HAS_LIEF = True -except ImportError: - HAS_LIEF = False - raise ImportError("Please install lief: https://github.com/lief-project/LIEF") - - -if __name__ == '__main__': - pymisp = PyMISP('https://mispbeta.circl.lu', 'et9ZEgn70YJ6URkCr6741LpJNAVUMYD1rM063od3') - - - # fo, peo, seos = make_objects('/home/raphael/.viper/projects/troopers17/vt_samples/1189/566ab945f61be016bfd9e83cc1b64f783b9b8deb891e6d504d3442bc8281b092') - import glob - for f in glob.glob('/home/raphael/.viper/projects/troopers17/vt_samples/*/*'): - #for f in glob.glob('/home/raphael/gits/pefile-tests/tests/corkami/*/*.exe'): - #for f in glob.glob('/home/raphael/gits/pefile-tests/tests/corkami/pocs/version_mini.exe'): - #for f in glob.glob('/home/raphael/gits/pefile-tests/tests/corkami/pocs/version_cust.exe'): - #for f in glob.glob('/home/raphael/gits/pefile-tests/tests/data/*.dll'): - print('\n', f) - try: - fo, peo, seos = make_binary_objects(f) - except Exception as e: - traceback.print_exc() - continue - continue - if fo: - response = pymisp.add_object(2221, 7, fo) - print(response) - if peo: - pymisp.add_object(2221, 11, peo) - if seos: - for s in seos: - pymisp.add_object(2221, 12, s) - - #with open('fileobj.json', 'w') as f: - # json.dump(fo, f) - #with open('peobj.json', 'w') as f: - # json.dump(peo, f) - #with open('seobj.json', 'w') as f: - # json.dump(seos, f) - break From 1d6c63c54caaa4941383b45c5d48a8398549cb54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 26 Jul 2017 10:10:12 +0200 Subject: [PATCH 04/29] Doesn't require describe_types in MISPAttribute's constructor. --- pymisp/mispevent.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 3a9665d..03bcebf 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -50,7 +50,12 @@ except NameError: class MISPAttribute(object): - def __init__(self, describe_types): + def __init__(self, describe_types=None): + if not describe_types: + with open(os.path.join(self.ressources_path, 'describeTypes.json'), 'r') as f: + t = json.load(f) + describe_types = t['result'] + self.describe_types = describe_types self.categories = describe_types['categories'] self.types = describe_types['types'] self.category_type_mapping = describe_types['category_type_mappings'] From 77845bd813ea6bb647ab6e6a62dd65cfa861c21b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 23 Aug 2017 15:36:13 +0200 Subject: [PATCH 05/29] Update file/pe/pe-sections objects creator. --- examples/add_file_object.py | 42 +++++------ pymisp/__init__.py | 1 + pymisp/abstract.py | 70 +++++++++++++++++++ pymisp/api.py | 6 ++ pymisp/data/misp-objects | 2 +- pymisp/mispevent.py | 6 +- pymisp/tools/create_misp_object.py | 2 +- pymisp/tools/fileobject.py | 40 +++-------- pymisp/tools/objectgenerator.py | 83 +++++++++++----------- pymisp/tools/peobject.py | 107 +++++++++-------------------- setup.py | 6 +- 11 files changed, 191 insertions(+), 174 deletions(-) create mode 100644 pymisp/abstract.py diff --git a/examples/add_file_object.py b/examples/add_file_object.py index 91568ef..7780ab9 100755 --- a/examples/add_file_object.py +++ b/examples/add_file_object.py @@ -7,7 +7,7 @@ import traceback from keys import misp_url, misp_key, misp_verifycert import glob import argparse - +import json if __name__ == '__main__': parser = argparse.ArgumentParser(description='Extract indicators out of binaries and add MISP objects to a MISP instance.') @@ -18,31 +18,27 @@ if __name__ == '__main__': pymisp = PyMISP(misp_url, misp_key, misp_verifycert) for f in glob.glob(args.path): - print('\n', f) try: fo, peo, seos = make_binary_objects(f) except Exception as e: traceback.print_exc() - continue - if fo: - template_id = pymisp.get_object_template_id(fo['name']) - try: - response = pymisp.add_object(args.event, template_id, fo) - print(response) - except Exception as e: - traceback.print_exc() - continue - continue - if peo: - template_id = pymisp.get_object_template_id(peo['name']) - print(template_id) - r = pymisp.add_object(args.event, template_id, peo) - print(r) - continue + if seos: for s in seos: - print(s) - template_id = pymisp.get_object_template_id(s['name']) - r = pymisp.add_object(args.event, template_id, s) - print(r) - break + obj, refs = s + template_id = pymisp.get_object_template_id(obj['name']) + r = pymisp.add_object(args.event, template_id, obj) + + if peo: + obj, refs = peo + template_id = pymisp.get_object_template_id(obj['name']) + r = pymisp.add_object(args.event, template_id, obj) + for ref in refs: + r = pymisp.add_object_reference(obj['uuid'], ref) + + if fo: + obj, refs = fo + template_id = pymisp.get_object_template_id(obj['name']) + response = pymisp.add_object(args.event, template_id, obj) + for ref in refs: + r = pymisp.add_object_reference(obj['uuid'], ref) diff --git a/pymisp/__init__.py b/pymisp/__init__.py index 8bf84d1..27967ec 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -2,6 +2,7 @@ __version__ = '2.4.77' from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey from .api import PyMISP +from .abstract import AbstractMISP from .mispevent import MISPEvent, MISPAttribute, EncodeUpdate, EncodeFull from .tools import Neo4j from .tools import stix diff --git a/pymisp/abstract.py b/pymisp/abstract.py new file mode 100644 index 0000000..ceea59b --- /dev/null +++ b/pymisp/abstract.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import six # Remove that import when discarding python2 support. +import abc +import json +import collections + + +@six.add_metaclass(abc.ABCMeta) # Remove that line when discarding python2 support. +# Python3 way: class MISPObjectGenerator(metaclass=abc.ABCMeta): +class AbstractMISP(collections.MutableMapping): + + attributes = None + + def __init__(self): + """Initialize the list of class-level attributes to set in the JSON dump""" + # The attribute names will be set automatically by the schemas when we will have them. + if self.attributes is None: + raise NotImplementedError('{} must define attributes'.format(type(self).__name__)) + self.attributes = sorted(self.attributes) + + def __check_dict_key(self, key): + if key not in self.attributes: + raise Exception('{} not a valid key in {}. Alowed keys: {}'.format( + key, type(self).__name__, ', '.join(self.attributes))) + return True + + def from_dict(self, **kwargs): + for attribute in self.attributes: + val = kwargs.pop(attribute, None) + if val is None: + continue + setattr(self, attribute, val) + if kwargs: + raise Exception('Unused parameter(s): {}'.format(', '.join(kwargs.keys()))) + + def from_json(self, json_string): + """Load a JSON string""" + self.from_dict(json.loads(json_string)) + + def to_dict(self): + to_return = {} + for attribute in self.attributes: + val = getattr(self, attribute, None) + if val is None: + continue + to_return[attribute] = val + return to_return + + def to_json(self): + return json.dumps(self.to_dict(), indent=4, sort_keys=True) + + def __getitem__(self, key): + if self.__check_dict_key(key): + return getattr(self, key) + + def __setitem__(self, key, value): + if self.__check_dict_key(key): + setattr(self, key, value) + + def __delitem__(self, key): + if self.__check_dict_key(key): + delattr(self, key) + + def __iter__(self): + return iter(self.to_dict()) + + def __len__(self): + return len(self.to_dict()) diff --git a/pymisp/api.py b/pymisp/api.py index 777926e..ecbd62d 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -1588,6 +1588,12 @@ class PyMISP(object): response = session.post(url, data=json.dumps(misp_object)) return self._check_response(response) + def add_object_reference(self, parent_uuid, misp_object_reference): + session = self.__prepare_session() + url = urljoin(self.root_url, 'object_references/add/{}'.format(parent_uuid)) + response = session.post(url, data=json.dumps(misp_object_reference)) + return self._check_response(response) + def get_object_templates_list(self): session = self.__prepare_session() url = urljoin(self.root_url, 'objectTemplates') diff --git a/pymisp/data/misp-objects b/pymisp/data/misp-objects index ca24684..96d7aeb 160000 --- a/pymisp/data/misp-objects +++ b/pymisp/data/misp-objects @@ -1 +1 @@ -Subproject commit ca24684e2f49bcfbd886212ff003472716c26de9 +Subproject commit 96d7aeb0729428a43f38f45b6c00a60b9fdba2b6 diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 03bcebf..2bbbe3f 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -11,6 +11,7 @@ import base64 from io import BytesIO from zipfile import ZipFile import hashlib +from .abstract import AbstractMISP try: from dateutil.parser import parse @@ -198,8 +199,7 @@ class MISPAttribute(object): self.malware_filename = self.value m = hashlib.md5() m.update(self.data.getvalue()) - md5 = m.hexdigest() - self.value = '{}|{}'.format(self.malware_filename, md5) + self.value = self.malware_filename self.malware_binary = self.data self.encrypt = True @@ -280,7 +280,7 @@ def _int_to_str(d): return d -class MISPEvent(object): +class MISPEvent(AbstractMISP): def __init__(self, describe_types=None, strict_validation=False): self.ressources_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data') diff --git a/pymisp/tools/create_misp_object.py b/pymisp/tools/create_misp_object.py index 3c2a619..4d4df6e 100644 --- a/pymisp/tools/create_misp_object.py +++ b/pymisp/tools/create_misp_object.py @@ -16,7 +16,7 @@ class FileTypeNotImplemented(MISPObjectException): def make_pe_objects(lief_parsed, misp_file): misp_pe = PEObject(parsed=lief_parsed) - misp_file.add_link(misp_pe.uuid, 'PE indicators') + misp_file.add_reference(misp_pe.uuid, 'included-in', 'PE indicators') file_object = misp_file.dump() pe_object = misp_pe.dump() pe_sections = [] diff --git a/pymisp/tools/fileobject.py b/pymisp/tools/fileobject.py index 6035b43..f21cd2f 100644 --- a/pymisp/tools/fileobject.py +++ b/pymisp/tools/fileobject.py @@ -45,15 +45,17 @@ class FileObject(MISPObjectGenerator): self.generate_attributes() def generate_attributes(self): - self.size = len(self.data) - if self.size > 0: - self.entropy = self.__entropy_H(self.data) - self.md5 = md5(self.data).hexdigest() - self.sha1 = sha1(self.data).hexdigest() - self.sha256 = sha256(self.data).hexdigest() - self.sha512 = sha512(self.data).hexdigest() - self.filetype = magic.from_buffer(self.data) - self.ssdeep = pydeep.hash_buf(self.data).decode() + self._create_attribute('filename', value=self.filename) + self._create_attribute('size-in-bytes', value=len(self.data)) + if getattr(self, 'size-in-bytes').value > 0: + self._create_attribute('entropy', value=self.__entropy_H(self.data)) + self._create_attribute('md5', value=md5(self.data).hexdigest()) + self._create_attribute('sha1', value=sha1(self.data).hexdigest()) + self._create_attribute('sha256', value=sha256(self.data).hexdigest()) + self._create_attribute('sha512', value=sha512(self.data).hexdigest()) + self._create_attribute('mimetype', value=magic.from_buffer(self.data)) + self._create_attribute('ssdeep', value=pydeep.hash_buf(self.data).decode()) + self._create_attribute('malware-sample', value=self.filename.value, data=self.pseudofile) def __entropy_H(self, data): """Calculate the entropy of a chunk of data.""" @@ -70,23 +72,3 @@ class FileObject(MISPObjectGenerator): entropy -= p_x * math.log(p_x, 2) return entropy - - def dump(self): - file_object = {} - file_object['filename'] = {'value': self.filename} - file_object['size-in-bytes'] = {'value': self.size} - if self.size > 0: - file_object['entropy'] = {'value': self.entropy} - file_object['ssdeep'] = {'value': self.ssdeep} - file_object['sha512'] = {'value': self.sha512} - file_object['md5'] = {'value': self.md5} - file_object['sha1'] = {'value': self.sha1} - file_object['sha256'] = {'value': self.sha256} - file_object['malware-sample'] = {'value': '{}|{}'.format(self.filename, self.md5), 'data': self.pseudofile} - # file_object['authentihash'] = self. - # file_object['sha-224'] = self. - # file_object['sha-384'] = self. - # file_object['sha512/224'] = self. - # file_object['sha512/256'] = self. - # file_object['tlsh'] = self. - return self._fill_object(file_object) diff --git a/pymisp/tools/objectgenerator.py b/pymisp/tools/objectgenerator.py index 086b583..970c877 100644 --- a/pymisp/tools/objectgenerator.py +++ b/pymisp/tools/objectgenerator.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from pymisp import MISPEvent, MISPAttribute +from pymisp import MISPEvent, MISPAttribute, AbstractMISP import os import json import uuid @@ -24,9 +24,17 @@ if six.PY2: warnings.warn("You're using python 2, it is strongly recommended to use python >=3.4") -@six.add_metaclass(abc.ABCMeta) # Remove that line when discarding python2 support. -# Python3 way: class MISPObjectGenerator(metaclass=abc.ABCMeta): -class MISPObjectGenerator(): +class MISPObjectReference(AbstractMISP): + + attributes = ['uuid', 'relationship_type', 'comment'] + + def __init__(self, uuid, relationship_type, comment=None): + self['uuid'] = uuid + self['relationship_type'] = relationship_type + self['comment'] = comment + + +class MISPObjectGenerator(AbstractMISP): def __init__(self, template_dir): """This class is used to fill a new MISP object with the default values defined in the object template @@ -38,46 +46,46 @@ class MISPObjectGenerator(): 'data', 'misp-objects', 'objects') with open(os.path.join(self.misp_objects_path, template_dir, 'definition.json'), 'r') as f: self.definition = json.load(f) + self.attributes = self.definition['attributes'].keys() self.misp_event = MISPEvent() self.uuid = str(uuid.uuid4()) - self.links = [] + self.references = [] - def _fill_object(self, values, strict=True): + def _create_attribute(self, object_type, **value): + if value.get('value') is None: + return None + # Initialize the new MISPAttribute + # Get the misp attribute type from the definition + value['type'] = self.definition['attributes'][object_type]['misp-attribute'] + if value.get('disable_correlation') is None: + # The correlation can be disabled by default in the object definition. + # Use this value if it isn't overloaded by the object + value['disable_correlation'] = self.definition['attributes'][object_type].get('disable_correlation') + if value.get('to_ids') is None: + # Same for the to_ids flag + value['to_ids'] = self.definition['attributes'][object_type].get('to_ids') + # Set all the values in the MISP attribute + attribute = MISPAttribute(self.misp_event.describe_types) + attribute.set_all_values(**value) + self[object_type] = attribute + + def dump(self, strict=True): """Create a new object with the values gathered by the sub-class, use the default values from the template if needed""" if strict: - self._validate(values) + self._validate() # Create an empty object based om the object definition new_object = self.__new_empty_object(self.definition) - if self.links: - # Set the links to other objects - new_object["ObjectReference"] = [] - for link in self.links: - uuid, comment = link - new_object['ObjectReference'].append({'referenced_object_uuid': uuid, 'comment': comment}) - for object_type, value in values.items(): + for object_type, attribute in self.items(): # Add all the values as MISPAttributes to the current object - if value.get('value') is None: + if attribute.value is None: continue - # Initialize the new MISPAttribute - attribute = MISPAttribute(self.misp_event.describe_types) - # Get the misp attribute type from the definition - value['type'] = self.definition['attributes'][object_type]['misp-attribute'] - if value.get('disable_correlation') is None: - # The correlation can be disabled by default in the object definition. - # Use this value if it isn't overloaded by the object - value['disable_correlation'] = self.definition['attributes'][object_type].get('disable_correlation') - if value.get('to_ids') is None: - # Same for the to_ids flag - value['to_ids'] = self.definition['attributes'][object_type].get('to_ids') - # Set all the values in the MISP attribute - attribute.set_all_values(**value) # Finalize the actual MISP Object - new_object['Attribute'].append({'type': object_type, 'Attribute': attribute._json()}) - return new_object + new_object['Attribute'].append({'object_relation': object_type, **attribute._json()}) + return new_object, [r.to_dict() for r in self.references] - def _validate(self, dump): + def _validate(self): """Make sure the object we're creating has the required fields""" - all_attribute_names = set(dump.keys()) + all_attribute_names = set(self.keys()) if self.definition.get('requiredOneOf'): if not set(self.definition['requiredOneOf']) & all_attribute_names: raise InvalidMISPObject('At least one of the following attributes is required: {}'.format(', '.join(self.definition['requiredOneOf']))) @@ -87,9 +95,9 @@ class MISPObjectGenerator(): raise InvalidMISPObject('{} is required is required'.format(r)) return True - def add_link(self, uuid, comment=None): + def add_reference(self, uuid, relationship_type, comment=None): """Add a link (uuid) to an other object""" - self.links.append((uuid, comment)) + self.references.append(MISPObjectReference(uuid, relationship_type, comment)) def __new_empty_object(self, object_definiton): """Create a new empty object out of the template""" @@ -101,10 +109,3 @@ class MISPObjectGenerator(): def generate_attributes(self): """Contains the logic where all the values of the object are gathered""" pass - - @abc.abstractmethod - def dump(self): - """This method normalize the attributes to add to the object. - It returns an python dictionary where the key is the type defined in the object, - and the value the value of the MISP Attribute""" - pass diff --git a/pymisp/tools/peobject.py b/pymisp/tools/peobject.py index a72062b..3a86924 100644 --- a/pymisp/tools/peobject.py +++ b/pymisp/tools/peobject.py @@ -60,18 +60,21 @@ class PEObject(MISPObjectGenerator): return True return False - def generate_attributes(self): + def _get_pe_type(self): if self._is_dll(): - self.pe_type = 'dll' + return 'dll' elif self._is_driver(): - self.pe_type = 'driver' + return 'driver' elif self._is_exe(): - self.pe_type = 'exe' + return 'exe' else: - self.pe_type = 'unknown' + return 'unknown' + + def generate_attributes(self): + self._create_attribute('type', value=self._get_pe_type()) # General information - self.entrypoint_address = self.pe.entrypoint - self.compilation_timestamp = datetime.utcfromtimestamp(self.pe.header.time_date_stamps).isoformat() + self._create_attribute('entrypoint-address', value=self.pe.entrypoint) + self._create_attribute('compilation-timestamp', value=datetime.utcfromtimestamp(self.pe.header.time_date_stamps).isoformat()) # self.imphash = self.pe.get_imphash() try: if (self.pe.has_resources and @@ -79,15 +82,15 @@ class PEObject(MISPObjectGenerator): self.pe.resources_manager.version.has_string_file_info and self.pe.resources_manager.version.string_file_info.langcode_items): fileinfo = dict(self.pe.resources_manager.version.string_file_info.langcode_items[0].items.items()) - self.original_filename = fileinfo.get('OriginalFilename') - self.internal_filename = fileinfo.get('InternalName') - self.file_description = fileinfo.get('FileDescription') - self.file_version = fileinfo.get('FileVersion') - self.lang_id = self.pe.resources_manager.version.string_file_info.langcode_items[0].key - self.product_name = fileinfo.get('ProductName') - self.product_version = fileinfo.get('ProductVersion') - self.company_name = fileinfo.get('CompanyName') - self.legal_copyright = fileinfo.get('LegalCopyright') + self._create_attribute('original-filename', value=fileinfo.get('OriginalFilename')) + self._create_attribute('internal-filename', value=fileinfo.get('InternalName')) + self._create_attribute('file-description', value=fileinfo.get('FileDescription')) + self._create_attribute('file-version', value=fileinfo.get('FileVersion')) + self._create_attribute('lang-id', value=self.pe.resources_manager.version.string_file_info.langcode_items[0].key) + self._create_attribute('product-name', value=fileinfo.get('ProductName')) + self._create_attribute('product-version', value=fileinfo.get('ProductVersion')) + self._create_attribute('company-name', value=fileinfo.get('CompanyName')) + self._create_attribute('legal-copyright', value=fileinfo.get('LegalCopyright')) except lief.read_out_of_bound: # The file is corrupted pass @@ -97,46 +100,15 @@ class PEObject(MISPObjectGenerator): pos = 0 for section in self.pe.sections: s = PESectionObject(section) - self.add_link(s.uuid, 'Section {} of PE'.format(pos)) - if ((self.entrypoint_address >= section.virtual_address) and - (self.entrypoint_address < (section.virtual_address + section.virtual_size))): - self.entrypoint_section = (section.name, pos) # Tuple: (section_name, position) + self.add_reference(s.uuid, 'included-in', 'Section {} of PE'.format(pos)) + if ((self.pe.entrypoint >= section.virtual_address) and + (self.pe.entrypoint < (section.virtual_address + section.virtual_size))): + self._create_attribute('entrypoint-section|position', value='{}|{}'.format(section.name, pos)) pos += 1 self.sections.append(s) - self.nb_sections = len(self.sections) + self._create_attribute('number-sections', value=len(self.sections)) # TODO: TLSSection / DIRECTORY_ENTRY_TLS - def dump(self): - pe_object = {} - pe_object['type'] = {'value': self.pe_type} - if hasattr(self, 'imphash'): - pe_object['imphash'] = {'value': self.imphash} - if hasattr(self, 'original_filename'): - pe_object['original-filename'] = {'value': self.original_filename} - if hasattr(self, 'internal_filename'): - pe_object['internal-filename'] = {'value': self.internal_filename} - if hasattr(self, 'compilation_timestamp'): - pe_object['compilation-timestamp'] = {'value': self.compilation_timestamp} - if hasattr(self, 'entrypoint_section'): - pe_object['entrypoint-section|position'] = {'value': '{}|{}'.format(*self.entrypoint_section)} - if hasattr(self, 'entrypoint_address'): - pe_object['entrypoint-address'] = {'value': self.entrypoint_address} - if hasattr(self, 'file_description'): - pe_object['file-description'] = {'value': self.file_description} - if hasattr(self, 'file_version'): - pe_object['file-version'] = {'value': self.file_version} - if hasattr(self, 'lang_id'): - pe_object['lang-id'] = {'value': self.lang_id} - if hasattr(self, 'product_name'): - pe_object['product-name'] = {'value': self.product_name} - if hasattr(self, 'product_version'): - pe_object['product-version'] = {'value': self.product_version} - if hasattr(self, 'company_name'): - pe_object['company-name'] = {'value': self.company_name} - if hasattr(self, 'nb_sections'): - pe_object['number-sections'] = {'value': self.nb_sections} - return self._fill_object(pe_object) - class PESectionObject(MISPObjectGenerator): @@ -147,26 +119,13 @@ class PESectionObject(MISPObjectGenerator): self.generate_attributes() def generate_attributes(self): - self.name = self.section.name - self.size = self.section.size - if self.size > 0: - self.entropy = self.section.entropy - self.md5 = md5(self.data).hexdigest() - self.sha1 = sha1(self.data).hexdigest() - self.sha256 = sha256(self.data).hexdigest() - self.sha512 = sha512(self.data).hexdigest() + self._create_attribute('name', value=self.section.name) + self._create_attribute('size-in-bytes', value=self.section.size) + if getattr(self, 'size-in-bytes').value > 0: + self._create_attribute('entropy', value=self.section.entropy) + self._create_attribute('md5', value=md5(self.data).hexdigest()) + self._create_attribute('sha1', value=sha1(self.data).hexdigest()) + self._create_attribute('sha256', value=sha256(self.data).hexdigest()) + self._create_attribute('sha512', value=sha512(self.data).hexdigest()) if HAS_PYDEEP: - self.ssdeep = pydeep.hash_buf(self.data).decode() - - def dump(self): - section = {} - section['name'] = {'value': self.name} - section['size-in-bytes'] = {'value': self.size} - if self.size > 0: - section['entropy'] = {'value': self.entropy} - section['md5'] = {'value': self.md5} - section['sha1'] = {'value': self.sha1} - section['sha256'] = {'value': self.sha256} - section['sha512'] = {'value': self.sha512} - section['ssdeep'] = {'value': self.ssdeep} - return self._fill_object(section) + self._create_attribute('ssdeep', value=pydeep.hash_buf(self.data).decode()) diff --git a/setup.py b/setup.py index ad1f928..67f86fd 100644 --- a/setup.py +++ b/setup.py @@ -29,6 +29,8 @@ setup( test_suite="tests", install_requires=['requests', 'python-dateutil', 'jsonschema'], include_package_data=True, - package_data={'pymisp': ['data/*.json', 'data/misp-objects/schema.json', - 'data/misp-objects/objects/*/definition.json']}, + package_data={'pymisp': ['data/*.json', 'data/misp-objects/schema_objects.json', + 'data/misp-objects/schema_relationships.json', + 'data/misp-objects/objects/*/definition.json', + 'data/misp-objects/relationships/definition.json']}, ) From 314f7eaed109e6c3888d78c27d1d8545c87f34ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 23 Aug 2017 16:33:07 +0200 Subject: [PATCH 06/29] Re-enable python < 3.5 support. --- pymisp/api.py | 2 +- pymisp/mispevent.py | 2 +- pymisp/tools/objectgenerator.py | 11 +++++++++-- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/pymisp/api.py b/pymisp/api.py index ecbd62d..8f898d5 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -18,7 +18,7 @@ try: from urllib.parse import urljoin except ImportError: from urlparse import urljoin - warnings.warn("You're using python 2, it is strongly recommended to use python >=3.4") + warnings.warn("You're using python 2, it is strongly recommended to use python >=3.5") from io import BytesIO, open import zipfile diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 2bbbe3f..61c3b27 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -43,7 +43,7 @@ from .exceptions import PyMISPError, NewEventError, NewAttributeError try: basestring unicode - warnings.warn("You're using python 2, it is strongly recommended to use python >=3.4") + warnings.warn("You're using python 2, it is strongly recommended to use python >=3.5") except NameError: basestring = str unicode = str diff --git a/pymisp/tools/objectgenerator.py b/pymisp/tools/objectgenerator.py index 970c877..72b5ba0 100644 --- a/pymisp/tools/objectgenerator.py +++ b/pymisp/tools/objectgenerator.py @@ -21,7 +21,7 @@ class InvalidMISPObject(MISPObjectException): if six.PY2: import warnings - warnings.warn("You're using python 2, it is strongly recommended to use python >=3.4") + warnings.warn("You're using python 2, it is strongly recommended to use python >=3.5") class MISPObjectReference(AbstractMISP): @@ -80,7 +80,14 @@ class MISPObjectGenerator(AbstractMISP): if attribute.value is None: continue # Finalize the actual MISP Object - new_object['Attribute'].append({'object_relation': object_type, **attribute._json()}) + # FIXME: This only works on python >= 3.5 + # new_object['Attribute'].append({'object_relation': object_type, **attribute._json()}) + # ### BEGIN #### + # Because we still need to support old python. + temp_attribute = {'object_relation': object_type} + temp_attribute.update(attribute._json()) + new_object['Attribute'].append(temp_attribute) + # ### END #### return new_object, [r.to_dict() for r in self.references] def _validate(self): From e567ddc8776cee0519f6dc96cd16063d47555248 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 24 Aug 2017 17:09:16 +0200 Subject: [PATCH 07/29] Update function names in mispevent --- pymisp/mispevent.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 61c3b27..9d3fb63 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -221,6 +221,10 @@ class MISPAttribute(object): self._prepare_new_malware_sample() def _json(self): + # DEPRECATED + return self.to_dict() + + def to_dict(self): to_return = {'type': self.type, 'category': self.category, 'to_ids': self.to_ids, 'distribution': self.distribution, 'value': self.value, 'comment': self.comment, 'disable_correlation': self.disable_correlation} @@ -507,6 +511,10 @@ class MISPEvent(AbstractMISP): self.global_sig = kwargs['global_sig'] def _json(self): + # DEPTECATED + return self.to_dict() + + def to_dict(self): to_return = {'Event': {}} to_return['Event'] = {'distribution': self.distribution, 'info': self.info, 'date': self.date.isoformat(), 'published': self.published, From bd6deba55e581db2d734180575cf2dd1864d56fc Mon Sep 17 00:00:00 2001 From: edhoedt Date: Thu, 27 Jul 2017 14:54:02 +0200 Subject: [PATCH 08/29] Fixing undefined ressources_path --- pymisp/mispevent.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 9d3fb63..d71be70 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -53,6 +53,7 @@ class MISPAttribute(object): def __init__(self, describe_types=None): if not describe_types: + self.ressources_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data') with open(os.path.join(self.ressources_path, 'describeTypes.json'), 'r') as f: t = json.load(f) describe_types = t['result'] From c09ce0032c16fa5ad7962dc4afe963ac83dd413a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 24 Aug 2017 19:21:52 +0200 Subject: [PATCH 09/29] Refactor all the things Add script for MISP core, make everything generic. --- examples/add_file_object.py | 24 +++---- examples/generate_file_objects.py | 63 +++++++++++++++++ pymisp/__init__.py | 2 +- pymisp/abstract.py | 14 +++- pymisp/api.py | 8 +-- pymisp/tools/create_misp_object.py | 10 +-- pymisp/tools/fileobject.py | 17 +++-- pymisp/tools/objectgenerator.py | 110 +++++++++++++++++------------ pymisp/tools/peobject.py | 7 +- 9 files changed, 173 insertions(+), 82 deletions(-) create mode 100755 examples/generate_file_objects.py diff --git a/examples/add_file_object.py b/examples/add_file_object.py index 7780ab9..e61d809 100755 --- a/examples/add_file_object.py +++ b/examples/add_file_object.py @@ -7,7 +7,6 @@ import traceback from keys import misp_url, misp_key, misp_verifycert import glob import argparse -import json if __name__ == '__main__': parser = argparse.ArgumentParser(description='Extract indicators out of binaries and add MISP objects to a MISP instance.') @@ -25,20 +24,17 @@ if __name__ == '__main__': if seos: for s in seos: - obj, refs = s - template_id = pymisp.get_object_template_id(obj['name']) - r = pymisp.add_object(args.event, template_id, obj) + template_id = pymisp.get_object_template_id(s['name']) + r = pymisp.add_object(args.event, template_id, s) if peo: - obj, refs = peo - template_id = pymisp.get_object_template_id(obj['name']) - r = pymisp.add_object(args.event, template_id, obj) - for ref in refs: - r = pymisp.add_object_reference(obj['uuid'], ref) + template_id = pymisp.get_object_template_id(peo['name']) + r = pymisp.add_object(args.event, template_id, peo) + for ref in peo.references: + r = pymisp.add_object_reference(ref) if fo: - obj, refs = fo - template_id = pymisp.get_object_template_id(obj['name']) - response = pymisp.add_object(args.event, template_id, obj) - for ref in refs: - r = pymisp.add_object_reference(obj['uuid'], ref) + template_id = pymisp.get_object_template_id(fo['name']) + response = pymisp.add_object(args.event, template_id, fo) + for ref in fo.references: + r = pymisp.add_object_reference(ref) diff --git a/examples/generate_file_objects.py b/examples/generate_file_objects.py new file mode 100755 index 0000000..5b9ad33 --- /dev/null +++ b/examples/generate_file_objects.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +from pymisp import MISPEncode +from pymisp.tools import make_binary_objects +import argparse +import json + + +def check(): + missing_dependencies = {'pydeep': False, 'lief': False, 'magic': False, 'pymisp': False} + try: + import pymisp # noqa + except ImportError: + missing_dependencies['pymisp'] = 'Please install pydeep: pip install pymisp' + try: + import pydeep # noqa + except ImportError: + missing_dependencies['pydeep'] = 'Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git' + try: + import lief # noqa + except ImportError: + missing_dependencies['lief'] = 'Please install lief, documentation here: https://github.com/lief-project/LIEF' + try: + import magic # noqa + except ImportError: + missing_dependencies['magic'] = 'Please install python-magic: pip install python-magic.' + return json.dumps(missing_dependencies) + + +def make_objects(path): + to_return = {'objects': [], 'references': []} + fo, peo, seos = make_binary_objects(path) + + if seos: + for s in seos: + to_return['objects'].append(s) + if s.references: + to_return['references'] += s.references + + if peo: + to_return['objects'].append(peo) + if peo.references: + to_return['references'] += peo.references + + if fo: + to_return['objects'].append(fo) + if fo.references: + to_return['references'] += fo.references + return json.dumps(to_return, cls=MISPEncode) + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Extract indicators out of binaries and returns MISP objects.') + group = parser.add_mutually_exclusive_group() + group.add_argument("-p", "--path", help="Path to process.") + group.add_argument("-c", "--check", action='store_true', help="Check the dependencies.") + args = parser.parse_args() + + if args.check: + print(check()) + if args.path: + obj = make_objects(args.path) + print(obj) diff --git a/pymisp/__init__.py b/pymisp/__init__.py index 27967ec..9d3e949 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -2,7 +2,7 @@ __version__ = '2.4.77' from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey from .api import PyMISP -from .abstract import AbstractMISP +from .abstract import AbstractMISP, MISPEncode from .mispevent import MISPEvent, MISPAttribute, EncodeUpdate, EncodeFull from .tools import Neo4j from .tools import stix diff --git a/pymisp/abstract.py b/pymisp/abstract.py index ceea59b..e2bf76e 100644 --- a/pymisp/abstract.py +++ b/pymisp/abstract.py @@ -4,9 +4,18 @@ import six # Remove that import when discarding python2 support. import abc import json +from json import JSONEncoder import collections +class MISPEncode(JSONEncoder): + + def default(self, obj): + if isinstance(obj, AbstractMISP): + return obj.jsonable() + return JSONEncoder.default(self, obj) + + @six.add_metaclass(abc.ABCMeta) # Remove that line when discarding python2 support. # Python3 way: class MISPObjectGenerator(metaclass=abc.ABCMeta): class AbstractMISP(collections.MutableMapping): @@ -48,8 +57,11 @@ class AbstractMISP(collections.MutableMapping): to_return[attribute] = val return to_return + def jsonable(self): + return self.to_dict() + def to_json(self): - return json.dumps(self.to_dict(), indent=4, sort_keys=True) + return json.dumps(self.to_dict(), cls=MISPEncode) def __getitem__(self, key): if self.__check_dict_key(key): diff --git a/pymisp/api.py b/pymisp/api.py index 8f898d5..4638726 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -1585,13 +1585,13 @@ class PyMISP(object): def add_object(self, event_id, template_id, misp_object): session = self.__prepare_session() url = urljoin(self.root_url, 'objects/add/{}/{}'.format(event_id, template_id)) - response = session.post(url, data=json.dumps(misp_object)) + response = session.post(url, data=misp_object.to_json()) return self._check_response(response) - def add_object_reference(self, parent_uuid, misp_object_reference): + def add_object_reference(self, misp_object_reference): session = self.__prepare_session() - url = urljoin(self.root_url, 'object_references/add/{}'.format(parent_uuid)) - response = session.post(url, data=json.dumps(misp_object_reference)) + url = urljoin(self.root_url, 'object_references/add') + response = session.post(url, data=misp_object_reference.to_json()) return self._check_response(response) def get_object_templates_list(self): diff --git a/pymisp/tools/create_misp_object.py b/pymisp/tools/create_misp_object.py index 4d4df6e..a69bfdb 100644 --- a/pymisp/tools/create_misp_object.py +++ b/pymisp/tools/create_misp_object.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python # -*- coding: utf-8 -*- from pymisp.tools import FileObject, PEObject, MISPObjectException @@ -17,11 +17,11 @@ class FileTypeNotImplemented(MISPObjectException): def make_pe_objects(lief_parsed, misp_file): misp_pe = PEObject(parsed=lief_parsed) misp_file.add_reference(misp_pe.uuid, 'included-in', 'PE indicators') - file_object = misp_file.dump() - pe_object = misp_pe.dump() + file_object = misp_file + pe_object = misp_pe pe_sections = [] for s in misp_pe.sections: - pe_sections.append(s.dump()) + pe_sections.append(s) return file_object, pe_object, pe_sections @@ -45,5 +45,5 @@ def make_binary_objects(filepath): print('\tParser error: ', e) except FileTypeNotImplemented as e: print(e) - file_object = misp_file.dump() + file_object = misp_file.to_json() return file_object, None, None diff --git a/pymisp/tools/fileobject.py b/pymisp/tools/fileobject.py index f21cd2f..770307c 100644 --- a/pymisp/tools/fileobject.py +++ b/pymisp/tools/fileobject.py @@ -7,6 +7,7 @@ from io import BytesIO from hashlib import md5, sha1, sha256, sha512 import math from collections import Counter +import warnings try: import pydeep @@ -25,9 +26,9 @@ class FileObject(MISPObjectGenerator): def __init__(self, filepath=None, pseudofile=None, filename=None): if not HAS_PYDEEP: - raise ImportError("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git") + warnings.warn("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git") if not HAS_MAGIC: - raise ImportError("Please install python-magic: pip install python-magic.") + warnings.warn("Please install python-magic: pip install python-magic.") if filepath: self.filepath = filepath self.filename = os.path.basename(self.filepath) @@ -46,16 +47,18 @@ class FileObject(MISPObjectGenerator): def generate_attributes(self): self._create_attribute('filename', value=self.filename) - self._create_attribute('size-in-bytes', value=len(self.data)) - if getattr(self, 'size-in-bytes').value > 0: + size = self._create_attribute('size-in-bytes', value=len(self.data)) + if int(size.value) > 0: self._create_attribute('entropy', value=self.__entropy_H(self.data)) self._create_attribute('md5', value=md5(self.data).hexdigest()) self._create_attribute('sha1', value=sha1(self.data).hexdigest()) self._create_attribute('sha256', value=sha256(self.data).hexdigest()) self._create_attribute('sha512', value=sha512(self.data).hexdigest()) - self._create_attribute('mimetype', value=magic.from_buffer(self.data)) - self._create_attribute('ssdeep', value=pydeep.hash_buf(self.data).decode()) - self._create_attribute('malware-sample', value=self.filename.value, data=self.pseudofile) + self._create_attribute('malware-sample', value=self.filename, data=self.pseudofile) + if HAS_MAGIC: + self._create_attribute('mimetype', value=magic.from_buffer(self.data)) + if HAS_PYDEEP: + self._create_attribute('ssdeep', value=pydeep.hash_buf(self.data).decode()) def __entropy_H(self, data): """Calculate the entropy of a chunk of data.""" diff --git a/pymisp/tools/objectgenerator.py b/pymisp/tools/objectgenerator.py index 72b5ba0..1617580 100644 --- a/pymisp/tools/objectgenerator.py +++ b/pymisp/tools/objectgenerator.py @@ -26,16 +26,53 @@ if six.PY2: class MISPObjectReference(AbstractMISP): - attributes = ['uuid', 'relationship_type', 'comment'] + attributes = ['source_uuid', 'destination_uuid', 'relationship_type', 'comment'] - def __init__(self, uuid, relationship_type, comment=None): - self['uuid'] = uuid - self['relationship_type'] = relationship_type - self['comment'] = comment + def __init__(self, source_uuid, destination_uuid, relationship_type, comment=None): + self.source_uuid = source_uuid + self.destination_uuid = destination_uuid + self.relationship_type = relationship_type + self.comment = comment + + +class MISPObjectAttribute(AbstractMISP): + + # This list is very limited and hardcoded to fit the current needs (file/pe/pesection creation): MISPAttriute will follow the + # same spec and just add one attribute: object_relation + attributes = ['object_relation', 'value', 'type', 'category', 'disable_correlation', 'to_ids', + 'data', 'encrypt', 'distribution', 'comment'] + + def __init__(self, definition, object_relation, value, **kwargs): + self.object_relation = object_relation + self.value = value + # Initialize the new MISPAttribute + # Get the misp attribute type from the definition + self.type = kwargs.pop('type', None) + if self.type is None: + self.type = definition['misp-attribute'] + self.disable_correlation = kwargs.pop('disable_correlation', None) + if self.disable_correlation is None: + # The correlation can be disabled by default in the object definition. + # Use this value if it isn't overloaded by the object + self.disable_correlation = definition.get('disable_correlation') + self.to_ids = kwargs.pop('to_ids', None) + if self.to_ids is None: + # Same for the to_ids flag + self.to_ids = definition.get('to_ids') + # Initialise rest of the values + for k, v in kwargs.items(): + self[k] = v + # FIXME: dirty hack until all the classes are ported to the new format but we get the default values + temp_attribute = MISPAttribute() + temp_attribute.set_all_values(**self) + # Update default values + self.from_dict(**temp_attribute.to_dict()) class MISPObjectGenerator(AbstractMISP): + attributes = ['name', 'meta-category', 'uuid', 'description', 'version', 'Attribute'] + def __init__(self, template_dir): """This class is used to fill a new MISP object with the default values defined in the object template * template is the path to the template within the misp-object repository @@ -46,53 +83,38 @@ class MISPObjectGenerator(AbstractMISP): 'data', 'misp-objects', 'objects') with open(os.path.join(self.misp_objects_path, template_dir, 'definition.json'), 'r') as f: self.definition = json.load(f) - self.attributes = self.definition['attributes'].keys() + self.object_attributes = self.definition['attributes'].keys() self.misp_event = MISPEvent() + self.name = self.definition['name'] + setattr(self, 'meta-category', self.definition['meta-category']) self.uuid = str(uuid.uuid4()) + self.description = self.definition['description'] + self.version = self.definition['version'] + self.Attribute = [] self.references = [] def _create_attribute(self, object_type, **value): if value.get('value') is None: return None - # Initialize the new MISPAttribute - # Get the misp attribute type from the definition - value['type'] = self.definition['attributes'][object_type]['misp-attribute'] - if value.get('disable_correlation') is None: - # The correlation can be disabled by default in the object definition. - # Use this value if it isn't overloaded by the object - value['disable_correlation'] = self.definition['attributes'][object_type].get('disable_correlation') - if value.get('to_ids') is None: - # Same for the to_ids flag - value['to_ids'] = self.definition['attributes'][object_type].get('to_ids') - # Set all the values in the MISP attribute - attribute = MISPAttribute(self.misp_event.describe_types) - attribute.set_all_values(**value) - self[object_type] = attribute + attribute = MISPObjectAttribute(self.definition['attributes'][object_type], object_type, **value) + self.Attribute.append(attribute) + return attribute - def dump(self, strict=True): - """Create a new object with the values gathered by the sub-class, use the default values from the template if needed""" + def to_dict(self, strict=True): if strict: self._validate() - # Create an empty object based om the object definition - new_object = self.__new_empty_object(self.definition) - for object_type, attribute in self.items(): - # Add all the values as MISPAttributes to the current object - if attribute.value is None: - continue - # Finalize the actual MISP Object - # FIXME: This only works on python >= 3.5 - # new_object['Attribute'].append({'object_relation': object_type, **attribute._json()}) - # ### BEGIN #### - # Because we still need to support old python. - temp_attribute = {'object_relation': object_type} - temp_attribute.update(attribute._json()) - new_object['Attribute'].append(temp_attribute) - # ### END #### - return new_object, [r.to_dict() for r in self.references] + return super(MISPObjectGenerator, self).to_dict() + + def to_json(self, strict=True): + if strict: + self._validate() + return super(MISPObjectGenerator, self).to_json() def _validate(self): """Make sure the object we're creating has the required fields""" - all_attribute_names = set(self.keys()) + all_attribute_names = set() + for a in self.Attribute: + all_attribute_names.add(a.object_relation) if self.definition.get('requiredOneOf'): if not set(self.definition['requiredOneOf']) & all_attribute_names: raise InvalidMISPObject('At least one of the following attributes is required: {}'.format(', '.join(self.definition['requiredOneOf']))) @@ -102,15 +124,9 @@ class MISPObjectGenerator(AbstractMISP): raise InvalidMISPObject('{} is required is required'.format(r)) return True - def add_reference(self, uuid, relationship_type, comment=None): + def add_reference(self, destination_uuid, relationship_type, comment=None): """Add a link (uuid) to an other object""" - self.references.append(MISPObjectReference(uuid, relationship_type, comment)) - - def __new_empty_object(self, object_definiton): - """Create a new empty object out of the template""" - return {'name': object_definiton['name'], 'meta-category': object_definiton['meta-category'], - 'uuid': self.uuid, 'description': object_definiton['description'], - 'version': object_definiton['version'], 'Attribute': []} + self.references.append(MISPObjectReference(self.uuid, destination_uuid, relationship_type, comment)) @abc.abstractmethod def generate_attributes(self): diff --git a/pymisp/tools/peobject.py b/pymisp/tools/peobject.py index 3a86924..e7dc206 100644 --- a/pymisp/tools/peobject.py +++ b/pymisp/tools/peobject.py @@ -5,6 +5,7 @@ from pymisp.tools import MISPObjectGenerator from io import BytesIO from hashlib import md5, sha1, sha256, sha512 from datetime import datetime +import warnings try: @@ -24,7 +25,7 @@ class PEObject(MISPObjectGenerator): def __init__(self, parsed=None, filepath=None, pseudofile=None): if not HAS_PYDEEP: - raise ImportError("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git") + warnings.warn("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git") if not HAS_LIEF: raise ImportError('Please install lief, documentation here: https://github.com/lief-project/LIEF') if pseudofile: @@ -120,8 +121,8 @@ class PESectionObject(MISPObjectGenerator): def generate_attributes(self): self._create_attribute('name', value=self.section.name) - self._create_attribute('size-in-bytes', value=self.section.size) - if getattr(self, 'size-in-bytes').value > 0: + size = self._create_attribute('size-in-bytes', value=self.section.size) + if int(size.value) > 0: self._create_attribute('entropy', value=self.section.entropy) self._create_attribute('md5', value=md5(self.data).hexdigest()) self._create_attribute('sha1', value=sha1(self.data).hexdigest()) From f66af15c62eb5c1674c11257bb3068c3eb3d5508 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 25 Aug 2017 09:45:56 +0200 Subject: [PATCH 10/29] Update get_template_id, cleanup --- examples/add_file_object.py | 6 +++--- pymisp/api.py | 6 +++--- pymisp/tools/fileobject.py | 4 +++- pymisp/tools/objectgenerator.py | 6 +++--- pymisp/tools/peobject.py | 8 ++++++-- 5 files changed, 18 insertions(+), 12 deletions(-) diff --git a/examples/add_file_object.py b/examples/add_file_object.py index e61d809..aa8ab8e 100755 --- a/examples/add_file_object.py +++ b/examples/add_file_object.py @@ -24,17 +24,17 @@ if __name__ == '__main__': if seos: for s in seos: - template_id = pymisp.get_object_template_id(s['name']) + template_id = pymisp.get_object_template_id(s.template_uuid) r = pymisp.add_object(args.event, template_id, s) if peo: - template_id = pymisp.get_object_template_id(peo['name']) + template_id = pymisp.get_object_template_id(peo.template_uuid) r = pymisp.add_object(args.event, template_id, peo) for ref in peo.references: r = pymisp.add_object_reference(ref) if fo: - template_id = pymisp.get_object_template_id(fo['name']) + template_id = pymisp.get_object_template_id(fo.template_uuid) response = pymisp.add_object(args.event, template_id, fo) for ref in fo.references: r = pymisp.add_object_reference(ref) diff --git a/pymisp/api.py b/pymisp/api.py index 4638726..8d7856e 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -1600,12 +1600,12 @@ class PyMISP(object): response = session.get(url) return self._check_response(response)['response'] - def get_object_template_id(self, object_name): + def get_object_template_id(self, object_uuid): templates = self.get_object_templates_list() for t in templates: - if t['ObjectTemplate']['name'] == object_name: + if t['ObjectTemplate']['uuid'] == object_uuid: return t['ObjectTemplate']['id'] - raise Exception('Unable to find template name {} on the MISP instance'.format(object_name)) + raise Exception('Unable to find template uuid {} on the MISP instance'.format(object_uuid)) # ########################### # ####### Deprecated ######## diff --git a/pymisp/tools/fileobject.py b/pymisp/tools/fileobject.py index 770307c..ebea362 100644 --- a/pymisp/tools/fileobject.py +++ b/pymisp/tools/fileobject.py @@ -41,7 +41,9 @@ class FileObject(MISPObjectGenerator): self.filename = filename else: raise Exception('File buffer (BytesIO) or a path is required.') - MISPObjectGenerator.__init__(self, 'file') + # PY3 way: + # super().__init__('file') + super(FileObject, self).__init__('file') self.data = self.pseudofile.getvalue() self.generate_attributes() diff --git a/pymisp/tools/objectgenerator.py b/pymisp/tools/objectgenerator.py index 1617580..d4b5105 100644 --- a/pymisp/tools/objectgenerator.py +++ b/pymisp/tools/objectgenerator.py @@ -83,13 +83,13 @@ class MISPObjectGenerator(AbstractMISP): 'data', 'misp-objects', 'objects') with open(os.path.join(self.misp_objects_path, template_dir, 'definition.json'), 'r') as f: self.definition = json.load(f) - self.object_attributes = self.definition['attributes'].keys() self.misp_event = MISPEvent() self.name = self.definition['name'] setattr(self, 'meta-category', self.definition['meta-category']) - self.uuid = str(uuid.uuid4()) + self.template_uuid = self.definition['uuid'] self.description = self.definition['description'] self.version = self.definition['version'] + self.uuid = str(uuid.uuid4()) self.Attribute = [] self.references = [] @@ -121,7 +121,7 @@ class MISPObjectGenerator(AbstractMISP): if self.definition.get('required'): for r in self.definition.get('required'): if r not in all_attribute_names: - raise InvalidMISPObject('{} is required is required'.format(r)) + raise InvalidMISPObject('{} is required'.format(r)) return True def add_reference(self, destination_uuid, relationship_type, comment=None): diff --git a/pymisp/tools/peobject.py b/pymisp/tools/peobject.py index e7dc206..bea117a 100644 --- a/pymisp/tools/peobject.py +++ b/pymisp/tools/peobject.py @@ -43,7 +43,9 @@ class PEObject(MISPObjectGenerator): self.pe = parsed else: raise Exception('Not a lief.PE.Binary: {}'.format(type(parsed))) - MISPObjectGenerator.__init__(self, 'pe') + # Python3 way + # super().__init__('pe') + super(PEObject, self).__init__('pe') self.generate_attributes() def _is_exe(self): @@ -114,7 +116,9 @@ class PEObject(MISPObjectGenerator): class PESectionObject(MISPObjectGenerator): def __init__(self, section): - MISPObjectGenerator.__init__(self, 'pe-section') + # Python3 way + # super().__init__('pe-section') + super(PESectionObject, self).__init__('pe-section') self.section = section self.data = bytes(self.section.content) self.generate_attributes() From 44008d1c0c285e5871ea14f6638568dfc48624ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 25 Aug 2017 15:57:12 +0200 Subject: [PATCH 11/29] Add support for ELF and MachO objects --- pymisp/data/misp-objects | 2 +- pymisp/tools/__init__.py | 12 ++++--- pymisp/tools/create_misp_object.py | 53 +++++++++++++++++++++--------- 3 files changed, 45 insertions(+), 22 deletions(-) diff --git a/pymisp/data/misp-objects b/pymisp/data/misp-objects index 96d7aeb..fc6d54f 160000 --- a/pymisp/data/misp-objects +++ b/pymisp/data/misp-objects @@ -1 +1 @@ -Subproject commit 96d7aeb0729428a43f38f45b6c00a60b9fdba2b6 +Subproject commit fc6d54f93b47d9c2ca951d68e568d1715e6eabf0 diff --git a/pymisp/tools/__init__.py b/pymisp/tools/__init__.py index dc1748c..2a81b2a 100644 --- a/pymisp/tools/__init__.py +++ b/pymisp/tools/__init__.py @@ -1,5 +1,7 @@ -from .neo4j import Neo4j -from .objectgenerator import MISPObjectGenerator, MISPObjectException, InvalidMISPObject -from .fileobject import FileObject -from .peobject import PEObject, PESectionObject -from .create_misp_object import make_binary_objects +from .neo4j import Neo4j # noqa +from .objectgenerator import MISPObjectGenerator, MISPObjectException, InvalidMISPObject # noqa +from .fileobject import FileObject # noqa +from .peobject import PEObject, PESectionObject # noqa +from .elfobject import ELFObject, ELFSectionObject # noqa +from .machoobject import MachOObject, MachOSectionObject # noqa +from .create_misp_object import make_binary_objects # noqa diff --git a/pymisp/tools/create_misp_object.py b/pymisp/tools/create_misp_object.py index a69bfdb..a77fb8c 100644 --- a/pymisp/tools/create_misp_object.py +++ b/pymisp/tools/create_misp_object.py @@ -1,10 +1,12 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from pymisp.tools import FileObject, PEObject, MISPObjectException +from pymisp.tools import FileObject, PEObject, ELFObject, MachOObject, MISPObjectException try: import lief + from lief import Logger + Logger.disable() HAS_LIEF = True except ImportError: HAS_LIEF = False @@ -15,14 +17,30 @@ class FileTypeNotImplemented(MISPObjectException): def make_pe_objects(lief_parsed, misp_file): - misp_pe = PEObject(parsed=lief_parsed) - misp_file.add_reference(misp_pe.uuid, 'included-in', 'PE indicators') - file_object = misp_file - pe_object = misp_pe + pe_object = PEObject(parsed=lief_parsed) + misp_file.add_reference(pe_object.uuid, 'included-in', 'PE indicators') pe_sections = [] - for s in misp_pe.sections: + for s in pe_object.sections: pe_sections.append(s) - return file_object, pe_object, pe_sections + return misp_file, pe_object, pe_sections + + +def make_elf_objects(lief_parsed, misp_file): + elf_object = ELFObject(parsed=lief_parsed) + misp_file.add_reference(elf_object.uuid, 'included-in', 'ELF indicators') + elf_sections = [] + for s in elf_object.sections: + elf_sections.append(s) + return misp_file, elf_object, elf_sections + + +def make_macho_objects(lief_parsed, misp_file): + macho_object = MachOObject(parsed=lief_parsed) + misp_file.add_reference(macho_object.uuid, 'included-in', 'MachO indicators') + macho_sections = [] + for s in macho_object.sections: + macho_sections.append(s) + return misp_file, macho_object, macho_sections def make_binary_objects(filepath): @@ -34,16 +52,19 @@ def make_binary_objects(filepath): if isinstance(lief_parsed, lief.PE.Binary): return make_pe_objects(lief_parsed, misp_file) elif isinstance(lief_parsed, lief.ELF.Binary): - raise FileTypeNotImplemented('ELF not implemented yet.') + return make_elf_objects(lief_parsed, misp_file) elif isinstance(lief_parsed, lief.MachO.Binary): - raise FileTypeNotImplemented('MachO not implemented yet.') + return make_macho_objects(lief_parsed, misp_file) except lief.bad_format as e: - print('\tBad format: ', e) + # print('\tBad format: ', e) + pass except lief.bad_file as e: - print('\tBad file: ', e) + # print('\tBad file: ', e) + pass except lief.parser_error as e: - print('\tParser error: ', e) - except FileTypeNotImplemented as e: - print(e) - file_object = misp_file.to_json() - return file_object, None, None + # print('\tParser error: ', e) + pass + except FileTypeNotImplemented as e: # noqa + # print(e) + pass + return misp_file, None, None From f06bfd310bb8652279e73118ec3f00ab07297668 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 25 Aug 2017 16:08:05 +0200 Subject: [PATCH 12/29] Do not fail if pymisp is not installed --- examples/generate_file_objects.py | 7 ++- pymisp/__init__.py | 17 +++--- pymisp/abstract.py | 2 +- pymisp/tools/elfobject.py | 91 +++++++++++++++++++++++++++++++ pymisp/tools/machoobject.py | 88 ++++++++++++++++++++++++++++++ setup.py | 2 +- 6 files changed, 196 insertions(+), 11 deletions(-) create mode 100644 pymisp/tools/elfobject.py create mode 100644 pymisp/tools/machoobject.py diff --git a/examples/generate_file_objects.py b/examples/generate_file_objects.py index 5b9ad33..5ff6bb6 100755 --- a/examples/generate_file_objects.py +++ b/examples/generate_file_objects.py @@ -1,11 +1,14 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -from pymisp import MISPEncode -from pymisp.tools import make_binary_objects import argparse import json +try: + from pymisp import MISPEncode + from pymisp.tools import make_binary_objects +except ImportError: + pass def check(): missing_dependencies = {'pydeep': False, 'lief': False, 'magic': False, 'pymisp': False} diff --git a/pymisp/__init__.py b/pymisp/__init__.py index 9d3e949..532dc15 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -1,9 +1,12 @@ __version__ = '2.4.77' -from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey -from .api import PyMISP -from .abstract import AbstractMISP, MISPEncode -from .mispevent import MISPEvent, MISPAttribute, EncodeUpdate, EncodeFull -from .tools import Neo4j -from .tools import stix -from .tools import MISPObjectGenerator +try: + from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey + from .api import PyMISP + from .abstract import AbstractMISP, MISPEncode + from .mispevent import MISPEvent, MISPAttribute, EncodeUpdate, EncodeFull + from .tools import Neo4j + from .tools import stix + from .tools import MISPObjectGenerator +except ImportError: + pass diff --git a/pymisp/abstract.py b/pymisp/abstract.py index e2bf76e..c1abca6 100644 --- a/pymisp/abstract.py +++ b/pymisp/abstract.py @@ -1,12 +1,12 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -import six # Remove that import when discarding python2 support. import abc import json from json import JSONEncoder import collections +import six # Remove that import when discarding python2 support. class MISPEncode(JSONEncoder): diff --git a/pymisp/tools/elfobject.py b/pymisp/tools/elfobject.py new file mode 100644 index 0000000..579d012 --- /dev/null +++ b/pymisp/tools/elfobject.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +from pymisp.tools import MISPObjectGenerator +from io import BytesIO +from hashlib import md5, sha1, sha256, sha512 +import warnings + + +try: + import lief + HAS_LIEF = True +except ImportError: + HAS_LIEF = False + +try: + import pydeep + HAS_PYDEEP = True +except ImportError: + HAS_PYDEEP = False + + +class ELFObject(MISPObjectGenerator): + + def __init__(self, parsed=None, filepath=None, pseudofile=None): + if not HAS_PYDEEP: + warnings.warn("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git") + if not HAS_LIEF: + raise ImportError('Please install lief, documentation here: https://github.com/lief-project/LIEF') + if pseudofile: + if isinstance(pseudofile, BytesIO): + self.elf = lief.ELF.parse(raw=pseudofile.getvalue()) + elif isinstance(pseudofile, bytes): + self.elf = lief.ELF.parse(raw=pseudofile) + else: + raise Exception('Pseudo file can be BytesIO or bytes got {}'.format(type(pseudofile))) + elif filepath: + self.elf = lief.ELF.parse(filepath) + elif parsed: + # Got an already parsed blob + if isinstance(parsed, lief.ELF.Binary): + self.elf = parsed + else: + raise Exception('Not a lief.ELF.Binary: {}'.format(type(parsed))) + # Python3 way + # super().__init__('elf') + super(ELFObject, self).__init__('elf') + self.generate_attributes() + + def generate_attributes(self): + # General information + self._create_attribute('type', value=str(self.elf.header.file_type).split('.')[1]) + self._create_attribute('entrypoint-address', value=self.elf.entrypoint) + self._create_attribute('arch', value=str(self.elf.header.machine_type).split('.')[1]) + self._create_attribute('os_abi', value=str(self.elf.header.identity_os_abi).split('.')[1]) + # Sections + self.sections = [] + if self.elf.sections: + pos = 0 + for section in self.elf.sections: + s = ELFSectionObject(section) + self.add_reference(s.uuid, 'included-in', 'Section {} of ELF'.format(pos)) + pos += 1 + self.sections.append(s) + self._create_attribute('number-sections', value=len(self.sections)) + + +class ELFSectionObject(MISPObjectGenerator): + + def __init__(self, section): + # Python3 way + # super().__init__('pe-section') + super(ELFSectionObject, self).__init__('elf-section') + self.section = section + self.data = bytes(self.section.content) + self.generate_attributes() + + def generate_attributes(self): + self._create_attribute('name', value=self.section.name) + self._create_attribute('type', value=str(self.section.type).split('.')[1]) + print(self.section.flags) + # self._create_attribute('flag', value=str(self.section.flags).split('.')[1]) + size = self._create_attribute('size-in-bytes', value=self.section.size) + if int(size.value) > 0: + self._create_attribute('entropy', value=self.section.entropy) + self._create_attribute('md5', value=md5(self.data).hexdigest()) + self._create_attribute('sha1', value=sha1(self.data).hexdigest()) + self._create_attribute('sha256', value=sha256(self.data).hexdigest()) + self._create_attribute('sha512', value=sha512(self.data).hexdigest()) + if HAS_PYDEEP: + self._create_attribute('ssdeep', value=pydeep.hash_buf(self.data).decode()) diff --git a/pymisp/tools/machoobject.py b/pymisp/tools/machoobject.py new file mode 100644 index 0000000..e688fba --- /dev/null +++ b/pymisp/tools/machoobject.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +from pymisp.tools import MISPObjectGenerator +from io import BytesIO +from hashlib import md5, sha1, sha256, sha512 +import warnings + + +try: + import lief + HAS_LIEF = True +except ImportError: + HAS_LIEF = False + +try: + import pydeep + HAS_PYDEEP = True +except ImportError: + HAS_PYDEEP = False + + +class MachOObject(MISPObjectGenerator): + + def __init__(self, parsed=None, filepath=None, pseudofile=None): + if not HAS_PYDEEP: + warnings.warn("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git") + if not HAS_LIEF: + raise ImportError('Please install lief, documentation here: https://github.com/lief-project/LIEF') + if pseudofile: + if isinstance(pseudofile, BytesIO): + self.macho = lief.MachO.parse(raw=pseudofile.getvalue()) + elif isinstance(pseudofile, bytes): + self.macho = lief.MachO.parse(raw=pseudofile) + else: + raise Exception('Pseudo file can be BytesIO or bytes got {}'.format(type(pseudofile))) + elif filepath: + self.macho = lief.MachO.parse(filepath) + elif parsed: + # Got an already parsed blob + if isinstance(parsed, lief.MachO.Binary): + self.macho = parsed + else: + raise Exception('Not a lief.MachO.Binary: {}'.format(type(parsed))) + # Python3 way + # super().__init__('elf') + super(MachOObject, self).__init__('macho') + self.generate_attributes() + + def generate_attributes(self): + self._create_attribute('type', value=str(self.macho.header.file_type).split('.')[1]) + self._create_attribute('name', value=self.macho.name) + # General information + if self.macho.has_entrypoint: + self._create_attribute('entrypoint-address', value=self.macho.entrypoint) + # Sections + self.sections = [] + if self.macho.sections: + pos = 0 + for section in self.macho.sections: + s = MachOSectionObject(section) + self.add_reference(s.uuid, 'included-in', 'Section {} of MachO'.format(pos)) + pos += 1 + self.sections.append(s) + self._create_attribute('number-sections', value=len(self.sections)) + + +class MachOSectionObject(MISPObjectGenerator): + + def __init__(self, section): + # Python3 way + # super().__init__('pe-section') + super(MachOSectionObject, self).__init__('macho-section') + self.section = section + self.data = bytes(self.section.content) + self.generate_attributes() + + def generate_attributes(self): + self._create_attribute('name', value=self.section.name) + size = self._create_attribute('size-in-bytes', value=self.section.size) + if int(size.value) > 0: + self._create_attribute('entropy', value=self.section.entropy) + self._create_attribute('md5', value=md5(self.data).hexdigest()) + self._create_attribute('sha1', value=sha1(self.data).hexdigest()) + self._create_attribute('sha256', value=sha256(self.data).hexdigest()) + self._create_attribute('sha512', value=sha512(self.data).hexdigest()) + if HAS_PYDEEP: + self._create_attribute('ssdeep', value=pydeep.hash_buf(self.data).decode()) diff --git a/setup.py b/setup.py index 67f86fd..553666f 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ setup( 'Topic :: Internet', ], test_suite="tests", - install_requires=['requests', 'python-dateutil', 'jsonschema'], + install_requires=['six', 'requests', 'python-dateutil', 'jsonschema'], include_package_data=True, package_data={'pymisp': ['data/*.json', 'data/misp-objects/schema_objects.json', 'data/misp-objects/schema_relationships.json', From 38a488c70c4fe19a0d0d6fafdf67074f54d980e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 25 Aug 2017 17:14:33 +0200 Subject: [PATCH 13/29] Update tests --- .travis.yml | 5 ++++- tests/test_offline.py | 41 ++++++++++++++++++++++++++++++++++++----- 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index ed53358..5c7bc11 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,8 +16,11 @@ install: - pip install -U nose - pip install coveralls - pip install codecov - - pip install requests-mock + - pip install requests-mock pytest - pip install . + - pushd tests + - git clone https://github.com/viper-framework/viper-test-files.git + - popd script: - nosetests --with-coverage --cover-package=pymisp tests/test_offline.py diff --git a/tests/test_offline.py b/tests/test_offline.py index 92f7a5c..14c5aad 100644 --- a/tests/test_offline.py +++ b/tests/test_offline.py @@ -13,6 +13,9 @@ from pymisp import MISPEvent from pymisp import EncodeUpdate from pymisp import EncodeFull +from pymisp import MISPEncode +from pymisp.tools import make_binary_objects + @requests_mock.Mocker() class TestOffline(unittest.TestCase): @@ -155,12 +158,12 @@ class TestOffline(unittest.TestCase): p = MockPyMISP(self.domain, self.key) evt = p.get(1) self.assertEqual(3, p.add_hashes(evt, md5='68b329da9893e34099c7d8ad5cb9c940', - sha1='adc83b19e793491b1c6ea0fd8b46cd9f32e592fc', - sha256='01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b', - filename='foobar.exe')) + sha1='adc83b19e793491b1c6ea0fd8b46cd9f32e592fc', + sha256='01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b', + filename='foobar.exe')) self.assertEqual(3, p.add_hashes(evt, md5='68b329da9893e34099c7d8ad5cb9c940', - sha1='adc83b19e793491b1c6ea0fd8b46cd9f32e592fc', - sha256='01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b')) + sha1='adc83b19e793491b1c6ea0fd8b46cd9f32e592fc', + sha256='01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b')) p.av_detection_link(evt, 'https://foocorp.com') p.add_detection_name(evt, 'WATERMELON') p.add_filename(evt, 'foobar.exe') @@ -220,5 +223,33 @@ class TestOffline(unittest.TestCase): p.add_internal_other(evt, 'foobar') p.add_attachment(evt, "testFile") + def make_objects(self, path): + to_return = {'objects': [], 'references': []} + fo, peo, seos = make_binary_objects(path) + + if seos: + for s in seos: + to_return['objects'].append(s) + if s.references: + to_return['references'] += s.references + + if peo: + to_return['objects'].append(peo) + if peo.references: + to_return['references'] += peo.references + + if fo: + to_return['objects'].append(fo) + if fo.references: + to_return['references'] += fo.references + return json.dumps(to_return, cls=MISPEncode) + + def test_objects(self, m): + paths = ['cmd.exe', 'tmux', 'MachO-OSX-x64-ls'] + for path in paths: + json_blob = self.make_objects(os.path.join('tests', + 'viper-test-files', 'test_files', path)) + print(json_blob) + if __name__ == '__main__': unittest.main() From 8e50ee825b93f3f19f7d70a2eee41b9a89165e95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 25 Aug 2017 17:23:50 +0200 Subject: [PATCH 14/29] Install deps in travis --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 5c7bc11..8a9168c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,6 +17,8 @@ install: - pip install coveralls - pip install codecov - pip install requests-mock pytest + - pip install https://github.com/lief-project/packages/raw/lief-master-latest/pylief-0.7.0.dev.zip + - pip install python-magic - pip install . - pushd tests - git clone https://github.com/viper-framework/viper-test-files.git From 9ff37397536410ae9200cd20b29f5ebad8f40131 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 25 Aug 2017 17:37:35 +0200 Subject: [PATCH 15/29] Remove ImportError --- pymisp/tools/create_misp_object.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pymisp/tools/create_misp_object.py b/pymisp/tools/create_misp_object.py index a77fb8c..8c9b304 100644 --- a/pymisp/tools/create_misp_object.py +++ b/pymisp/tools/create_misp_object.py @@ -44,8 +44,6 @@ def make_macho_objects(lief_parsed, misp_file): def make_binary_objects(filepath): - if not HAS_LIEF: - raise ImportError('Please install lief, documentation here: https://github.com/lief-project/LIEF') misp_file = FileObject(filepath) try: lief_parsed = lief.parse(filepath) From a5531990ae81f171cdb32cbdfad26282416ef90a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 25 Aug 2017 17:41:58 +0200 Subject: [PATCH 16/29] Do not try to run code requiring lief --- pymisp/tools/create_misp_object.py | 41 +++++++++++++++--------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/pymisp/tools/create_misp_object.py b/pymisp/tools/create_misp_object.py index 8c9b304..9e2d7fb 100644 --- a/pymisp/tools/create_misp_object.py +++ b/pymisp/tools/create_misp_object.py @@ -45,24 +45,25 @@ def make_macho_objects(lief_parsed, misp_file): def make_binary_objects(filepath): misp_file = FileObject(filepath) - try: - lief_parsed = lief.parse(filepath) - if isinstance(lief_parsed, lief.PE.Binary): - return make_pe_objects(lief_parsed, misp_file) - elif isinstance(lief_parsed, lief.ELF.Binary): - return make_elf_objects(lief_parsed, misp_file) - elif isinstance(lief_parsed, lief.MachO.Binary): - return make_macho_objects(lief_parsed, misp_file) - except lief.bad_format as e: - # print('\tBad format: ', e) - pass - except lief.bad_file as e: - # print('\tBad file: ', e) - pass - except lief.parser_error as e: - # print('\tParser error: ', e) - pass - except FileTypeNotImplemented as e: # noqa - # print(e) - pass + if HAS_LIEF: + try: + lief_parsed = lief.parse(filepath) + if isinstance(lief_parsed, lief.PE.Binary): + return make_pe_objects(lief_parsed, misp_file) + elif isinstance(lief_parsed, lief.ELF.Binary): + return make_elf_objects(lief_parsed, misp_file) + elif isinstance(lief_parsed, lief.MachO.Binary): + return make_macho_objects(lief_parsed, misp_file) + except lief.bad_format as e: + # print('\tBad format: ', e) + pass + except lief.bad_file as e: + # print('\tBad file: ', e) + pass + except lief.parser_error as e: + # print('\tParser error: ', e) + pass + except FileTypeNotImplemented as e: # noqa + # print(e) + pass return misp_file, None, None From ed441d6356910551393a1c980b1addf9b9af216d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 25 Aug 2017 17:48:57 +0200 Subject: [PATCH 17/29] Remove some python versions from travis --- .travis.yml | 11 ++++++++--- pymisp/tools/create_misp_object.py | 15 +++++++-------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8a9168c..5f93e2f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,15 +2,19 @@ language: python cache: pip +addons: + apt: + sources: [ 'ubuntu-toolchain-r-test' ] + packages: + - libstdc++6 + - libfuzzy-dev + python: - "2.7" - - "3.4" - "3.5" - "3.5-dev" - "3.6" - "3.6-dev" - - "3.7-dev" - - "nightly" install: - pip install -U nose @@ -18,6 +22,7 @@ install: - pip install codecov - pip install requests-mock pytest - pip install https://github.com/lief-project/packages/raw/lief-master-latest/pylief-0.7.0.dev.zip + - pip install git+https://github.com/kbandla/pydeep.git - pip install python-magic - pip install . - pushd tests diff --git a/pymisp/tools/create_misp_object.py b/pymisp/tools/create_misp_object.py index 9e2d7fb..cb18cb0 100644 --- a/pymisp/tools/create_misp_object.py +++ b/pymisp/tools/create_misp_object.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- from pymisp.tools import FileObject, PEObject, ELFObject, MachOObject, MISPObjectException +import warnings try: import lief @@ -55,15 +56,13 @@ def make_binary_objects(filepath): elif isinstance(lief_parsed, lief.MachO.Binary): return make_macho_objects(lief_parsed, misp_file) except lief.bad_format as e: - # print('\tBad format: ', e) - pass + warnings.warn('\tBad format: ', e) except lief.bad_file as e: - # print('\tBad file: ', e) - pass + warnings.warn('\tBad file: ', e) except lief.parser_error as e: - # print('\tParser error: ', e) - pass + warnings.warn('\tParser error: ', e) except FileTypeNotImplemented as e: # noqa - # print(e) - pass + warnings.warn(e) + else: + warnings.warn('Please install lief, documentation here: https://github.com/lief-project/LIEF') return misp_file, None, None From 44f32bc443eb06a0abcb64f4bb27fd09a2b8b908 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Sun, 27 Aug 2017 18:12:42 +0200 Subject: [PATCH 18/29] Add support for multiple entries of the same type in an object --- pymisp/data/misp-objects | 2 +- pymisp/tools/elfobject.py | 4 ++-- pymisp/tools/objectgenerator.py | 12 ++++++++++-- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/pymisp/data/misp-objects b/pymisp/data/misp-objects index fc6d54f..d34dd5f 160000 --- a/pymisp/data/misp-objects +++ b/pymisp/data/misp-objects @@ -1 +1 @@ -Subproject commit fc6d54f93b47d9c2ca951d68e568d1715e6eabf0 +Subproject commit d34dd5fb606f1c4d882733d16c16103fe429991c diff --git a/pymisp/tools/elfobject.py b/pymisp/tools/elfobject.py index 579d012..c62a336 100644 --- a/pymisp/tools/elfobject.py +++ b/pymisp/tools/elfobject.py @@ -78,8 +78,8 @@ class ELFSectionObject(MISPObjectGenerator): def generate_attributes(self): self._create_attribute('name', value=self.section.name) self._create_attribute('type', value=str(self.section.type).split('.')[1]) - print(self.section.flags) - # self._create_attribute('flag', value=str(self.section.flags).split('.')[1]) + for flag in self.section.flags_list: + self._create_attribute('flag', value=str(flag).split('.')[1]) size = self._create_attribute('size-in-bytes', value=self.section.size) if int(size.value) > 0: self._create_attribute('entropy', value=self.section.entropy) diff --git a/pymisp/tools/objectgenerator.py b/pymisp/tools/objectgenerator.py index d4b5105..46e4143 100644 --- a/pymisp/tools/objectgenerator.py +++ b/pymisp/tools/objectgenerator.py @@ -1,6 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +from collections import Counter from pymisp import MISPEvent, MISPAttribute, AbstractMISP import os import json @@ -112,9 +113,16 @@ class MISPObjectGenerator(AbstractMISP): def _validate(self): """Make sure the object we're creating has the required fields""" - all_attribute_names = set() + all_object_relations = [] for a in self.Attribute: - all_attribute_names.add(a.object_relation) + all_object_relations.append(a.object_relation) + count_relations = dict(Counter(all_object_relations)) + for key, counter in count_relations.items(): + if counter == 1: + continue + if not self.definition['attributes'][key].get('multiple'): + raise InvalidMISPObject('Multiple occurrences of {} is not allowed'.format(key)) + all_attribute_names = set(count_relations.keys()) if self.definition.get('requiredOneOf'): if not set(self.definition['requiredOneOf']) & all_attribute_names: raise InvalidMISPObject('At least one of the following attributes is required: {}'.format(', '.join(self.definition['requiredOneOf']))) From 2bc0745fbf9db54adca649479802d0a9a6ec21f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 28 Aug 2017 19:01:53 +0200 Subject: [PATCH 19/29] Refactoring in order to load objects --- examples/add_file_object.py | 4 +- examples/generate_file_objects.py | 13 +- pymisp/__init__.py | 14 +-- pymisp/abstract.py | 7 +- pymisp/defaultobjects.py | 195 +++++++++++++++++++++++++++++ pymisp/exceptions.py | 14 +++ pymisp/mispevent.py | 11 +- pymisp/tools/__init__.py | 1 - pymisp/tools/create_misp_object.py | 3 +- pymisp/tools/elfobject.py | 36 +++--- pymisp/tools/fileobject.py | 24 ++-- pymisp/tools/machoobject.py | 30 ++--- pymisp/tools/objectgenerator.py | 142 --------------------- pymisp/tools/peobject.py | 50 ++++---- tests/test_offline.py | 12 +- 15 files changed, 317 insertions(+), 239 deletions(-) create mode 100644 pymisp/defaultobjects.py delete mode 100644 pymisp/tools/objectgenerator.py diff --git a/examples/add_file_object.py b/examples/add_file_object.py index aa8ab8e..71f032b 100755 --- a/examples/add_file_object.py +++ b/examples/add_file_object.py @@ -30,11 +30,11 @@ if __name__ == '__main__': if peo: template_id = pymisp.get_object_template_id(peo.template_uuid) r = pymisp.add_object(args.event, template_id, peo) - for ref in peo.references: + for ref in peo.ObjectReference: r = pymisp.add_object_reference(ref) if fo: template_id = pymisp.get_object_template_id(fo.template_uuid) response = pymisp.add_object(args.event, template_id, fo) - for ref in fo.references: + for ref in fo.ObjectReference: r = pymisp.add_object_reference(ref) diff --git a/examples/generate_file_objects.py b/examples/generate_file_objects.py index 5ff6bb6..3269845 100755 --- a/examples/generate_file_objects.py +++ b/examples/generate_file_objects.py @@ -10,6 +10,7 @@ try: except ImportError: pass + def check(): missing_dependencies = {'pydeep': False, 'lief': False, 'magic': False, 'pymisp': False} try: @@ -38,18 +39,18 @@ def make_objects(path): if seos: for s in seos: to_return['objects'].append(s) - if s.references: - to_return['references'] += s.references + if s.ObjectReference: + to_return['references'] += s.ObjectReference if peo: to_return['objects'].append(peo) - if peo.references: - to_return['references'] += peo.references + if peo.ObjectReference: + to_return['references'] += peo.ObjectReference if fo: to_return['objects'].append(fo) - if fo.references: - to_return['references'] += fo.references + if fo.ObjectReference: + to_return['references'] += fo.ObjectReference return json.dumps(to_return, cls=MISPEncode) if __name__ == '__main__': diff --git a/pymisp/__init__.py b/pymisp/__init__.py index 532dc15..0565fe2 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -1,12 +1,12 @@ __version__ = '2.4.77' try: - from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey - from .api import PyMISP - from .abstract import AbstractMISP, MISPEncode - from .mispevent import MISPEvent, MISPAttribute, EncodeUpdate, EncodeFull - from .tools import Neo4j - from .tools import stix - from .tools import MISPObjectGenerator + from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey, InvalidMISPObject, UnknownMISPObjectTemplate # noqa + from .api import PyMISP # noqa + from .abstract import AbstractMISP, MISPEncode # noqa + from .defaultobjects import MISPObject, AbstractMISPObjectGenerator # noqa + from .mispevent import MISPEvent, MISPAttribute, EncodeUpdate, EncodeFull # noqa + from .tools import Neo4j # noqa + from .tools import stix # noqa except ImportError: pass diff --git a/pymisp/abstract.py b/pymisp/abstract.py index c1abca6..302a14d 100644 --- a/pymisp/abstract.py +++ b/pymisp/abstract.py @@ -5,9 +5,13 @@ import abc import json from json import JSONEncoder import collections - import six # Remove that import when discarding python2 support. +if six.PY2: + import warnings + warnings.warn("You're using python 2, it is strongly recommended to use python >=3.5") + + class MISPEncode(JSONEncoder): def default(self, obj): @@ -17,7 +21,6 @@ class MISPEncode(JSONEncoder): @six.add_metaclass(abc.ABCMeta) # Remove that line when discarding python2 support. -# Python3 way: class MISPObjectGenerator(metaclass=abc.ABCMeta): class AbstractMISP(collections.MutableMapping): attributes = None diff --git a/pymisp/defaultobjects.py b/pymisp/defaultobjects.py new file mode 100644 index 0000000..67ec076 --- /dev/null +++ b/pymisp/defaultobjects.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import abc +import os +import json +import sys +import uuid +from collections import Counter + +from .abstract import AbstractMISP +from .exceptions import UnknownMISPObjectTemplate, InvalidMISPObject +import six # Remove that import when discarding python2 support. + + +if six.PY2: + import warnings + warnings.warn("You're using python 2, it is strongly recommended to use python >=3.5") + + +class MISPObjectReference(AbstractMISP): + + attributes = ['source_uuid', 'destination_uuid', 'relationship_type', 'comment', 'uuid', 'deleted'] + + def __init__(self): + super(MISPObjectReference, self).__init__() + + def from_dict(self, source_uuid, destination_uuid, relationship_type, comment=None, **kwargs): + self.source_uuid = source_uuid + self.destination_uuid = destination_uuid + self.relationship_type = relationship_type + self.comment = comment + for k, v in kwargs: + setattr(self, k, v) + + +class MISPObjectAttribute(AbstractMISP): + + # This list is very limited and hardcoded to fit the current needs (file/pe/pesection creation): MISPAttriute will follow the + # same spec and just add one attribute: object_relation + attributes = ['object_relation', 'value', 'type', 'category', 'disable_correlation', 'to_ids', + 'data', 'encrypt', 'distribution', 'comment', 'uuid', 'event_id'] + + def __init__(self, definition): + super(MISPObjectAttribute, self).__init__() + self.definition = definition + + def from_dict(self, object_relation, value, **kwargs): + from .mispevent import MISPAttribute + self.object_relation = object_relation + self.value = value + # Initialize the new MISPAttribute + # Get the misp attribute type from the definition + self.type = kwargs.pop('type', None) + if self.type is None: + self.type = self.definition.get('misp-attribute') + self.disable_correlation = kwargs.pop('disable_correlation', None) + if self.disable_correlation is None: + # The correlation can be disabled by default in the object definition. + # Use this value if it isn't overloaded by the object + self.disable_correlation = self.definition.get('disable_correlation') + self.to_ids = kwargs.pop('to_ids', None) + if self.to_ids is None: + # Same for the to_ids flag + self.to_ids = self.definition.get('to_ids') + # FIXME: dirty hack until all the classes are ported to the new format but we get the default values + # Initialise rest of the values + for k, v in kwargs.items(): + setattr(self, k, v) + temp_attribute = MISPAttribute() + temp_attribute.set_all_values(**self) + # Update default values + for k, v in temp_attribute.to_dict().items(): + setattr(self, k, v) + + +class MISPObject(AbstractMISP): + + attributes = ['name', 'meta-category', 'uuid', 'description', 'template_version', 'template_uuid', 'Attribute'] + + def __init__(self, name, strict=True): + super(MISPObject, self).__init__() + self.strict = strict + self.name = name + self.misp_objects_path = os.path.join( + os.path.abspath(os.path.dirname(sys.modules['pymisp'].__file__)), + 'data', 'misp-objects', 'objects') + if os.path.exists(os.path.join(self.misp_objects_path, self.name, 'definition.json')): + self.known_template = True + else: + if self.strict: + raise UnknownMISPObjectTemplate('{} is unknown in the MISP object directory.') + else: + self.known_template = False + if self.known_template: + with open(os.path.join(self.misp_objects_path, self.name, 'definition.json'), 'r') as f: + self.definition = json.load(f) + setattr(self, 'meta-category', self.definition['meta-category']) + self.template_uuid = self.definition['uuid'] + self.description = self.definition['description'] + self.template_version = self.definition['version'] + else: + # FIXME We need to set something for meta-category, template_uuid, description and template_version + pass + self.uuid = str(uuid.uuid4()) + self.Attribute = [] + self.ObjectReference = [] + + def from_dict(self, **kwargs): + if self.known_template: + if kwargs.get('template_uuid') and kwargs['template_uuid'] != self.template_uuid: + if self.strict: + raise UnknownMISPObjectTemplate('UUID of the object is different from the one of the template.') + else: + self.known_template = False + if kwargs.get('template_version') and int(kwargs['template_version']) != self.template_version: + if self.strict: + raise UnknownMISPObjectTemplate('Version of the object ({}) is different from the one of the template ({}).'.format(kwargs['template_version'], self.template_version)) + else: + self.known_template = False + + for key, value in kwargs.items(): + if key == 'Attribute': + for v in value: + self.add_attribute(**v) + elif key == 'ObjectReference': + for v in value: + self.add_reference(**v) + else: + setattr(self, key, value) + + def to_dict(self, strict=True): + if strict or self.strict and self.known_template: + self._validate() + return super(MISPObject, self).to_dict() + + def to_json(self, strict=True): + if strict or self.strict and self.known_template: + self._validate() + return super(MISPObject, self).to_json() + + def _validate(self): + """Make sure the object we're creating has the required fields""" + all_object_relations = [] + for a in self.Attribute: + all_object_relations.append(a.object_relation) + count_relations = dict(Counter(all_object_relations)) + for key, counter in count_relations.items(): + if counter == 1: + continue + if not self.definition['attributes'][key].get('multiple'): + raise InvalidMISPObject('Multiple occurrences of {} is not allowed'.format(key)) + all_attribute_names = set(count_relations.keys()) + if self.definition.get('requiredOneOf'): + if not set(self.definition['requiredOneOf']) & all_attribute_names: + raise InvalidMISPObject('At least one of the following attributes is required: {}'.format(', '.join(self.definition['requiredOneOf']))) + if self.definition.get('required'): + for r in self.definition.get('required'): + if r not in all_attribute_names: + raise InvalidMISPObject('{} is required'.format(r)) + return True + + def add_reference(self, destination_uuid, relationship_type, comment=None, **kwargs): + """Add a link (uuid) to an other object""" + if kwargs.get('source_uuid'): + # Load existing object + source_uuid = kwargs.get('source_uuid') + else: + # New reference + source_uuid = self.uuid + reference = MISPObjectReference() + reference.from_dict(source_uuid=source_uuid, destination_uuid=destination_uuid, + relationship_type=relationship_type, comment=comment, **kwargs) + self.ObjectReference.append(reference) + + def add_attribute(self, object_relation, **value): + if value.get('value') is None: + return None + if self.known_template: + attribute = MISPObjectAttribute(self.definition['attributes'][object_relation]) + else: + attribute = MISPObjectAttribute({}) + attribute.from_dict(object_relation, **value) + self.Attribute.append(attribute) + return attribute + + +@six.add_metaclass(abc.ABCMeta) # Remove that line when discarding python2 support. +# Python3 way: class MISPObjectGenerator(metaclass=abc.ABCMeta): +class AbstractMISPObjectGenerator(MISPObject): + + @abc.abstractmethod + def generate_attributes(self): + """Contains the logic where all the values of the object are gathered""" + pass diff --git a/pymisp/exceptions.py b/pymisp/exceptions.py index f4db340..3ec3d7b 100644 --- a/pymisp/exceptions.py +++ b/pymisp/exceptions.py @@ -1,6 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- + class PyMISPError(Exception): def __init__(self, message): super(PyMISPError, self).__init__(message) @@ -29,3 +30,16 @@ class NoURL(PyMISPError): class NoKey(PyMISPError): pass + + +class MISPObjectException(PyMISPError): + pass + + +class InvalidMISPObject(MISPObjectException): + """Exception raised when an object doesn't respect the contrains in the definition""" + pass + +class UnknownMISPObjectTemplate(MISPObjectException): + """Exception raised when the template is unknown""" + pass diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index d71be70..226b57d 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -11,7 +11,6 @@ import base64 from io import BytesIO from zipfile import ZipFile import hashlib -from .abstract import AbstractMISP try: from dateutil.parser import parse @@ -38,6 +37,7 @@ except ImportError: has_pyme = False from .exceptions import PyMISPError, NewEventError, NewAttributeError +from .defaultobjects import MISPObject # Least dirty way to support python 2 and 3 try: @@ -285,7 +285,7 @@ def _int_to_str(d): return d -class MISPEvent(AbstractMISP): +class MISPEvent(object): def __init__(self, describe_types=None, strict_validation=False): self.ressources_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data') @@ -338,6 +338,7 @@ class MISPEvent(AbstractMISP): self.RelatedEvent = [] self.Tag = [] self.Galaxy = None + self.Object = None def _serialize(self): return '{date}{threat_level_id}{info}{uuid}{analysis}{timestamp}'.format( @@ -510,6 +511,12 @@ class MISPEvent(AbstractMISP): self.sig = kwargs['sig'] if kwargs.get('global_sig'): self.global_sig = kwargs['global_sig'] + if kwargs.get('Object'): + self.Object = [] + for obj in kwargs['Object']: + tmp_object = MISPObject(obj['name']) + tmp_object.from_dict(**obj) + self.Object.append(tmp_object) def _json(self): # DEPTECATED diff --git a/pymisp/tools/__init__.py b/pymisp/tools/__init__.py index 2a81b2a..3a73be5 100644 --- a/pymisp/tools/__init__.py +++ b/pymisp/tools/__init__.py @@ -1,5 +1,4 @@ from .neo4j import Neo4j # noqa -from .objectgenerator import MISPObjectGenerator, MISPObjectException, InvalidMISPObject # noqa from .fileobject import FileObject # noqa from .peobject import PEObject, PESectionObject # noqa from .elfobject import ELFObject, ELFSectionObject # noqa diff --git a/pymisp/tools/create_misp_object.py b/pymisp/tools/create_misp_object.py index cb18cb0..f6d82c3 100644 --- a/pymisp/tools/create_misp_object.py +++ b/pymisp/tools/create_misp_object.py @@ -1,7 +1,8 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from pymisp.tools import FileObject, PEObject, ELFObject, MachOObject, MISPObjectException +from pymisp.tools import FileObject, PEObject, ELFObject, MachOObject +from pymisp.exceptions import MISPObjectException import warnings try: diff --git a/pymisp/tools/elfobject.py b/pymisp/tools/elfobject.py index c62a336..872f554 100644 --- a/pymisp/tools/elfobject.py +++ b/pymisp/tools/elfobject.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -from pymisp.tools import MISPObjectGenerator +from pymisp.defaultobjects import AbstractMISPObjectGenerator from io import BytesIO from hashlib import md5, sha1, sha256, sha512 import warnings @@ -20,7 +20,7 @@ except ImportError: HAS_PYDEEP = False -class ELFObject(MISPObjectGenerator): +class ELFObject(AbstractMISPObjectGenerator): def __init__(self, parsed=None, filepath=None, pseudofile=None): if not HAS_PYDEEP: @@ -49,10 +49,10 @@ class ELFObject(MISPObjectGenerator): def generate_attributes(self): # General information - self._create_attribute('type', value=str(self.elf.header.file_type).split('.')[1]) - self._create_attribute('entrypoint-address', value=self.elf.entrypoint) - self._create_attribute('arch', value=str(self.elf.header.machine_type).split('.')[1]) - self._create_attribute('os_abi', value=str(self.elf.header.identity_os_abi).split('.')[1]) + self.add_attribute('type', value=str(self.elf.header.file_type).split('.')[1]) + self.add_attribute('entrypoint-address', value=self.elf.entrypoint) + self.add_attribute('arch', value=str(self.elf.header.machine_type).split('.')[1]) + self.add_attribute('os_abi', value=str(self.elf.header.identity_os_abi).split('.')[1]) # Sections self.sections = [] if self.elf.sections: @@ -62,10 +62,10 @@ class ELFObject(MISPObjectGenerator): self.add_reference(s.uuid, 'included-in', 'Section {} of ELF'.format(pos)) pos += 1 self.sections.append(s) - self._create_attribute('number-sections', value=len(self.sections)) + self.add_attribute('number-sections', value=len(self.sections)) -class ELFSectionObject(MISPObjectGenerator): +class ELFSectionObject(AbstractMISPObjectGenerator): def __init__(self, section): # Python3 way @@ -76,16 +76,16 @@ class ELFSectionObject(MISPObjectGenerator): self.generate_attributes() def generate_attributes(self): - self._create_attribute('name', value=self.section.name) - self._create_attribute('type', value=str(self.section.type).split('.')[1]) + self.add_attribute('name', value=self.section.name) + self.add_attribute('type', value=str(self.section.type).split('.')[1]) for flag in self.section.flags_list: - self._create_attribute('flag', value=str(flag).split('.')[1]) - size = self._create_attribute('size-in-bytes', value=self.section.size) + self.add_attribute('flag', value=str(flag).split('.')[1]) + size = self.add_attribute('size-in-bytes', value=self.section.size) if int(size.value) > 0: - self._create_attribute('entropy', value=self.section.entropy) - self._create_attribute('md5', value=md5(self.data).hexdigest()) - self._create_attribute('sha1', value=sha1(self.data).hexdigest()) - self._create_attribute('sha256', value=sha256(self.data).hexdigest()) - self._create_attribute('sha512', value=sha512(self.data).hexdigest()) + self.add_attribute('entropy', value=self.section.entropy) + self.add_attribute('md5', value=md5(self.data).hexdigest()) + self.add_attribute('sha1', value=sha1(self.data).hexdigest()) + self.add_attribute('sha256', value=sha256(self.data).hexdigest()) + self.add_attribute('sha512', value=sha512(self.data).hexdigest()) if HAS_PYDEEP: - self._create_attribute('ssdeep', value=pydeep.hash_buf(self.data).decode()) + self.add_attribute('ssdeep', value=pydeep.hash_buf(self.data).decode()) diff --git a/pymisp/tools/fileobject.py b/pymisp/tools/fileobject.py index ebea362..ed496e5 100644 --- a/pymisp/tools/fileobject.py +++ b/pymisp/tools/fileobject.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -from pymisp.tools import MISPObjectGenerator +from pymisp.defaultobjects import AbstractMISPObjectGenerator import os from io import BytesIO from hashlib import md5, sha1, sha256, sha512 @@ -22,7 +22,7 @@ except ImportError: HAS_MAGIC = False -class FileObject(MISPObjectGenerator): +class FileObject(AbstractMISPObjectGenerator): def __init__(self, filepath=None, pseudofile=None, filename=None): if not HAS_PYDEEP: @@ -48,19 +48,19 @@ class FileObject(MISPObjectGenerator): self.generate_attributes() def generate_attributes(self): - self._create_attribute('filename', value=self.filename) - size = self._create_attribute('size-in-bytes', value=len(self.data)) + self.add_attribute('filename', value=self.filename) + size = self.add_attribute('size-in-bytes', value=len(self.data)) if int(size.value) > 0: - self._create_attribute('entropy', value=self.__entropy_H(self.data)) - self._create_attribute('md5', value=md5(self.data).hexdigest()) - self._create_attribute('sha1', value=sha1(self.data).hexdigest()) - self._create_attribute('sha256', value=sha256(self.data).hexdigest()) - self._create_attribute('sha512', value=sha512(self.data).hexdigest()) - self._create_attribute('malware-sample', value=self.filename, data=self.pseudofile) + self.add_attribute('entropy', value=self.__entropy_H(self.data)) + self.add_attribute('md5', value=md5(self.data).hexdigest()) + self.add_attribute('sha1', value=sha1(self.data).hexdigest()) + self.add_attribute('sha256', value=sha256(self.data).hexdigest()) + self.add_attribute('sha512', value=sha512(self.data).hexdigest()) + self.add_attribute('malware-sample', value=self.filename, data=self.pseudofile) if HAS_MAGIC: - self._create_attribute('mimetype', value=magic.from_buffer(self.data)) + self.add_attribute('mimetype', value=magic.from_buffer(self.data)) if HAS_PYDEEP: - self._create_attribute('ssdeep', value=pydeep.hash_buf(self.data).decode()) + self.add_attribute('ssdeep', value=pydeep.hash_buf(self.data).decode()) def __entropy_H(self, data): """Calculate the entropy of a chunk of data.""" diff --git a/pymisp/tools/machoobject.py b/pymisp/tools/machoobject.py index e688fba..53e8e0b 100644 --- a/pymisp/tools/machoobject.py +++ b/pymisp/tools/machoobject.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -from pymisp.tools import MISPObjectGenerator +from pymisp.defaultobjects import AbstractMISPObjectGenerator from io import BytesIO from hashlib import md5, sha1, sha256, sha512 import warnings @@ -20,7 +20,7 @@ except ImportError: HAS_PYDEEP = False -class MachOObject(MISPObjectGenerator): +class MachOObject(AbstractMISPObjectGenerator): def __init__(self, parsed=None, filepath=None, pseudofile=None): if not HAS_PYDEEP: @@ -48,11 +48,11 @@ class MachOObject(MISPObjectGenerator): self.generate_attributes() def generate_attributes(self): - self._create_attribute('type', value=str(self.macho.header.file_type).split('.')[1]) - self._create_attribute('name', value=self.macho.name) + self.add_attribute('type', value=str(self.macho.header.file_type).split('.')[1]) + self.add_attribute('name', value=self.macho.name) # General information if self.macho.has_entrypoint: - self._create_attribute('entrypoint-address', value=self.macho.entrypoint) + self.add_attribute('entrypoint-address', value=self.macho.entrypoint) # Sections self.sections = [] if self.macho.sections: @@ -62,10 +62,10 @@ class MachOObject(MISPObjectGenerator): self.add_reference(s.uuid, 'included-in', 'Section {} of MachO'.format(pos)) pos += 1 self.sections.append(s) - self._create_attribute('number-sections', value=len(self.sections)) + self.add_attribute('number-sections', value=len(self.sections)) -class MachOSectionObject(MISPObjectGenerator): +class MachOSectionObject(AbstractMISPObjectGenerator): def __init__(self, section): # Python3 way @@ -76,13 +76,13 @@ class MachOSectionObject(MISPObjectGenerator): self.generate_attributes() def generate_attributes(self): - self._create_attribute('name', value=self.section.name) - size = self._create_attribute('size-in-bytes', value=self.section.size) + self.add_attribute('name', value=self.section.name) + size = self.add_attribute('size-in-bytes', value=self.section.size) if int(size.value) > 0: - self._create_attribute('entropy', value=self.section.entropy) - self._create_attribute('md5', value=md5(self.data).hexdigest()) - self._create_attribute('sha1', value=sha1(self.data).hexdigest()) - self._create_attribute('sha256', value=sha256(self.data).hexdigest()) - self._create_attribute('sha512', value=sha512(self.data).hexdigest()) + self.add_attribute('entropy', value=self.section.entropy) + self.add_attribute('md5', value=md5(self.data).hexdigest()) + self.add_attribute('sha1', value=sha1(self.data).hexdigest()) + self.add_attribute('sha256', value=sha256(self.data).hexdigest()) + self.add_attribute('sha512', value=sha512(self.data).hexdigest()) if HAS_PYDEEP: - self._create_attribute('ssdeep', value=pydeep.hash_buf(self.data).decode()) + self.add_attribute('ssdeep', value=pydeep.hash_buf(self.data).decode()) diff --git a/pymisp/tools/objectgenerator.py b/pymisp/tools/objectgenerator.py deleted file mode 100644 index 46e4143..0000000 --- a/pymisp/tools/objectgenerator.py +++ /dev/null @@ -1,142 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from collections import Counter -from pymisp import MISPEvent, MISPAttribute, AbstractMISP -import os -import json -import uuid -import abc -import sys -import six # Remove that import when discarding python2 support. - - -class MISPObjectException(Exception): - pass - - -class InvalidMISPObject(MISPObjectException): - """Exception raised when an object doesn't contains the required field(s)""" - pass - - -if six.PY2: - import warnings - warnings.warn("You're using python 2, it is strongly recommended to use python >=3.5") - - -class MISPObjectReference(AbstractMISP): - - attributes = ['source_uuid', 'destination_uuid', 'relationship_type', 'comment'] - - def __init__(self, source_uuid, destination_uuid, relationship_type, comment=None): - self.source_uuid = source_uuid - self.destination_uuid = destination_uuid - self.relationship_type = relationship_type - self.comment = comment - - -class MISPObjectAttribute(AbstractMISP): - - # This list is very limited and hardcoded to fit the current needs (file/pe/pesection creation): MISPAttriute will follow the - # same spec and just add one attribute: object_relation - attributes = ['object_relation', 'value', 'type', 'category', 'disable_correlation', 'to_ids', - 'data', 'encrypt', 'distribution', 'comment'] - - def __init__(self, definition, object_relation, value, **kwargs): - self.object_relation = object_relation - self.value = value - # Initialize the new MISPAttribute - # Get the misp attribute type from the definition - self.type = kwargs.pop('type', None) - if self.type is None: - self.type = definition['misp-attribute'] - self.disable_correlation = kwargs.pop('disable_correlation', None) - if self.disable_correlation is None: - # The correlation can be disabled by default in the object definition. - # Use this value if it isn't overloaded by the object - self.disable_correlation = definition.get('disable_correlation') - self.to_ids = kwargs.pop('to_ids', None) - if self.to_ids is None: - # Same for the to_ids flag - self.to_ids = definition.get('to_ids') - # Initialise rest of the values - for k, v in kwargs.items(): - self[k] = v - # FIXME: dirty hack until all the classes are ported to the new format but we get the default values - temp_attribute = MISPAttribute() - temp_attribute.set_all_values(**self) - # Update default values - self.from_dict(**temp_attribute.to_dict()) - - -class MISPObjectGenerator(AbstractMISP): - - attributes = ['name', 'meta-category', 'uuid', 'description', 'version', 'Attribute'] - - def __init__(self, template_dir): - """This class is used to fill a new MISP object with the default values defined in the object template - * template is the path to the template within the misp-object repository - * misp_objects_path is the path to the misp-object repository - """ - self.misp_objects_path = os.path.join( - os.path.abspath(os.path.dirname(sys.modules['pymisp'].__file__)), - 'data', 'misp-objects', 'objects') - with open(os.path.join(self.misp_objects_path, template_dir, 'definition.json'), 'r') as f: - self.definition = json.load(f) - self.misp_event = MISPEvent() - self.name = self.definition['name'] - setattr(self, 'meta-category', self.definition['meta-category']) - self.template_uuid = self.definition['uuid'] - self.description = self.definition['description'] - self.version = self.definition['version'] - self.uuid = str(uuid.uuid4()) - self.Attribute = [] - self.references = [] - - def _create_attribute(self, object_type, **value): - if value.get('value') is None: - return None - attribute = MISPObjectAttribute(self.definition['attributes'][object_type], object_type, **value) - self.Attribute.append(attribute) - return attribute - - def to_dict(self, strict=True): - if strict: - self._validate() - return super(MISPObjectGenerator, self).to_dict() - - def to_json(self, strict=True): - if strict: - self._validate() - return super(MISPObjectGenerator, self).to_json() - - def _validate(self): - """Make sure the object we're creating has the required fields""" - all_object_relations = [] - for a in self.Attribute: - all_object_relations.append(a.object_relation) - count_relations = dict(Counter(all_object_relations)) - for key, counter in count_relations.items(): - if counter == 1: - continue - if not self.definition['attributes'][key].get('multiple'): - raise InvalidMISPObject('Multiple occurrences of {} is not allowed'.format(key)) - all_attribute_names = set(count_relations.keys()) - if self.definition.get('requiredOneOf'): - if not set(self.definition['requiredOneOf']) & all_attribute_names: - raise InvalidMISPObject('At least one of the following attributes is required: {}'.format(', '.join(self.definition['requiredOneOf']))) - if self.definition.get('required'): - for r in self.definition.get('required'): - if r not in all_attribute_names: - raise InvalidMISPObject('{} is required'.format(r)) - return True - - def add_reference(self, destination_uuid, relationship_type, comment=None): - """Add a link (uuid) to an other object""" - self.references.append(MISPObjectReference(self.uuid, destination_uuid, relationship_type, comment)) - - @abc.abstractmethod - def generate_attributes(self): - """Contains the logic where all the values of the object are gathered""" - pass diff --git a/pymisp/tools/peobject.py b/pymisp/tools/peobject.py index bea117a..71d6795 100644 --- a/pymisp/tools/peobject.py +++ b/pymisp/tools/peobject.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -from pymisp.tools import MISPObjectGenerator +from pymisp.defaultobjects import AbstractMISPObjectGenerator from io import BytesIO from hashlib import md5, sha1, sha256, sha512 from datetime import datetime @@ -21,7 +21,7 @@ except ImportError: HAS_PYDEEP = False -class PEObject(MISPObjectGenerator): +class PEObject(AbstractMISPObjectGenerator): def __init__(self, parsed=None, filepath=None, pseudofile=None): if not HAS_PYDEEP: @@ -74,10 +74,10 @@ class PEObject(MISPObjectGenerator): return 'unknown' def generate_attributes(self): - self._create_attribute('type', value=self._get_pe_type()) + self.add_attribute('type', value=self._get_pe_type()) # General information - self._create_attribute('entrypoint-address', value=self.pe.entrypoint) - self._create_attribute('compilation-timestamp', value=datetime.utcfromtimestamp(self.pe.header.time_date_stamps).isoformat()) + self.add_attribute('entrypoint-address', value=self.pe.entrypoint) + self.add_attribute('compilation-timestamp', value=datetime.utcfromtimestamp(self.pe.header.time_date_stamps).isoformat()) # self.imphash = self.pe.get_imphash() try: if (self.pe.has_resources and @@ -85,15 +85,15 @@ class PEObject(MISPObjectGenerator): self.pe.resources_manager.version.has_string_file_info and self.pe.resources_manager.version.string_file_info.langcode_items): fileinfo = dict(self.pe.resources_manager.version.string_file_info.langcode_items[0].items.items()) - self._create_attribute('original-filename', value=fileinfo.get('OriginalFilename')) - self._create_attribute('internal-filename', value=fileinfo.get('InternalName')) - self._create_attribute('file-description', value=fileinfo.get('FileDescription')) - self._create_attribute('file-version', value=fileinfo.get('FileVersion')) - self._create_attribute('lang-id', value=self.pe.resources_manager.version.string_file_info.langcode_items[0].key) - self._create_attribute('product-name', value=fileinfo.get('ProductName')) - self._create_attribute('product-version', value=fileinfo.get('ProductVersion')) - self._create_attribute('company-name', value=fileinfo.get('CompanyName')) - self._create_attribute('legal-copyright', value=fileinfo.get('LegalCopyright')) + self.add_attribute('original-filename', value=fileinfo.get('OriginalFilename')) + self.add_attribute('internal-filename', value=fileinfo.get('InternalName')) + self.add_attribute('file-description', value=fileinfo.get('FileDescription')) + self.add_attribute('file-version', value=fileinfo.get('FileVersion')) + self.add_attribute('lang-id', value=self.pe.resources_manager.version.string_file_info.langcode_items[0].key) + self.add_attribute('product-name', value=fileinfo.get('ProductName')) + self.add_attribute('product-version', value=fileinfo.get('ProductVersion')) + self.add_attribute('company-name', value=fileinfo.get('CompanyName')) + self.add_attribute('legal-copyright', value=fileinfo.get('LegalCopyright')) except lief.read_out_of_bound: # The file is corrupted pass @@ -106,14 +106,14 @@ class PEObject(MISPObjectGenerator): self.add_reference(s.uuid, 'included-in', 'Section {} of PE'.format(pos)) if ((self.pe.entrypoint >= section.virtual_address) and (self.pe.entrypoint < (section.virtual_address + section.virtual_size))): - self._create_attribute('entrypoint-section|position', value='{}|{}'.format(section.name, pos)) + self.add_attribute('entrypoint-section|position', value='{}|{}'.format(section.name, pos)) pos += 1 self.sections.append(s) - self._create_attribute('number-sections', value=len(self.sections)) + self.add_attribute('number-sections', value=len(self.sections)) # TODO: TLSSection / DIRECTORY_ENTRY_TLS -class PESectionObject(MISPObjectGenerator): +class PESectionObject(AbstractMISPObjectGenerator): def __init__(self, section): # Python3 way @@ -124,13 +124,13 @@ class PESectionObject(MISPObjectGenerator): self.generate_attributes() def generate_attributes(self): - self._create_attribute('name', value=self.section.name) - size = self._create_attribute('size-in-bytes', value=self.section.size) + self.add_attribute('name', value=self.section.name) + size = self.add_attribute('size-in-bytes', value=self.section.size) if int(size.value) > 0: - self._create_attribute('entropy', value=self.section.entropy) - self._create_attribute('md5', value=md5(self.data).hexdigest()) - self._create_attribute('sha1', value=sha1(self.data).hexdigest()) - self._create_attribute('sha256', value=sha256(self.data).hexdigest()) - self._create_attribute('sha512', value=sha512(self.data).hexdigest()) + self.add_attribute('entropy', value=self.section.entropy) + self.add_attribute('md5', value=md5(self.data).hexdigest()) + self.add_attribute('sha1', value=sha1(self.data).hexdigest()) + self.add_attribute('sha256', value=sha256(self.data).hexdigest()) + self.add_attribute('sha512', value=sha512(self.data).hexdigest()) if HAS_PYDEEP: - self._create_attribute('ssdeep', value=pydeep.hash_buf(self.data).decode()) + self.add_attribute('ssdeep', value=pydeep.hash_buf(self.data).decode()) diff --git a/tests/test_offline.py b/tests/test_offline.py index 14c5aad..b8eb097 100644 --- a/tests/test_offline.py +++ b/tests/test_offline.py @@ -230,18 +230,18 @@ class TestOffline(unittest.TestCase): if seos: for s in seos: to_return['objects'].append(s) - if s.references: - to_return['references'] += s.references + if s.ObjectReference: + to_return['references'] += s.ObjectReference if peo: to_return['objects'].append(peo) - if peo.references: - to_return['references'] += peo.references + if peo.ObjectReference: + to_return['references'] += peo.ObjectReference if fo: to_return['objects'].append(fo) - if fo.references: - to_return['references'] += fo.references + if fo.ObjectReference: + to_return['references'] += fo.ObjectReference return json.dumps(to_return, cls=MISPEncode) def test_objects(self, m): From 8452e16dbd23e2eaf822dfcf38ca25fc084c9afd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 29 Aug 2017 10:25:45 +0200 Subject: [PATCH 20/29] Update PE generator --- pymisp/data/misp-objects | 2 +- pymisp/tools/peobject.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pymisp/data/misp-objects b/pymisp/data/misp-objects index d34dd5f..9da5eae 160000 --- a/pymisp/data/misp-objects +++ b/pymisp/data/misp-objects @@ -1 +1 @@ -Subproject commit d34dd5fb606f1c4d882733d16c16103fe429991c +Subproject commit 9da5eaed3520dde8ed624bc900a26e6280a356a4 diff --git a/pymisp/tools/peobject.py b/pymisp/tools/peobject.py index 71d6795..d8ab22d 100644 --- a/pymisp/tools/peobject.py +++ b/pymisp/tools/peobject.py @@ -106,7 +106,7 @@ class PEObject(AbstractMISPObjectGenerator): self.add_reference(s.uuid, 'included-in', 'Section {} of PE'.format(pos)) if ((self.pe.entrypoint >= section.virtual_address) and (self.pe.entrypoint < (section.virtual_address + section.virtual_size))): - self.add_attribute('entrypoint-section|position', value='{}|{}'.format(section.name, pos)) + self.add_attribute('entrypoint-section-at-position', value='{}|{}'.format(section.name, pos)) pos += 1 self.sections.append(s) self.add_attribute('number-sections', value=len(self.sections)) From 69ede74ce734d3c14375b174dda6d9161168526b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 29 Aug 2017 18:38:28 +0200 Subject: [PATCH 21/29] Update object definitions --- pymisp/data/misp-objects | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/data/misp-objects b/pymisp/data/misp-objects index 9da5eae..0445ebd 160000 --- a/pymisp/data/misp-objects +++ b/pymisp/data/misp-objects @@ -1 +1 @@ -Subproject commit 9da5eaed3520dde8ed624bc900a26e6280a356a4 +Subproject commit 0445ebd350900aaaebc47309800dbc156f697592 From 74037cb6fa845285815f558d5dcdcfb89fb2f59a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 30 Aug 2017 12:47:32 +0200 Subject: [PATCH 22/29] Some more refactoring and cleanup --- pymisp/__init__.py | 5 +- pymisp/defaultobjects.py | 195 ----------------------------- pymisp/mispevent.py | 194 +++++++++++++++++++++++++++- pymisp/tools/create_misp_object.py | 4 +- pymisp/tools/elfobject.py | 2 +- pymisp/tools/fileobject.py | 2 +- pymisp/tools/machoobject.py | 2 +- pymisp/tools/neo4j.py | 4 +- pymisp/tools/openioc.py | 2 +- pymisp/tools/peobject.py | 2 +- 10 files changed, 200 insertions(+), 212 deletions(-) delete mode 100644 pymisp/defaultobjects.py diff --git a/pymisp/__init__.py b/pymisp/__init__.py index 0565fe2..6afb49e 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -1,11 +1,10 @@ -__version__ = '2.4.77' +__version__ = '2.4.80' try: from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey, InvalidMISPObject, UnknownMISPObjectTemplate # noqa from .api import PyMISP # noqa from .abstract import AbstractMISP, MISPEncode # noqa - from .defaultobjects import MISPObject, AbstractMISPObjectGenerator # noqa - from .mispevent import MISPEvent, MISPAttribute, EncodeUpdate, EncodeFull # noqa + from .mispevent import MISPEvent, MISPAttribute, EncodeUpdate, EncodeFull, MISPObjectReference, MISPObjectAttribute, MISPObject, AbstractMISPObjectGenerator # noqa from .tools import Neo4j # noqa from .tools import stix # noqa except ImportError: diff --git a/pymisp/defaultobjects.py b/pymisp/defaultobjects.py deleted file mode 100644 index 67ec076..0000000 --- a/pymisp/defaultobjects.py +++ /dev/null @@ -1,195 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import abc -import os -import json -import sys -import uuid -from collections import Counter - -from .abstract import AbstractMISP -from .exceptions import UnknownMISPObjectTemplate, InvalidMISPObject -import six # Remove that import when discarding python2 support. - - -if six.PY2: - import warnings - warnings.warn("You're using python 2, it is strongly recommended to use python >=3.5") - - -class MISPObjectReference(AbstractMISP): - - attributes = ['source_uuid', 'destination_uuid', 'relationship_type', 'comment', 'uuid', 'deleted'] - - def __init__(self): - super(MISPObjectReference, self).__init__() - - def from_dict(self, source_uuid, destination_uuid, relationship_type, comment=None, **kwargs): - self.source_uuid = source_uuid - self.destination_uuid = destination_uuid - self.relationship_type = relationship_type - self.comment = comment - for k, v in kwargs: - setattr(self, k, v) - - -class MISPObjectAttribute(AbstractMISP): - - # This list is very limited and hardcoded to fit the current needs (file/pe/pesection creation): MISPAttriute will follow the - # same spec and just add one attribute: object_relation - attributes = ['object_relation', 'value', 'type', 'category', 'disable_correlation', 'to_ids', - 'data', 'encrypt', 'distribution', 'comment', 'uuid', 'event_id'] - - def __init__(self, definition): - super(MISPObjectAttribute, self).__init__() - self.definition = definition - - def from_dict(self, object_relation, value, **kwargs): - from .mispevent import MISPAttribute - self.object_relation = object_relation - self.value = value - # Initialize the new MISPAttribute - # Get the misp attribute type from the definition - self.type = kwargs.pop('type', None) - if self.type is None: - self.type = self.definition.get('misp-attribute') - self.disable_correlation = kwargs.pop('disable_correlation', None) - if self.disable_correlation is None: - # The correlation can be disabled by default in the object definition. - # Use this value if it isn't overloaded by the object - self.disable_correlation = self.definition.get('disable_correlation') - self.to_ids = kwargs.pop('to_ids', None) - if self.to_ids is None: - # Same for the to_ids flag - self.to_ids = self.definition.get('to_ids') - # FIXME: dirty hack until all the classes are ported to the new format but we get the default values - # Initialise rest of the values - for k, v in kwargs.items(): - setattr(self, k, v) - temp_attribute = MISPAttribute() - temp_attribute.set_all_values(**self) - # Update default values - for k, v in temp_attribute.to_dict().items(): - setattr(self, k, v) - - -class MISPObject(AbstractMISP): - - attributes = ['name', 'meta-category', 'uuid', 'description', 'template_version', 'template_uuid', 'Attribute'] - - def __init__(self, name, strict=True): - super(MISPObject, self).__init__() - self.strict = strict - self.name = name - self.misp_objects_path = os.path.join( - os.path.abspath(os.path.dirname(sys.modules['pymisp'].__file__)), - 'data', 'misp-objects', 'objects') - if os.path.exists(os.path.join(self.misp_objects_path, self.name, 'definition.json')): - self.known_template = True - else: - if self.strict: - raise UnknownMISPObjectTemplate('{} is unknown in the MISP object directory.') - else: - self.known_template = False - if self.known_template: - with open(os.path.join(self.misp_objects_path, self.name, 'definition.json'), 'r') as f: - self.definition = json.load(f) - setattr(self, 'meta-category', self.definition['meta-category']) - self.template_uuid = self.definition['uuid'] - self.description = self.definition['description'] - self.template_version = self.definition['version'] - else: - # FIXME We need to set something for meta-category, template_uuid, description and template_version - pass - self.uuid = str(uuid.uuid4()) - self.Attribute = [] - self.ObjectReference = [] - - def from_dict(self, **kwargs): - if self.known_template: - if kwargs.get('template_uuid') and kwargs['template_uuid'] != self.template_uuid: - if self.strict: - raise UnknownMISPObjectTemplate('UUID of the object is different from the one of the template.') - else: - self.known_template = False - if kwargs.get('template_version') and int(kwargs['template_version']) != self.template_version: - if self.strict: - raise UnknownMISPObjectTemplate('Version of the object ({}) is different from the one of the template ({}).'.format(kwargs['template_version'], self.template_version)) - else: - self.known_template = False - - for key, value in kwargs.items(): - if key == 'Attribute': - for v in value: - self.add_attribute(**v) - elif key == 'ObjectReference': - for v in value: - self.add_reference(**v) - else: - setattr(self, key, value) - - def to_dict(self, strict=True): - if strict or self.strict and self.known_template: - self._validate() - return super(MISPObject, self).to_dict() - - def to_json(self, strict=True): - if strict or self.strict and self.known_template: - self._validate() - return super(MISPObject, self).to_json() - - def _validate(self): - """Make sure the object we're creating has the required fields""" - all_object_relations = [] - for a in self.Attribute: - all_object_relations.append(a.object_relation) - count_relations = dict(Counter(all_object_relations)) - for key, counter in count_relations.items(): - if counter == 1: - continue - if not self.definition['attributes'][key].get('multiple'): - raise InvalidMISPObject('Multiple occurrences of {} is not allowed'.format(key)) - all_attribute_names = set(count_relations.keys()) - if self.definition.get('requiredOneOf'): - if not set(self.definition['requiredOneOf']) & all_attribute_names: - raise InvalidMISPObject('At least one of the following attributes is required: {}'.format(', '.join(self.definition['requiredOneOf']))) - if self.definition.get('required'): - for r in self.definition.get('required'): - if r not in all_attribute_names: - raise InvalidMISPObject('{} is required'.format(r)) - return True - - def add_reference(self, destination_uuid, relationship_type, comment=None, **kwargs): - """Add a link (uuid) to an other object""" - if kwargs.get('source_uuid'): - # Load existing object - source_uuid = kwargs.get('source_uuid') - else: - # New reference - source_uuid = self.uuid - reference = MISPObjectReference() - reference.from_dict(source_uuid=source_uuid, destination_uuid=destination_uuid, - relationship_type=relationship_type, comment=comment, **kwargs) - self.ObjectReference.append(reference) - - def add_attribute(self, object_relation, **value): - if value.get('value') is None: - return None - if self.known_template: - attribute = MISPObjectAttribute(self.definition['attributes'][object_relation]) - else: - attribute = MISPObjectAttribute({}) - attribute.from_dict(object_relation, **value) - self.Attribute.append(attribute) - return attribute - - -@six.add_metaclass(abc.ABCMeta) # Remove that line when discarding python2 support. -# Python3 way: class MISPObjectGenerator(metaclass=abc.ABCMeta): -class AbstractMISPObjectGenerator(MISPObject): - - @abc.abstractmethod - def generate_attributes(self): - """Contains the logic where all the values of the object are gathered""" - pass diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 226b57d..4853c75 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -6,11 +6,24 @@ import time import json from json import JSONEncoder import os -import warnings import base64 from io import BytesIO from zipfile import ZipFile import hashlib +import abc +import sys +import uuid +from collections import Counter + +from .abstract import AbstractMISP +from .exceptions import UnknownMISPObjectTemplate, InvalidMISPObject, PyMISPError, NewEventError, NewAttributeError + + +import six # Remove that import when discarding python2 support. + +if six.PY2: + import warnings + warnings.warn("You're using python 2, it is strongly recommended to use python >=3.5") try: from dateutil.parser import parse @@ -36,14 +49,10 @@ except ImportError: except ImportError: has_pyme = False -from .exceptions import PyMISPError, NewEventError, NewAttributeError -from .defaultobjects import MISPObject - # Least dirty way to support python 2 and 3 try: basestring unicode - warnings.warn("You're using python 2, it is strongly recommended to use python >=3.5") except NameError: basestring = str unicode = str @@ -124,6 +133,10 @@ class MISPAttribute(object): return {self.uuid: False} def set_all_values(self, **kwargs): + # to be deprecated + self.from_dict(**kwargs) + + def from_dict(self, **kwargs): if kwargs.get('type') and kwargs.get('category'): if kwargs['type'] not in self.category_type_mapping[kwargs['category']]: raise NewAttributeError('{} and {} is an invalid combination, type for this category has to be in {}'.format(kwargs.get('type'), kwargs.get('category'), (', '.join(self.category_type_mapping[kwargs['category']])))) @@ -622,3 +635,174 @@ class MISPEvent(object): else: attribute.set_all_values(type=type, value=value, **kwargs) self.attributes.append(attribute) + + +class MISPObjectReference(AbstractMISP): + + attributes = ['source_uuid', 'destination_uuid', 'relationship_type', 'comment', 'uuid', 'deleted'] + + def __init__(self): + super(MISPObjectReference, self).__init__() + + def from_dict(self, source_uuid, destination_uuid, relationship_type, comment=None, **kwargs): + self.source_uuid = source_uuid + self.destination_uuid = destination_uuid + self.relationship_type = relationship_type + self.comment = comment + for k, v in kwargs: + setattr(self, k, v) + + +class MISPObjectAttribute(MISPAttribute, AbstractMISP): + + # This list is very limited and hardcoded to fit the current needs (file/pe/pesection creation): MISPAttriute will follow the + # same spec and just add one attribute: object_relation + attributes = ['object_relation', 'value', 'type', 'category', 'disable_correlation', 'to_ids', + 'data', 'encrypt', 'distribution', 'comment', 'uuid', 'event_id'] + + def __init__(self, definition): + MISPAttribute.__init__(self) + AbstractMISP.__init__(self) + self.definition = definition + + def from_dict(self, object_relation, value, **kwargs): + self.object_relation = object_relation + self.value = value + # Initialize the new MISPAttribute + # Get the misp attribute type from the definition + self.type = kwargs.pop('type', None) + if self.type is None: + self.type = self.definition.get('misp-attribute') + self.disable_correlation = kwargs.pop('disable_correlation', None) + if self.disable_correlation is None: + # The correlation can be disabled by default in the object definition. + # Use this value if it isn't overloaded by the object + self.disable_correlation = self.definition.get('disable_correlation') + self.to_ids = kwargs.pop('to_ids', None) + if self.to_ids is None: + # Same for the to_ids flag + self.to_ids = self.definition.get('to_ids') + # FIXME: dirty hack until all the classes are ported to the new format but we get the default values + kwargs.update(**self) + MISPAttribute.from_dict(self, **kwargs) + + +class MISPObject(AbstractMISP): + + attributes = ['name', 'meta-category', 'uuid', 'description', 'template_version', 'template_uuid', 'Attribute'] + + def __init__(self, name, strict=True): + super(MISPObject, self).__init__() + self.strict = strict + self.name = name + self.misp_objects_path = os.path.join( + os.path.abspath(os.path.dirname(sys.modules['pymisp'].__file__)), + 'data', 'misp-objects', 'objects') + if os.path.exists(os.path.join(self.misp_objects_path, self.name, 'definition.json')): + self.known_template = True + else: + if self.strict: + raise UnknownMISPObjectTemplate('{} is unknown in the MISP object directory.') + else: + self.known_template = False + if self.known_template: + with open(os.path.join(self.misp_objects_path, self.name, 'definition.json'), 'r') as f: + self.definition = json.load(f) + setattr(self, 'meta-category', self.definition['meta-category']) + self.template_uuid = self.definition['uuid'] + self.description = self.definition['description'] + self.template_version = self.definition['version'] + else: + # FIXME We need to set something for meta-category, template_uuid, description and template_version + pass + self.uuid = str(uuid.uuid4()) + self.Attribute = [] + self.ObjectReference = [] + + def from_dict(self, **kwargs): + if self.known_template: + if kwargs.get('template_uuid') and kwargs['template_uuid'] != self.template_uuid: + if self.strict: + raise UnknownMISPObjectTemplate('UUID of the object is different from the one of the template.') + else: + self.known_template = False + if kwargs.get('template_version') and int(kwargs['template_version']) != self.template_version: + if self.strict: + raise UnknownMISPObjectTemplate('Version of the object ({}) is different from the one of the template ({}).'.format(kwargs['template_version'], self.template_version)) + else: + self.known_template = False + + for key, value in kwargs.items(): + if key == 'Attribute': + for v in value: + self.add_attribute(**v) + elif key == 'ObjectReference': + for v in value: + self.add_reference(**v) + else: + setattr(self, key, value) + + def to_dict(self, strict=True): + if strict or self.strict and self.known_template: + self._validate() + return super(MISPObject, self).to_dict() + + def to_json(self, strict=True): + if strict or self.strict and self.known_template: + self._validate() + return super(MISPObject, self).to_json() + + def _validate(self): + """Make sure the object we're creating has the required fields""" + all_object_relations = [] + for a in self.Attribute: + all_object_relations.append(a.object_relation) + count_relations = dict(Counter(all_object_relations)) + for key, counter in count_relations.items(): + if counter == 1: + continue + if not self.definition['attributes'][key].get('multiple'): + raise InvalidMISPObject('Multiple occurrences of {} is not allowed'.format(key)) + all_attribute_names = set(count_relations.keys()) + if self.definition.get('requiredOneOf'): + if not set(self.definition['requiredOneOf']) & all_attribute_names: + raise InvalidMISPObject('At least one of the following attributes is required: {}'.format(', '.join(self.definition['requiredOneOf']))) + if self.definition.get('required'): + for r in self.definition.get('required'): + if r not in all_attribute_names: + raise InvalidMISPObject('{} is required'.format(r)) + return True + + def add_reference(self, destination_uuid, relationship_type, comment=None, **kwargs): + """Add a link (uuid) to an other object""" + if kwargs.get('source_uuid'): + # Load existing object + source_uuid = kwargs.get('source_uuid') + else: + # New reference + source_uuid = self.uuid + reference = MISPObjectReference() + reference.from_dict(source_uuid=source_uuid, destination_uuid=destination_uuid, + relationship_type=relationship_type, comment=comment, **kwargs) + self.ObjectReference.append(reference) + + def add_attribute(self, object_relation, **value): + if value.get('value') is None: + return None + if self.known_template: + attribute = MISPObjectAttribute(self.definition['attributes'][object_relation]) + else: + attribute = MISPObjectAttribute({}) + attribute.from_dict(object_relation, **value) + self.Attribute.append(attribute) + return attribute + + +@six.add_metaclass(abc.ABCMeta) # Remove that line when discarding python2 support. +# Python3 way: class MISPObjectGenerator(metaclass=abc.ABCMeta): +class AbstractMISPObjectGenerator(MISPObject): + + @abc.abstractmethod + def generate_attributes(self): + """Contains the logic where all the values of the object are gathered""" + pass diff --git a/pymisp/tools/create_misp_object.py b/pymisp/tools/create_misp_object.py index f6d82c3..dd5950d 100644 --- a/pymisp/tools/create_misp_object.py +++ b/pymisp/tools/create_misp_object.py @@ -1,8 +1,8 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from pymisp.tools import FileObject, PEObject, ELFObject, MachOObject -from pymisp.exceptions import MISPObjectException +from . import FileObject, PEObject, ELFObject, MachOObject +from ..exceptions import MISPObjectException import warnings try: diff --git a/pymisp/tools/elfobject.py b/pymisp/tools/elfobject.py index 872f554..28b0cb1 100644 --- a/pymisp/tools/elfobject.py +++ b/pymisp/tools/elfobject.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -from pymisp.defaultobjects import AbstractMISPObjectGenerator +from .. import AbstractMISPObjectGenerator from io import BytesIO from hashlib import md5, sha1, sha256, sha512 import warnings diff --git a/pymisp/tools/fileobject.py b/pymisp/tools/fileobject.py index ed496e5..95b39af 100644 --- a/pymisp/tools/fileobject.py +++ b/pymisp/tools/fileobject.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -from pymisp.defaultobjects import AbstractMISPObjectGenerator +from .. import AbstractMISPObjectGenerator import os from io import BytesIO from hashlib import md5, sha1, sha256, sha512 diff --git a/pymisp/tools/machoobject.py b/pymisp/tools/machoobject.py index 53e8e0b..84b5000 100644 --- a/pymisp/tools/machoobject.py +++ b/pymisp/tools/machoobject.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -from pymisp.defaultobjects import AbstractMISPObjectGenerator +from .. import AbstractMISPObjectGenerator from io import BytesIO from hashlib import md5, sha1, sha256, sha512 import warnings diff --git a/pymisp/tools/neo4j.py b/pymisp/tools/neo4j.py index 42f5214..823962f 100644 --- a/pymisp/tools/neo4j.py +++ b/pymisp/tools/neo4j.py @@ -3,7 +3,7 @@ import glob import os -from pymisp import MISPEvent +from .. import MISPEvent try: from py2neo import authenticate, Graph, Node, Relationship @@ -54,5 +54,5 @@ class Neo4j(): av = Relationship(attr_node, "is", val) s = val | ev | av tx.merge(s) - #tx.graph.push(s) + # tx.graph.push(s) tx.commit() diff --git a/pymisp/tools/openioc.py b/pymisp/tools/openioc.py index 3be3692..0746ccc 100755 --- a/pymisp/tools/openioc.py +++ b/pymisp/tools/openioc.py @@ -3,7 +3,7 @@ import os -from pymisp import MISPEvent +from .. import MISPEvent try: from bs4 import BeautifulSoup has_bs4 = True diff --git a/pymisp/tools/peobject.py b/pymisp/tools/peobject.py index d8ab22d..65a7123 100644 --- a/pymisp/tools/peobject.py +++ b/pymisp/tools/peobject.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -from pymisp.defaultobjects import AbstractMISPObjectGenerator +from .. import AbstractMISPObjectGenerator from io import BytesIO from hashlib import md5, sha1, sha256, sha512 from datetime import datetime From b74e558f75b5cf664f91281a9a72ccb83cf5c9ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 31 Aug 2017 10:40:18 +0200 Subject: [PATCH 23/29] Some more refactoring --- pymisp/mispevent.py | 11 ----------- pymisp/tools/abstractgenerator.py | 16 ++++++++++++++++ pymisp/tools/elfobject.py | 2 +- pymisp/tools/fileobject.py | 2 +- pymisp/tools/machoobject.py | 2 +- pymisp/tools/peobject.py | 2 +- 6 files changed, 20 insertions(+), 15 deletions(-) create mode 100644 pymisp/tools/abstractgenerator.py diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 4853c75..29ca271 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -10,7 +10,6 @@ import base64 from io import BytesIO from zipfile import ZipFile import hashlib -import abc import sys import uuid from collections import Counter @@ -796,13 +795,3 @@ class MISPObject(AbstractMISP): attribute.from_dict(object_relation, **value) self.Attribute.append(attribute) return attribute - - -@six.add_metaclass(abc.ABCMeta) # Remove that line when discarding python2 support. -# Python3 way: class MISPObjectGenerator(metaclass=abc.ABCMeta): -class AbstractMISPObjectGenerator(MISPObject): - - @abc.abstractmethod - def generate_attributes(self): - """Contains the logic where all the values of the object are gathered""" - pass diff --git a/pymisp/tools/abstractgenerator.py b/pymisp/tools/abstractgenerator.py new file mode 100644 index 0000000..2645bb5 --- /dev/null +++ b/pymisp/tools/abstractgenerator.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import abc +import six +from .. import MISPObject + + +@six.add_metaclass(abc.ABCMeta) # Remove that line when discarding python2 support. +# Python3 way: class MISPObjectGenerator(metaclass=abc.ABCMeta): +class AbstractMISPObjectGenerator(MISPObject): + + @abc.abstractmethod + def generate_attributes(self): + """Contains the logic where all the values of the object are gathered""" + pass diff --git a/pymisp/tools/elfobject.py b/pymisp/tools/elfobject.py index 28b0cb1..e5b90fa 100644 --- a/pymisp/tools/elfobject.py +++ b/pymisp/tools/elfobject.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -from .. import AbstractMISPObjectGenerator +from .abstractgenerator import AbstractMISPObjectGenerator from io import BytesIO from hashlib import md5, sha1, sha256, sha512 import warnings diff --git a/pymisp/tools/fileobject.py b/pymisp/tools/fileobject.py index 95b39af..5df7055 100644 --- a/pymisp/tools/fileobject.py +++ b/pymisp/tools/fileobject.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -from .. import AbstractMISPObjectGenerator +from .abstractgenerator import AbstractMISPObjectGenerator import os from io import BytesIO from hashlib import md5, sha1, sha256, sha512 diff --git a/pymisp/tools/machoobject.py b/pymisp/tools/machoobject.py index 84b5000..10a9bad 100644 --- a/pymisp/tools/machoobject.py +++ b/pymisp/tools/machoobject.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -from .. import AbstractMISPObjectGenerator +from .abstractgenerator import AbstractMISPObjectGenerator from io import BytesIO from hashlib import md5, sha1, sha256, sha512 import warnings diff --git a/pymisp/tools/peobject.py b/pymisp/tools/peobject.py index 65a7123..a6af720 100644 --- a/pymisp/tools/peobject.py +++ b/pymisp/tools/peobject.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -from .. import AbstractMISPObjectGenerator +from .abstractgenerator import AbstractMISPObjectGenerator from io import BytesIO from hashlib import md5, sha1, sha256, sha512 from datetime import datetime From c15b82c45ac452fa6861f565b4ce1e9baf6db289 Mon Sep 17 00:00:00 2001 From: iglocska Date: Thu, 7 Sep 2017 14:01:13 +0200 Subject: [PATCH 24/29] Changed two fields in object references - source_uuid => object_uuid - destination_uuid => referenced_uuid --- pymisp/data/misp-objects | 2 +- pymisp/mispevent.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pymisp/data/misp-objects b/pymisp/data/misp-objects index 0445ebd..fc6d54f 160000 --- a/pymisp/data/misp-objects +++ b/pymisp/data/misp-objects @@ -1 +1 @@ -Subproject commit 0445ebd350900aaaebc47309800dbc156f697592 +Subproject commit fc6d54f93b47d9c2ca951d68e568d1715e6eabf0 diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 29ca271..4e16b56 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -638,14 +638,14 @@ class MISPEvent(object): class MISPObjectReference(AbstractMISP): - attributes = ['source_uuid', 'destination_uuid', 'relationship_type', 'comment', 'uuid', 'deleted'] + attributes = ['source_uuid', 'referenced_uuid', 'relationship_type', 'comment', 'uuid', 'deleted'] def __init__(self): super(MISPObjectReference, self).__init__() - def from_dict(self, source_uuid, destination_uuid, relationship_type, comment=None, **kwargs): + def from_dict(self, source_uuid, referenced_uuid, relationship_type, comment=None, **kwargs): self.source_uuid = source_uuid - self.destination_uuid = destination_uuid + self.referenced_uuid = referenced_uuid self.relationship_type = relationship_type self.comment = comment for k, v in kwargs: @@ -772,7 +772,7 @@ class MISPObject(AbstractMISP): raise InvalidMISPObject('{} is required'.format(r)) return True - def add_reference(self, destination_uuid, relationship_type, comment=None, **kwargs): + def add_reference(self, referenced_uuid, relationship_type, comment=None, **kwargs): """Add a link (uuid) to an other object""" if kwargs.get('source_uuid'): # Load existing object @@ -781,7 +781,7 @@ class MISPObject(AbstractMISP): # New reference source_uuid = self.uuid reference = MISPObjectReference() - reference.from_dict(source_uuid=source_uuid, destination_uuid=destination_uuid, + reference.from_dict(source_uuid=source_uuid, referenced_uuid=referenced_uuid, relationship_type=relationship_type, comment=comment, **kwargs) self.ObjectReference.append(reference) From 2a5f3346e42b27dd13d91f481f899600344678cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 7 Sep 2017 14:30:05 +0200 Subject: [PATCH 25/29] Bump miso-objects --- pymisp/data/misp-objects | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/data/misp-objects b/pymisp/data/misp-objects index fc6d54f..50fe0c2 160000 --- a/pymisp/data/misp-objects +++ b/pymisp/data/misp-objects @@ -1 +1 @@ -Subproject commit fc6d54f93b47d9c2ca951d68e568d1715e6eabf0 +Subproject commit 50fe0c2993304e72d82c6cbdadd1bca4013a030e From b337e77070afdbc5997513056ffbc0b6a4a8e69d Mon Sep 17 00:00:00 2001 From: iglocska Date: Thu, 7 Sep 2017 16:09:45 +0200 Subject: [PATCH 26/29] changed source => object / destination => referenced --- pymisp/mispevent.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 4e16b56..5b2edb6 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -638,13 +638,13 @@ class MISPEvent(object): class MISPObjectReference(AbstractMISP): - attributes = ['source_uuid', 'referenced_uuid', 'relationship_type', 'comment', 'uuid', 'deleted'] + attributes = ['object_uuid', 'referenced_uuid', 'relationship_type', 'comment', 'uuid', 'deleted'] def __init__(self): super(MISPObjectReference, self).__init__() - def from_dict(self, source_uuid, referenced_uuid, relationship_type, comment=None, **kwargs): - self.source_uuid = source_uuid + def from_dict(self, object_uuid, referenced_uuid, relationship_type, comment=None, **kwargs): + self.object_uuid = object_uuid self.referenced_uuid = referenced_uuid self.relationship_type = relationship_type self.comment = comment @@ -774,14 +774,14 @@ class MISPObject(AbstractMISP): def add_reference(self, referenced_uuid, relationship_type, comment=None, **kwargs): """Add a link (uuid) to an other object""" - if kwargs.get('source_uuid'): + if kwargs.get('object_uuid'): # Load existing object - source_uuid = kwargs.get('source_uuid') + object_uuid = kwargs.get('object_uuid') else: # New reference - source_uuid = self.uuid + object_uuid = self.uuid reference = MISPObjectReference() - reference.from_dict(source_uuid=source_uuid, referenced_uuid=referenced_uuid, + reference.from_dict(object_uuid=object_uuid, referenced_uuid=referenced_uuid, relationship_type=relationship_type, comment=comment, **kwargs) self.ObjectReference.append(reference) From da2b28a927c29eed0c735ade33359af2f0dba176 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 12 Sep 2017 16:46:06 +0200 Subject: [PATCH 27/29] Use MISPAbstract as a master class everywhere. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is probably breaking everything.... ¯\_(ツ)_/¯ --- pymisp/__init__.py | 3 +- pymisp/abstract.py | 48 ++--- pymisp/api.py | 11 +- pymisp/data/misp-objects | 2 +- pymisp/mispevent.py | 407 +++++++++++++----------------------- pymisp/tools/__init__.py | 1 + pymisp/tools/elfobject.py | 50 ++--- pymisp/tools/fileobject.py | 26 +-- pymisp/tools/machoobject.py | 44 ++-- pymisp/tools/peobject.py | 64 +++--- tests/test_offline.py | 7 +- 11 files changed, 272 insertions(+), 391 deletions(-) diff --git a/pymisp/__init__.py b/pymisp/__init__.py index 6afb49e..ba8da22 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -4,7 +4,8 @@ try: from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey, InvalidMISPObject, UnknownMISPObjectTemplate # noqa from .api import PyMISP # noqa from .abstract import AbstractMISP, MISPEncode # noqa - from .mispevent import MISPEvent, MISPAttribute, EncodeUpdate, EncodeFull, MISPObjectReference, MISPObjectAttribute, MISPObject, AbstractMISPObjectGenerator # noqa + from .mispevent import MISPEvent, MISPAttribute, MISPObjectReference, MISPObjectAttribute, MISPObject # noqa + from .tools import AbstractMISPObjectGenerator # noqa from .tools import Neo4j # noqa from .tools import stix # noqa except ImportError: diff --git a/pymisp/abstract.py b/pymisp/abstract.py index 302a14d..48d1440 100644 --- a/pymisp/abstract.py +++ b/pymisp/abstract.py @@ -23,29 +23,28 @@ class MISPEncode(JSONEncoder): @six.add_metaclass(abc.ABCMeta) # Remove that line when discarding python2 support. class AbstractMISP(collections.MutableMapping): - attributes = None + __not_jsonable = [] - def __init__(self): - """Initialize the list of class-level attributes to set in the JSON dump""" - # The attribute names will be set automatically by the schemas when we will have them. - if self.attributes is None: - raise NotImplementedError('{} must define attributes'.format(type(self).__name__)) - self.attributes = sorted(self.attributes) - - def __check_dict_key(self, key): - if key not in self.attributes: - raise Exception('{} not a valid key in {}. Alowed keys: {}'.format( - key, type(self).__name__, ', '.join(self.attributes))) - return True + @property + def __properties(self): + to_return = [] + for prop, value in vars(self).items(): + if prop.startswith('_') or prop in self.__not_jsonable: + continue + to_return.append(prop) + return to_return def from_dict(self, **kwargs): - for attribute in self.attributes: - val = kwargs.pop(attribute, None) - if val is None: + for prop, value in kwargs.items(): + if value is None: continue - setattr(self, attribute, val) - if kwargs: - raise Exception('Unused parameter(s): {}'.format(', '.join(kwargs.keys()))) + setattr(self, prop, value) + + def update_not_jsonable(self, *args): + self.__not_jsonable += args + + def set_not_jsonable(self, *args): + self.__not_jsonable = args def from_json(self, json_string): """Load a JSON string""" @@ -53,7 +52,7 @@ class AbstractMISP(collections.MutableMapping): def to_dict(self): to_return = {} - for attribute in self.attributes: + for attribute in self.__properties: val = getattr(self, attribute, None) if val is None: continue @@ -67,16 +66,13 @@ class AbstractMISP(collections.MutableMapping): return json.dumps(self.to_dict(), cls=MISPEncode) def __getitem__(self, key): - if self.__check_dict_key(key): - return getattr(self, key) + return getattr(self, key) def __setitem__(self, key, value): - if self.__check_dict_key(key): - setattr(self, key, value) + setattr(self, key, value) def __delitem__(self, key): - if self.__check_dict_key(key): - delattr(self, key) + delattr(self, key) def __iter__(self): return iter(self.to_dict()) diff --git a/pymisp/api.py b/pymisp/api.py index 8d7856e..ede24ee 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -37,7 +37,8 @@ except ImportError: from . import __version__ from .exceptions import PyMISPError, SearchError, MissingDependency, NoURL, NoKey -from .mispevent import MISPEvent, MISPAttribute, EncodeUpdate +from .mispevent import MISPEvent, MISPAttribute +from .abstract import MISPEncode logger = logging.getLogger(__name__) @@ -318,7 +319,7 @@ class PyMISP(object): session = self.__prepare_session() url = urljoin(self.root_url, 'events') if isinstance(event, MISPEvent): - event = json.dumps(event, cls=EncodeUpdate) + event = json.dumps(event, cls=MISPEncode) if isinstance(event, basestring): response = session.post(url, data=event) else: @@ -334,7 +335,7 @@ class PyMISP(object): session = self.__prepare_session() url = urljoin(self.root_url, 'events/{}'.format(event_id)) if isinstance(event, MISPEvent): - event = json.dumps(event, cls=EncodeUpdate) + event = json.dumps(event, cls=MISPEncode) if isinstance(event, basestring): response = session.post(url, data=event) else: @@ -440,7 +441,7 @@ class PyMISP(object): else: session = self.__prepare_session() url = urljoin(self.root_url, 'attributes/add/{}'.format(eventID_to_update)) - response = self._check_response(session.post(url, data=json.dumps(a, cls=EncodeUpdate))) + response = self._check_response(session.post(url, data=json.dumps(a, cls=MISPEncode))) return response def add_named_attribute(self, event, type_value, value, category=None, to_ids=False, comment=None, distribution=None, proposal=False, **kwargs): @@ -737,7 +738,7 @@ class PyMISP(object): url = urljoin(self.root_url, 'shadow_attributes/{}/{}'.format(path, id)) if path in ['add', 'edit']: query = {'request': {'ShadowAttribute': attribute}} - response = session.post(url, data=json.dumps(query, cls=EncodeUpdate)) + response = session.post(url, data=json.dumps(query, cls=MISPEncode)) elif path == 'view': response = session.get(url) else: # accept or discard diff --git a/pymisp/data/misp-objects b/pymisp/data/misp-objects index 50fe0c2..96db4ae 160000 --- a/pymisp/data/misp-objects +++ b/pymisp/data/misp-objects @@ -1 +1 @@ -Subproject commit 50fe0c2993304e72d82c6cbdadd1bca4013a030e +Subproject commit 96db4ae070e0a83340a17607e9f7ebbbb20e747b diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 4e16b56..7f84995 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -4,7 +4,6 @@ import datetime import time import json -from json import JSONEncoder import os import base64 from io import BytesIO @@ -57,19 +56,26 @@ except NameError: unicode = str -class MISPAttribute(object): +def _int_to_str(d): + # transform all integer back to string + for k, v in d.items(): + if isinstance(v, (int, float)) and not isinstance(v, bool): + d[k] = str(v) + return d + + +class MISPAttribute(AbstractMISP): def __init__(self, describe_types=None): if not describe_types: - self.ressources_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data') - with open(os.path.join(self.ressources_path, 'describeTypes.json'), 'r') as f: + ressources_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data') + with open(os.path.join(ressources_path, 'describeTypes.json'), 'r') as f: t = json.load(f) describe_types = t['result'] - self.describe_types = describe_types - self.categories = describe_types['categories'] - self.types = describe_types['types'] - self.category_type_mapping = describe_types['category_type_mappings'] - self.sane_default = describe_types['sane_defaults'] + self.__categories = describe_types['categories'] + self.__types = describe_types['types'] + self.__category_type_mapping = describe_types['category_type_mappings'] + self.__sane_default = describe_types['sane_defaults'] self._reinitialize_attribute() def _reinitialize_attribute(self): @@ -137,72 +143,57 @@ class MISPAttribute(object): def from_dict(self, **kwargs): if kwargs.get('type') and kwargs.get('category'): - if kwargs['type'] not in self.category_type_mapping[kwargs['category']]: - raise NewAttributeError('{} and {} is an invalid combination, type for this category has to be in {}'.format(kwargs.get('type'), kwargs.get('category'), (', '.join(self.category_type_mapping[kwargs['category']])))) + if kwargs['type'] not in self.__category_type_mapping[kwargs['category']]: + raise NewAttributeError('{} and {} is an invalid combination, type for this category has to be in {}'.format( + kwargs.get('type'), kwargs.get('category'), (', '.join(self.__category_type_mapping[kwargs['category']])))) # Required - if kwargs.get('type'): - self.type = kwargs['type'] - if self.type not in self.types: - raise NewAttributeError('{} is invalid, type has to be in {}'.format(self.type, (', '.join(self.types)))) - elif not self.type: + self.type = kwargs.pop('type', None) + if self.type is None: raise NewAttributeError('The type of the attribute is required.') + if self.type not in self.__types: + raise NewAttributeError('{} is invalid, type has to be in {}'.format(self.type, (', '.join(self.__types)))) - type_defaults = self.sane_default[self.type] - - self.value = kwargs.get('value') + type_defaults = self.__sane_default[self.type] + self.value = kwargs.pop('value', None) if self.value is None: raise NewAttributeError('The value of the attribute is required.') # Default values - if kwargs.get('category'): - self.category = kwargs['category'] - if self.category not in self.categories: - raise NewAttributeError('{} is invalid, category has to be in {}'.format(self.category, (', '.join(self.categories)))) - else: - self.category = type_defaults['default_category'] + self.category = kwargs.pop('category', type_defaults['default_category']) + if self.category not in self.__categories: + raise NewAttributeError('{} is invalid, category has to be in {}'.format(self.category, (', '.join(self.__categories)))) - self.to_ids = kwargs.get('to_ids') - if self.to_ids is None: - self.to_ids = bool(int(type_defaults['to_ids'])) + self.to_ids = kwargs.pop('to_ids', bool(int(type_defaults['to_ids']))) if not isinstance(self.to_ids, bool): raise NewAttributeError('{} is invalid, to_ids has to be True or False'.format(self.to_ids)) - if kwargs.get('comment'): - self.comment = kwargs['comment'] if kwargs.get('distribution') is not None: - self.distribution = int(kwargs['distribution']) + self.distribution = int(kwargs.pop('distribution')) if self.distribution not in [0, 1, 2, 3, 4, 5]: raise NewAttributeError('{} is invalid, the distribution has to be in 0, 1, 2, 3, 4, 5'.format(self.distribution)) # other possible values if kwargs.get('data'): - self.data = kwargs['data'] + self.data = kwargs.pop('data') self._load_data() if kwargs.get('id'): - self.id = int(kwargs['id']) - if kwargs.get('uuid'): - self.uuid = kwargs['uuid'] + self.id = int(kwargs.pop('id')) if kwargs.get('timestamp'): - self.timestamp = datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=int(kwargs['timestamp'])) + self.timestamp = datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=int(kwargs.pop('timestamp'))) if kwargs.get('sharing_group_id'): - self.sharing_group_id = int(kwargs['sharing_group_id']) - if kwargs.get('deleted'): - self.deleted = kwargs['deleted'] - if kwargs.get('SharingGroup'): - self.SharingGroup = kwargs['SharingGroup'] - if kwargs.get('ShadowAttribute'): - self.ShadowAttribute = kwargs['ShadowAttribute'] - if kwargs.get('sig'): - self.sig = kwargs['sig'] + self.sharing_group_id = int(kwargs.pop('sharing_group_id')) if kwargs.get('Tag'): - self.Tag = [t for t in kwargs['Tag'] if t] + self.Tag = [t for t in kwargs.pop('Tag', []) if t] # If the user wants to disable correlation, let them. Defaults to False. - self.disable_correlation = kwargs.get("disable_correlation", False) + self.disable_correlation = kwargs.pop("disable_correlation", False) if self.disable_correlation is None: self.disable_correlation = False + for k, v in kwargs.items(): + setattr(self, k, v) + def _prepare_new_malware_sample(self): if '|' in self.value: # Get the filename, ignore the md5, because humans. @@ -237,87 +228,32 @@ class MISPAttribute(object): # DEPRECATED return self.to_dict() - def to_dict(self): - to_return = {'type': self.type, 'category': self.category, 'to_ids': self.to_ids, - 'distribution': self.distribution, 'value': self.value, - 'comment': self.comment, 'disable_correlation': self.disable_correlation} - if self.uuid: - to_return['uuid'] = self.uuid - if self.sig: - to_return['sig'] = self.sig - if self.sharing_group_id: - to_return['sharing_group_id'] = self.sharing_group_id - if self.Tag: - to_return['Tag'] = self.Tag - if self.data: + def to_dict(self, with_timestamp=False): + to_return = super(MISPAttribute, self).to_dict() + if to_return.get('data'): to_return['data'] = base64.b64encode(self.data.getvalue()).decode() - if self.encrypt: - to_return['encrypt'] = self.encrypt - to_return = _int_to_str(to_return) - return to_return - - def _json_full(self): - to_return = self._json() - if self.id: - to_return['id'] = self.id - if self.timestamp: - # Should never be set on an update, MISP will automatically set it to now + if with_timestamp and to_return.get('timestamp'): to_return['timestamp'] = int(time.mktime(self.timestamp.timetuple())) - if self.deleted is not None: - to_return['deleted'] = self.deleted - if self.ShadowAttribute: - to_return['ShadowAttribute'] = self.ShadowAttribute - if self.SharingGroup: - to_return['SharingGroup'] = self.SharingGroup + else: + to_return.pop('timestamp', None) to_return = _int_to_str(to_return) return to_return -class EncodeUpdate(JSONEncoder): - def default(self, obj): - try: - return obj._json() - except AttributeError: - return JSONEncoder.default(self, obj) - - -class EncodeFull(JSONEncoder): - def default(self, obj): - try: - return obj._json_full() - except AttributeError: - return JSONEncoder.default(self, obj) - - -def _int_to_str(d): - # transform all integer back to string - for k, v in d.items(): - if isinstance(v, (int, float)) and not isinstance(v, bool): - d[k] = str(v) - return d - - -class MISPEvent(object): +class MISPEvent(AbstractMISP): def __init__(self, describe_types=None, strict_validation=False): - self.ressources_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data') + ressources_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data') if strict_validation: - with open(os.path.join(self.ressources_path, 'schema.json'), 'r') as f: - self.json_schema = json.load(f) + with open(os.path.join(ressources_path, 'schema.json'), 'r') as f: + self.__json_schema = json.load(f) else: - with open(os.path.join(self.ressources_path, 'schema-lax.json'), 'r') as f: - self.json_schema = json.load(f) + with open(os.path.join(ressources_path, 'schema-lax.json'), 'r') as f: + self.__json_schema = json.load(f) if not describe_types: - with open(os.path.join(self.ressources_path, 'describeTypes.json'), 'r') as f: + with open(os.path.join(ressources_path, 'describeTypes.json'), 'r') as f: t = json.load(f) describe_types = t['result'] - self.describe_types = describe_types - self.categories = describe_types['categories'] - self.types = describe_types['types'] - self.category_type_mapping = describe_types['category_type_mappings'] - self.sane_default = describe_types['sane_defaults'] - self.new = True - self.dump_full = False self._reinitialize_event() @@ -416,8 +352,6 @@ class MISPEvent(object): self.load(f) def load(self, json_event): - self.new = False - self.dump_full = True if hasattr(json_event, 'read'): # python2 and python3 compatible to find if we have a file json_event = json_event.read() @@ -432,7 +366,7 @@ class MISPEvent(object): # Invalid event created by MISP up to 2.4.52 (attribute_count is none instead of '0') if event.get('Event') and event.get('Event').get('attribute_count') is None: event['Event']['attribute_count'] = '0' - jsonschema.validate(event, self.json_schema) + jsonschema.validate(event, self.__json_schema) e = event.get('Event') self._reinitialize_event() self.set_all_values(**e) @@ -451,150 +385,100 @@ class MISPEvent(object): raise NewEventError('Invalid format for the date: {} - {}'.format(date, type(date))) def set_all_values(self, **kwargs): + # to be deprecated + self.from_dict(**kwargs) + + def from_dict(self, **kwargs): # Required value - if kwargs.get('info'): - self.info = kwargs['info'] - elif not self.info: + self.info = kwargs.pop('info', None) + if not self.info: raise NewAttributeError('The info field of the new event is required.') # Default values for a valid event to send to a MISP instance if kwargs.get('distribution') is not None: - self.distribution = int(kwargs['distribution']) + self.distribution = int(kwargs.pop('distribution')) if self.distribution not in [0, 1, 2, 3, 4]: - raise NewEventError('{} is invalid, the distribution has to be in 0, 1, 2, 3, 4'.format(self.distribution)) + raise NewAttributeError('{} is invalid, the distribution has to be in 0, 1, 2, 3, 4'.format(self.distribution)) + if kwargs.get('threat_level_id') is not None: - self.threat_level_id = int(kwargs['threat_level_id']) + self.threat_level_id = int(kwargs.pop('threat_level_id')) if self.threat_level_id not in [1, 2, 3, 4]: raise NewEventError('{} is invalid, the threat_level has to be in 1, 2, 3, 4'.format(self.threat_level_id)) + if kwargs.get('analysis') is not None: - self.analysis = int(kwargs['analysis']) + self.analysis = int(kwargs.pop('analysis')) if self.analysis not in [0, 1, 2]: raise NewEventError('{} is invalid, the analysis has to be in 0, 1, 2'.format(self.analysis)) - if kwargs.get('published') is not None: - self.unpublish() - if kwargs.get("published") is True: + + self.published = kwargs.pop('published', None) + if self.published is True: self.publish() + else: + self.unpublish() + if kwargs.get('date'): - self.set_date(kwargs['date']) + self.set_date(kwargs.pop('date')) if kwargs.get('Attribute'): - for a in kwargs['Attribute']: - attribute = MISPAttribute(self.describe_types) + for a in kwargs.pop('Attribute'): + attribute = MISPAttribute() attribute.set_all_values(**a) self.attributes.append(attribute) # All other keys if kwargs.get('id'): - self.id = int(kwargs['id']) + self.id = int(kwargs.pop('id')) if kwargs.get('orgc_id'): - self.orgc_id = int(kwargs['orgc_id']) + self.orgc_id = int(kwargs.pop('orgc_id')) if kwargs.get('org_id'): - self.org_id = int(kwargs['org_id']) - if kwargs.get('uuid'): - self.uuid = kwargs['uuid'] - if kwargs.get('attribute_count'): - self.attribute_count = int(kwargs['attribute_count']) + self.org_id = int(kwargs.pop('org_id')) if kwargs.get('timestamp'): - self.timestamp = datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=int(kwargs['timestamp'])) - if kwargs.get('proposal_email_lock'): - self.proposal_email_lock = kwargs['proposal_email_lock'] - if kwargs.get('locked'): - self.locked = kwargs['locked'] + self.timestamp = datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=int(kwargs.pop('timestamp'))) if kwargs.get('publish_timestamp'): - self.publish_timestamp = datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=int(kwargs['publish_timestamp'])) + self.publish_timestamp = datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=int(kwargs.pop('publish_timestamp'))) if kwargs.get('sharing_group_id'): - self.sharing_group_id = int(kwargs['sharing_group_id']) - if kwargs.get('Org'): - self.Org = kwargs['Org'] - if kwargs.get('Orgc'): - self.Orgc = kwargs['Orgc'] - if kwargs.get('ShadowAttribute'): - self.ShadowAttribute = kwargs['ShadowAttribute'] + self.sharing_group_id = int(kwargs.pop('sharing_group_id')) if kwargs.get('RelatedEvent'): self.RelatedEvent = [] - for rel_event in kwargs['RelatedEvent']: + for rel_event in kwargs.pop('RelatedEvent'): sub_event = MISPEvent() sub_event.load(rel_event) self.RelatedEvent.append(sub_event) - if kwargs.get('Galaxy'): - self.Galaxy = kwargs['Galaxy'] if kwargs.get('Tag'): - self.Tag = [t for t in kwargs['Tag'] if t] - if kwargs.get('sig'): - self.sig = kwargs['sig'] - if kwargs.get('global_sig'): - self.global_sig = kwargs['global_sig'] + self.Tag = [t for t in kwargs.pop('Tag', []) if t] if kwargs.get('Object'): self.Object = [] - for obj in kwargs['Object']: + for obj in kwargs.pop('Object'): tmp_object = MISPObject(obj['name']) tmp_object.from_dict(**obj) self.Object.append(tmp_object) + for k, v in kwargs.items(): + setattr(self, k, v) + def _json(self): # DEPTECATED return self.to_dict() - def to_dict(self): - to_return = {'Event': {}} - to_return['Event'] = {'distribution': self.distribution, 'info': self.info, - 'date': self.date.isoformat(), 'published': self.published, - 'threat_level_id': self.threat_level_id, - 'analysis': self.analysis, 'Attribute': []} - if self.sig: - to_return['Event']['sig'] = self.sig - if self.global_sig: - to_return['Event']['global_sig'] = self.global_sig - if self.uuid: - to_return['Event']['uuid'] = self.uuid - if self.Tag: - to_return['Event']['Tag'] = self.Tag - if self.Orgc: - to_return['Event']['Orgc'] = self.Orgc - if self.Galaxy: - to_return['Event']['Galaxy'] = self.Galaxy - if self.sharing_group_id: - to_return['Event']['sharing_group_id'] = self.sharing_group_id - to_return['Event'] = _int_to_str(to_return['Event']) - if self.attributes: - to_return['Event']['Attribute'] = [a._json() for a in self.attributes] - jsonschema.validate(to_return, self.json_schema) - return to_return - - def _json_full(self): - to_return = self._json() - if self.id: - to_return['Event']['id'] = self.id - if self.orgc_id: - to_return['Event']['orgc_id'] = self.orgc_id - if self.org_id: - to_return['Event']['org_id'] = self.org_id - if self.locked is not None: - to_return['Event']['locked'] = self.locked - if self.attribute_count is not None: - to_return['Event']['attribute_count'] = self.attribute_count - if self.RelatedEvent: - to_return['Event']['RelatedEvent'] = [] - for rel_event in self.RelatedEvent: - to_return['Event']['RelatedEvent'].append(rel_event._json_full()) - if self.Org: - to_return['Event']['Org'] = self.Org - if self.sharing_group_id: - to_return['Event']['sharing_group_id'] = self.sharing_group_id - if self.ShadowAttribute: - to_return['Event']['ShadowAttribute'] = self.ShadowAttribute - if self.proposal_email_lock is not None: - to_return['Event']['proposal_email_lock'] = self.proposal_email_lock - if self.locked is not None: - to_return['Event']['locked'] = self.locked - if self.publish_timestamp: - to_return['Event']['publish_timestamp'] = int(time.mktime(self.publish_timestamp.timetuple())) - if self.timestamp: - # Should never be set on an update, MISP will automatically set it to now - to_return['Event']['timestamp'] = int(time.mktime(self.timestamp.timetuple())) - to_return['Event'] = _int_to_str(to_return['Event']) - if self.attributes: - to_return['Event']['Attribute'] = [a._json_full() for a in self.attributes] - jsonschema.validate(to_return, self.json_schema) + def to_dict(self, with_timestamp=False): + to_return = super(MISPEvent, self).to_dict() + if to_return.get('date'): + to_return['date'] = self.date.isoformat() + if to_return.get('attributes'): + attributes = to_return.pop('attributes') + to_return['Attribute'] = [attribute.to_dict(with_timestamp) for attribute in attributes] + if to_return.get('RelatedEvent'): + to_return['RelatedEvent'] = [rel_event.to_dict() for rel_event in self.RelatedEvent] + if with_timestamp and to_return.get('timestamp'): + to_return['timestamp'] = int(time.mktime(self.timestamp.timetuple())) + else: + to_return.pop('timestamp', None) + if with_timestamp and to_return.get('publish_timestamp'): + to_return['publish_timestamp'] = int(time.mktime(self.publish_timestamp.timetuple())) + else: + to_return.pop('publish_timestamp', None) + to_return = _int_to_str(to_return) + to_return = {'Event': to_return} + jsonschema.validate(to_return, self.__json_schema) return to_return def add_tag(self, tag): @@ -627,7 +511,7 @@ class MISPEvent(object): raise Exception('No attribute with UUID/ID {} found.'.format(attribute_id)) def add_attribute(self, type, value, **kwargs): - attribute = MISPAttribute(self.describe_types) + attribute = MISPAttribute() if isinstance(value, list): for a in value: self.add_attribute(type, a, **kwargs) @@ -638,8 +522,6 @@ class MISPEvent(object): class MISPObjectReference(AbstractMISP): - attributes = ['source_uuid', 'referenced_uuid', 'relationship_type', 'comment', 'uuid', 'deleted'] - def __init__(self): super(MISPObjectReference, self).__init__() @@ -652,17 +534,11 @@ class MISPObjectReference(AbstractMISP): setattr(self, k, v) -class MISPObjectAttribute(MISPAttribute, AbstractMISP): - - # This list is very limited and hardcoded to fit the current needs (file/pe/pesection creation): MISPAttriute will follow the - # same spec and just add one attribute: object_relation - attributes = ['object_relation', 'value', 'type', 'category', 'disable_correlation', 'to_ids', - 'data', 'encrypt', 'distribution', 'comment', 'uuid', 'event_id'] +class MISPObjectAttribute(MISPAttribute): def __init__(self, definition): - MISPAttribute.__init__(self) - AbstractMISP.__init__(self) - self.definition = definition + super(MISPAttribute, self).__init__() + self.__definition = definition def from_dict(self, object_relation, value, **kwargs): self.object_relation = object_relation @@ -671,46 +547,43 @@ class MISPObjectAttribute(MISPAttribute, AbstractMISP): # Get the misp attribute type from the definition self.type = kwargs.pop('type', None) if self.type is None: - self.type = self.definition.get('misp-attribute') + self.type = self.__definition.get('misp-attribute') self.disable_correlation = kwargs.pop('disable_correlation', None) if self.disable_correlation is None: # The correlation can be disabled by default in the object definition. # Use this value if it isn't overloaded by the object - self.disable_correlation = self.definition.get('disable_correlation') + self.disable_correlation = self.__definition.get('disable_correlation') self.to_ids = kwargs.pop('to_ids', None) if self.to_ids is None: # Same for the to_ids flag - self.to_ids = self.definition.get('to_ids') - # FIXME: dirty hack until all the classes are ported to the new format but we get the default values + self.to_ids = self.__definition.get('to_ids') kwargs.update(**self) - MISPAttribute.from_dict(self, **kwargs) + super(MISPAttribute, self).from_dict(**kwargs) class MISPObject(AbstractMISP): - attributes = ['name', 'meta-category', 'uuid', 'description', 'template_version', 'template_uuid', 'Attribute'] - def __init__(self, name, strict=True): super(MISPObject, self).__init__() - self.strict = strict + self.__strict = strict self.name = name - self.misp_objects_path = os.path.join( + self.__misp_objects_path = os.path.join( os.path.abspath(os.path.dirname(sys.modules['pymisp'].__file__)), 'data', 'misp-objects', 'objects') - if os.path.exists(os.path.join(self.misp_objects_path, self.name, 'definition.json')): - self.known_template = True + if os.path.exists(os.path.join(self.__misp_objects_path, self.name, 'definition.json')): + self.__known_template = True else: - if self.strict: + if self.__strict: raise UnknownMISPObjectTemplate('{} is unknown in the MISP object directory.') else: - self.known_template = False - if self.known_template: - with open(os.path.join(self.misp_objects_path, self.name, 'definition.json'), 'r') as f: - self.definition = json.load(f) - setattr(self, 'meta-category', self.definition['meta-category']) - self.template_uuid = self.definition['uuid'] - self.description = self.definition['description'] - self.template_version = self.definition['version'] + self.__known_template = False + if self.__known_template: + with open(os.path.join(self.__misp_objects_path, self.name, 'definition.json'), 'r') as f: + self.__definition = json.load(f) + setattr(self, 'meta-category', self.__definition['meta-category']) + self.template_uuid = self.__definition['uuid'] + self.description = self.__definition['description'] + self.template_version = self.__definition['version'] else: # FIXME We need to set something for meta-category, template_uuid, description and template_version pass @@ -719,17 +592,17 @@ class MISPObject(AbstractMISP): self.ObjectReference = [] def from_dict(self, **kwargs): - if self.known_template: + if self.__known_template: if kwargs.get('template_uuid') and kwargs['template_uuid'] != self.template_uuid: - if self.strict: + if self.__strict: raise UnknownMISPObjectTemplate('UUID of the object is different from the one of the template.') else: - self.known_template = False + self.__known_template = False if kwargs.get('template_version') and int(kwargs['template_version']) != self.template_version: if self.strict: raise UnknownMISPObjectTemplate('Version of the object ({}) is different from the one of the template ({}).'.format(kwargs['template_version'], self.template_version)) else: - self.known_template = False + self.__known_template = False for key, value in kwargs.items(): if key == 'Attribute': @@ -742,12 +615,12 @@ class MISPObject(AbstractMISP): setattr(self, key, value) def to_dict(self, strict=True): - if strict or self.strict and self.known_template: + if strict or self.__strict and self.__known_template: self._validate() return super(MISPObject, self).to_dict() def to_json(self, strict=True): - if strict or self.strict and self.known_template: + if strict or self.__strict and self.__known_template: self._validate() return super(MISPObject, self).to_json() @@ -760,14 +633,14 @@ class MISPObject(AbstractMISP): for key, counter in count_relations.items(): if counter == 1: continue - if not self.definition['attributes'][key].get('multiple'): + if not self.__definition['attributes'][key].get('multiple'): raise InvalidMISPObject('Multiple occurrences of {} is not allowed'.format(key)) all_attribute_names = set(count_relations.keys()) - if self.definition.get('requiredOneOf'): - if not set(self.definition['requiredOneOf']) & all_attribute_names: - raise InvalidMISPObject('At least one of the following attributes is required: {}'.format(', '.join(self.definition['requiredOneOf']))) - if self.definition.get('required'): - for r in self.definition.get('required'): + if self.__definition.get('requiredOneOf'): + if not set(self.__definition['requiredOneOf']) & all_attribute_names: + raise InvalidMISPObject('At least one of the following attributes is required: {}'.format(', '.join(self.__definition['requiredOneOf']))) + if self.__definition.get('required'): + for r in self.__definition.get('required'): if r not in all_attribute_names: raise InvalidMISPObject('{} is required'.format(r)) return True @@ -788,8 +661,8 @@ class MISPObject(AbstractMISP): def add_attribute(self, object_relation, **value): if value.get('value') is None: return None - if self.known_template: - attribute = MISPObjectAttribute(self.definition['attributes'][object_relation]) + if self.__known_template: + attribute = MISPObjectAttribute(self.__definition['attributes'][object_relation]) else: attribute = MISPObjectAttribute({}) attribute.from_dict(object_relation, **value) diff --git a/pymisp/tools/__init__.py b/pymisp/tools/__init__.py index 3a73be5..a0c667a 100644 --- a/pymisp/tools/__init__.py +++ b/pymisp/tools/__init__.py @@ -4,3 +4,4 @@ from .peobject import PEObject, PESectionObject # noqa from .elfobject import ELFObject, ELFSectionObject # noqa from .machoobject import MachOObject, MachOSectionObject # noqa from .create_misp_object import make_binary_objects # noqa +from .abstractgenerator import AbstractMISPObjectGenerator # noqa diff --git a/pymisp/tools/elfobject.py b/pymisp/tools/elfobject.py index e5b90fa..3a812a4 100644 --- a/pymisp/tools/elfobject.py +++ b/pymisp/tools/elfobject.py @@ -29,35 +29,35 @@ class ELFObject(AbstractMISPObjectGenerator): raise ImportError('Please install lief, documentation here: https://github.com/lief-project/LIEF') if pseudofile: if isinstance(pseudofile, BytesIO): - self.elf = lief.ELF.parse(raw=pseudofile.getvalue()) + self.__elf = lief.ELF.parse(raw=pseudofile.getvalue()) elif isinstance(pseudofile, bytes): - self.elf = lief.ELF.parse(raw=pseudofile) + self.__elf = lief.ELF.parse(raw=pseudofile) else: raise Exception('Pseudo file can be BytesIO or bytes got {}'.format(type(pseudofile))) elif filepath: - self.elf = lief.ELF.parse(filepath) + self.__elf = lief.ELF.parse(filepath) elif parsed: # Got an already parsed blob if isinstance(parsed, lief.ELF.Binary): - self.elf = parsed + self.__elf = parsed else: raise Exception('Not a lief.ELF.Binary: {}'.format(type(parsed))) - # Python3 way - # super().__init__('elf') super(ELFObject, self).__init__('elf') self.generate_attributes() + # Mark as non_jsonable because we need to add them manually + self.update_not_jsonable('ObjectReference') def generate_attributes(self): # General information - self.add_attribute('type', value=str(self.elf.header.file_type).split('.')[1]) - self.add_attribute('entrypoint-address', value=self.elf.entrypoint) - self.add_attribute('arch', value=str(self.elf.header.machine_type).split('.')[1]) - self.add_attribute('os_abi', value=str(self.elf.header.identity_os_abi).split('.')[1]) + self.add_attribute('type', value=str(self.__elf.header.file_type).split('.')[1]) + self.add_attribute('entrypoint-address', value=self.__elf.entrypoint) + self.add_attribute('arch', value=str(self.__elf.header.machine_type).split('.')[1]) + self.add_attribute('os_abi', value=str(self.__elf.header.identity_os_abi).split('.')[1]) # Sections self.sections = [] - if self.elf.sections: + if self.__elf.sections: pos = 0 - for section in self.elf.sections: + for section in self.__elf.sections: s = ELFSectionObject(section) self.add_reference(s.uuid, 'included-in', 'Section {} of ELF'.format(pos)) pos += 1 @@ -71,21 +71,23 @@ class ELFSectionObject(AbstractMISPObjectGenerator): # Python3 way # super().__init__('pe-section') super(ELFSectionObject, self).__init__('elf-section') - self.section = section - self.data = bytes(self.section.content) + self.__section = section + self.__data = bytes(self.__section.content) self.generate_attributes() + # Mark as non_jsonable because we need to add them manually + self.update_not_jsonable('ObjectReference') def generate_attributes(self): - self.add_attribute('name', value=self.section.name) - self.add_attribute('type', value=str(self.section.type).split('.')[1]) - for flag in self.section.flags_list: + self.add_attribute('name', value=self.__section.name) + self.add_attribute('type', value=str(self.__section.type).split('.')[1]) + for flag in self.__section.flags_list: self.add_attribute('flag', value=str(flag).split('.')[1]) - size = self.add_attribute('size-in-bytes', value=self.section.size) + size = self.add_attribute('size-in-bytes', value=self.__section.size) if int(size.value) > 0: - self.add_attribute('entropy', value=self.section.entropy) - self.add_attribute('md5', value=md5(self.data).hexdigest()) - self.add_attribute('sha1', value=sha1(self.data).hexdigest()) - self.add_attribute('sha256', value=sha256(self.data).hexdigest()) - self.add_attribute('sha512', value=sha512(self.data).hexdigest()) + self.add_attribute('entropy', value=self.__section.entropy) + self.add_attribute('md5', value=md5(self.__data).hexdigest()) + self.add_attribute('sha1', value=sha1(self.__data).hexdigest()) + self.add_attribute('sha256', value=sha256(self.__data).hexdigest()) + self.add_attribute('sha512', value=sha512(self.__data).hexdigest()) if HAS_PYDEEP: - self.add_attribute('ssdeep', value=pydeep.hash_buf(self.data).decode()) + self.add_attribute('ssdeep', value=pydeep.hash_buf(self.__data).decode()) diff --git a/pymisp/tools/fileobject.py b/pymisp/tools/fileobject.py index 5df7055..dd5d025 100644 --- a/pymisp/tools/fileobject.py +++ b/pymisp/tools/fileobject.py @@ -33,34 +33,36 @@ class FileObject(AbstractMISPObjectGenerator): self.filepath = filepath self.filename = os.path.basename(self.filepath) with open(filepath, 'rb') as f: - self.pseudofile = BytesIO(f.read()) + self.__pseudofile = BytesIO(f.read()) elif pseudofile and isinstance(pseudofile, BytesIO): # WARNING: lief.parse requires a path self.filepath = None - self.pseudofile = pseudofile + self.__pseudofile = pseudofile self.filename = filename else: raise Exception('File buffer (BytesIO) or a path is required.') # PY3 way: # super().__init__('file') super(FileObject, self).__init__('file') - self.data = self.pseudofile.getvalue() + self.__data = self.__pseudofile.getvalue() self.generate_attributes() + # Mark as non_jsonable because we need to add them manually + self.update_not_jsonable('ObjectReference') def generate_attributes(self): self.add_attribute('filename', value=self.filename) - size = self.add_attribute('size-in-bytes', value=len(self.data)) + size = self.add_attribute('size-in-bytes', value=len(self.__data)) if int(size.value) > 0: - self.add_attribute('entropy', value=self.__entropy_H(self.data)) - self.add_attribute('md5', value=md5(self.data).hexdigest()) - self.add_attribute('sha1', value=sha1(self.data).hexdigest()) - self.add_attribute('sha256', value=sha256(self.data).hexdigest()) - self.add_attribute('sha512', value=sha512(self.data).hexdigest()) - self.add_attribute('malware-sample', value=self.filename, data=self.pseudofile) + self.add_attribute('entropy', value=self.__entropy_H(self.__data)) + self.add_attribute('md5', value=md5(self.__data).hexdigest()) + self.add_attribute('sha1', value=sha1(self.__data).hexdigest()) + self.add_attribute('sha256', value=sha256(self.__data).hexdigest()) + self.add_attribute('sha512', value=sha512(self.__data).hexdigest()) + self.add_attribute('malware-sample', value=self.filename, data=self.__pseudofile) if HAS_MAGIC: - self.add_attribute('mimetype', value=magic.from_buffer(self.data)) + self.add_attribute('mimetype', value=magic.from_buffer(self.__data)) if HAS_PYDEEP: - self.add_attribute('ssdeep', value=pydeep.hash_buf(self.data).decode()) + self.add_attribute('ssdeep', value=pydeep.hash_buf(self.__data).decode()) def __entropy_H(self, data): """Calculate the entropy of a chunk of data.""" diff --git a/pymisp/tools/machoobject.py b/pymisp/tools/machoobject.py index 10a9bad..ccebd9f 100644 --- a/pymisp/tools/machoobject.py +++ b/pymisp/tools/machoobject.py @@ -29,35 +29,37 @@ class MachOObject(AbstractMISPObjectGenerator): raise ImportError('Please install lief, documentation here: https://github.com/lief-project/LIEF') if pseudofile: if isinstance(pseudofile, BytesIO): - self.macho = lief.MachO.parse(raw=pseudofile.getvalue()) + self.__macho = lief.MachO.parse(raw=pseudofile.getvalue()) elif isinstance(pseudofile, bytes): - self.macho = lief.MachO.parse(raw=pseudofile) + self.__macho = lief.MachO.parse(raw=pseudofile) else: raise Exception('Pseudo file can be BytesIO or bytes got {}'.format(type(pseudofile))) elif filepath: - self.macho = lief.MachO.parse(filepath) + self.__macho = lief.MachO.parse(filepath) elif parsed: # Got an already parsed blob if isinstance(parsed, lief.MachO.Binary): - self.macho = parsed + self.__macho = parsed else: raise Exception('Not a lief.MachO.Binary: {}'.format(type(parsed))) # Python3 way # super().__init__('elf') super(MachOObject, self).__init__('macho') self.generate_attributes() + # Mark as non_jsonable because we need to add them manually + self.update_not_jsonable(['ObjectReference']) def generate_attributes(self): - self.add_attribute('type', value=str(self.macho.header.file_type).split('.')[1]) - self.add_attribute('name', value=self.macho.name) + self.add_attribute('type', value=str(self.__macho.header.file_type).split('.')[1]) + self.add_attribute('name', value=self.__macho.name) # General information - if self.macho.has_entrypoint: - self.add_attribute('entrypoint-address', value=self.macho.entrypoint) + if self.__macho.has_entrypoint: + self.add_attribute('entrypoint-address', value=self.__macho.entrypoint) # Sections self.sections = [] - if self.macho.sections: + if self.__macho.sections: pos = 0 - for section in self.macho.sections: + for section in self.__macho.sections: s = MachOSectionObject(section) self.add_reference(s.uuid, 'included-in', 'Section {} of MachO'.format(pos)) pos += 1 @@ -71,18 +73,20 @@ class MachOSectionObject(AbstractMISPObjectGenerator): # Python3 way # super().__init__('pe-section') super(MachOSectionObject, self).__init__('macho-section') - self.section = section - self.data = bytes(self.section.content) + self.__section = section + self.__data = bytes(self.__section.content) self.generate_attributes() + # Mark as non_jsonable because we need to add them manually + self.update_not_jsonable(['ObjectReference']) def generate_attributes(self): - self.add_attribute('name', value=self.section.name) - size = self.add_attribute('size-in-bytes', value=self.section.size) + self.add_attribute('name', value=self.__section.name) + size = self.add_attribute('size-in-bytes', value=self.__section.size) if int(size.value) > 0: - self.add_attribute('entropy', value=self.section.entropy) - self.add_attribute('md5', value=md5(self.data).hexdigest()) - self.add_attribute('sha1', value=sha1(self.data).hexdigest()) - self.add_attribute('sha256', value=sha256(self.data).hexdigest()) - self.add_attribute('sha512', value=sha512(self.data).hexdigest()) + self.add_attribute('entropy', value=self.__section.entropy) + self.add_attribute('md5', value=md5(self.__data).hexdigest()) + self.add_attribute('sha1', value=sha1(self.__data).hexdigest()) + self.add_attribute('sha256', value=sha256(self.__data).hexdigest()) + self.add_attribute('sha512', value=sha512(self.__data).hexdigest()) if HAS_PYDEEP: - self.add_attribute('ssdeep', value=pydeep.hash_buf(self.data).decode()) + self.add_attribute('ssdeep', value=pydeep.hash_buf(self.__data).decode()) diff --git a/pymisp/tools/peobject.py b/pymisp/tools/peobject.py index a6af720..2786e45 100644 --- a/pymisp/tools/peobject.py +++ b/pymisp/tools/peobject.py @@ -30,36 +30,38 @@ class PEObject(AbstractMISPObjectGenerator): raise ImportError('Please install lief, documentation here: https://github.com/lief-project/LIEF') if pseudofile: if isinstance(pseudofile, BytesIO): - self.pe = lief.PE.parse(raw=pseudofile.getvalue()) + self.__pe = lief.PE.parse(raw=pseudofile.getvalue()) elif isinstance(pseudofile, bytes): - self.pe = lief.PE.parse(raw=pseudofile) + self.__pe = lief.PE.parse(raw=pseudofile) else: raise Exception('Pseudo file can be BytesIO or bytes got {}'.format(type(pseudofile))) elif filepath: - self.pe = lief.PE.parse(filepath) + self.__pe = lief.PE.parse(filepath) elif parsed: # Got an already parsed blob if isinstance(parsed, lief.PE.Binary): - self.pe = parsed + self.__pe = parsed else: raise Exception('Not a lief.PE.Binary: {}'.format(type(parsed))) # Python3 way # super().__init__('pe') super(PEObject, self).__init__('pe') self.generate_attributes() + # Mark as non_jsonable because we need to add them manually + self.update_not_jsonable('ObjectReference') def _is_exe(self): if not self._is_dll() and not self._is_driver(): - return self.pe.header.has_characteristic(lief.PE.HEADER_CHARACTERISTICS.EXECUTABLE_IMAGE) + return self.__pe.header.has_characteristic(lief.PE.HEADER_CHARACTERISTICS.EXECUTABLE_IMAGE) return False def _is_dll(self): - return self.pe.header.has_characteristic(lief.PE.HEADER_CHARACTERISTICS.DLL) + return self.__pe.header.has_characteristic(lief.PE.HEADER_CHARACTERISTICS.DLL) def _is_driver(self): # List from pefile system_DLLs = set(('ntoskrnl.exe', 'hal.dll', 'ndis.sys', 'bootvid.dll', 'kdcom.dll')) - if system_DLLs.intersection([imp.lower() for imp in self.pe.libraries]): + if system_DLLs.intersection([imp.lower() for imp in self.__pe.libraries]): return True return False @@ -76,20 +78,20 @@ class PEObject(AbstractMISPObjectGenerator): def generate_attributes(self): self.add_attribute('type', value=self._get_pe_type()) # General information - self.add_attribute('entrypoint-address', value=self.pe.entrypoint) - self.add_attribute('compilation-timestamp', value=datetime.utcfromtimestamp(self.pe.header.time_date_stamps).isoformat()) - # self.imphash = self.pe.get_imphash() + self.add_attribute('entrypoint-address', value=self.__pe.entrypoint) + self.add_attribute('compilation-timestamp', value=datetime.utcfromtimestamp(self.__pe.header.time_date_stamps).isoformat()) + # self.imphash = self.__pe.get_imphash() try: - if (self.pe.has_resources and - self.pe.resources_manager.has_version and - self.pe.resources_manager.version.has_string_file_info and - self.pe.resources_manager.version.string_file_info.langcode_items): - fileinfo = dict(self.pe.resources_manager.version.string_file_info.langcode_items[0].items.items()) + if (self.__pe.has_resources and + self.__pe.resources_manager.has_version and + self.__pe.resources_manager.version.has_string_file_info and + self.__pe.resources_manager.version.string_file_info.langcode_items): + fileinfo = dict(self.__pe.resources_manager.version.string_file_info.langcode_items[0].items.items()) self.add_attribute('original-filename', value=fileinfo.get('OriginalFilename')) self.add_attribute('internal-filename', value=fileinfo.get('InternalName')) self.add_attribute('file-description', value=fileinfo.get('FileDescription')) self.add_attribute('file-version', value=fileinfo.get('FileVersion')) - self.add_attribute('lang-id', value=self.pe.resources_manager.version.string_file_info.langcode_items[0].key) + self.add_attribute('lang-id', value=self.__pe.resources_manager.version.string_file_info.langcode_items[0].key) self.add_attribute('product-name', value=fileinfo.get('ProductName')) self.add_attribute('product-version', value=fileinfo.get('ProductVersion')) self.add_attribute('company-name', value=fileinfo.get('CompanyName')) @@ -99,13 +101,13 @@ class PEObject(AbstractMISPObjectGenerator): pass # Sections self.sections = [] - if self.pe.sections: + if self.__pe.sections: pos = 0 - for section in self.pe.sections: + for section in self.__pe.sections: s = PESectionObject(section) self.add_reference(s.uuid, 'included-in', 'Section {} of PE'.format(pos)) - if ((self.pe.entrypoint >= section.virtual_address) and - (self.pe.entrypoint < (section.virtual_address + section.virtual_size))): + if ((self.__pe.entrypoint >= section.virtual_address) and + (self.__pe.entrypoint < (section.virtual_address + section.virtual_size))): self.add_attribute('entrypoint-section-at-position', value='{}|{}'.format(section.name, pos)) pos += 1 self.sections.append(s) @@ -119,18 +121,20 @@ class PESectionObject(AbstractMISPObjectGenerator): # Python3 way # super().__init__('pe-section') super(PESectionObject, self).__init__('pe-section') - self.section = section - self.data = bytes(self.section.content) + self.__section = section + self.__data = bytes(self.__section.content) self.generate_attributes() + # Mark as non_jsonable because we need to add them manually + self.update_not_jsonable('ObjectReference') def generate_attributes(self): - self.add_attribute('name', value=self.section.name) - size = self.add_attribute('size-in-bytes', value=self.section.size) + self.add_attribute('name', value=self.__section.name) + size = self.add_attribute('size-in-bytes', value=self.__section.size) if int(size.value) > 0: - self.add_attribute('entropy', value=self.section.entropy) - self.add_attribute('md5', value=md5(self.data).hexdigest()) - self.add_attribute('sha1', value=sha1(self.data).hexdigest()) - self.add_attribute('sha256', value=sha256(self.data).hexdigest()) - self.add_attribute('sha512', value=sha512(self.data).hexdigest()) + self.add_attribute('entropy', value=self.__section.entropy) + self.add_attribute('md5', value=md5(self.__data).hexdigest()) + self.add_attribute('sha1', value=sha1(self.__data).hexdigest()) + self.add_attribute('sha256', value=sha256(self.__data).hexdigest()) + self.add_attribute('sha512', value=sha512(self.__data).hexdigest()) if HAS_PYDEEP: - self.add_attribute('ssdeep', value=pydeep.hash_buf(self.data).decode()) + self.add_attribute('ssdeep', value=pydeep.hash_buf(self.__data).decode()) diff --git a/tests/test_offline.py b/tests/test_offline.py index b8eb097..8a704f6 100644 --- a/tests/test_offline.py +++ b/tests/test_offline.py @@ -10,10 +10,8 @@ import pymisp as pm from pymisp import PyMISP # from pymisp import NewEventError from pymisp import MISPEvent -from pymisp import EncodeUpdate -from pymisp import EncodeFull - from pymisp import MISPEncode + from pymisp.tools import make_binary_objects @@ -135,8 +133,7 @@ class TestOffline(unittest.TestCase): misp_event = MISPEvent(pymisp.describe_types) with open('tests/57c4445b-c548-4654-af0b-4be3950d210f.json', 'r') as f: misp_event.load(f.read()) - json.dumps(misp_event, cls=EncodeUpdate) - json.dumps(misp_event, cls=EncodeFull) + json.dumps(misp_event, cls=MISPEncode) def test_searchIndexByTagId(self, m): self.initURI(m) From 6dfaa787223df6ef76b9c8eb18f21d6221134bbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 14 Sep 2017 14:34:53 +0200 Subject: [PATCH 28/29] Allow to get the list of known types out of MISPEvent again --- pymisp/mispevent.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 081bb7a..4718c4a 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -255,6 +255,8 @@ class MISPEvent(AbstractMISP): t = json.load(f) describe_types = t['result'] + self.__types = describe_types['types'] + self._reinitialize_event() def _reinitialize_event(self): @@ -288,6 +290,9 @@ class MISPEvent(AbstractMISP): self.Galaxy = None self.Object = None + def get_known_types(self): + return self.__types + def _serialize(self): return '{date}{threat_level_id}{info}{uuid}{analysis}{timestamp}'.format( date=self.date, threat_level_id=self.threat_level_id, info=self.info, From 9abaed0499a7764359b6a082544519c39b713024 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 18 Sep 2017 11:37:09 +0100 Subject: [PATCH 29/29] Bump misp object --- pymisp/data/misp-objects | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymisp/data/misp-objects b/pymisp/data/misp-objects index 96db4ae..d22ced3 160000 --- a/pymisp/data/misp-objects +++ b/pymisp/data/misp-objects @@ -1 +1 @@ -Subproject commit 96db4ae070e0a83340a17607e9f7ebbbb20e747b +Subproject commit d22ced3b82c4bf3012bf0162831d862685944c9a