From 09573997991a387264cd6d6d869d98acbe6225aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 25 Jul 2019 14:53:23 +0200 Subject: [PATCH] new: Allow to change the template on an object on-the-fly Related: #425 --- pymisp/mispevent.py | 67 ++- .../overwrite_file/definition.json | 457 ++++++++++++++++++ tests/testlive_comprehensive.py | 21 +- 3 files changed, 517 insertions(+), 28 deletions(-) create mode 100644 tests/mispevent_testfiles/overwrite_file/definition.json diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 4069390..df30ee2 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -1075,35 +1075,23 @@ class MISPObject(AbstractMISP): super(MISPObject, self).__init__(**kwargs) self._strict = strict self.name = name - misp_objects_path = os.path.join( - os.path.abspath(os.path.dirname(sys.modules['pymisp'].__file__)), - 'data', 'misp-objects', 'objects') - misp_objects_path_custom = kwargs.get('misp_objects_path_custom') - if misp_objects_path_custom and os.path.exists(os.path.join(misp_objects_path_custom, self.name, 'definition.json')): - # Use the local object path by default if provided (allows to overwrite a default template) - template_path = os.path.join(misp_objects_path_custom, self.name, 'definition.json') - self._known_template = True - elif os.path.exists(os.path.join(misp_objects_path, self.name, 'definition.json')): - template_path = os.path.join(misp_objects_path, self.name, 'definition.json') - self._known_template = True - else: - if self._strict: - raise UnknownMISPObjectTemplate('{} is unknown in the MISP object directory.'.format(self.name)) - else: - self._known_template = False - if self._known_template: - with open(template_path, 'rb') as f: - if OLD_PY3: - self._definition = json.loads(f.read().decode()) - else: - self._definition = json.load(f) - setattr(self, 'meta-category', self._definition['meta-category']) - self.template_uuid = self._definition['uuid'] - self.description = self._definition['description'] - self.template_version = self._definition['version'] + self._known_template = False + + if kwargs.get('misp_objects_path_custom'): + # If misp_objects_path_custom is given, and an object with the given name exists, use that. + self._known_template = self._load_template_path(os.path.join(kwargs.get('misp_objects_path_custom'), self.name, 'definition.json')) + + if not self._known_template: + # Check if the object is known in the default templates bundled in with PyMISP + misp_objects_path = os.path.join(os.path.abspath(os.path.dirname(sys.modules['pymisp'].__file__)), 'data', 'misp-objects', 'objects') + self._known_template = self._load_template_path(os.path.join(misp_objects_path, self.name, 'definition.json')) + + if not self._known_template and self._strict: + raise UnknownMISPObjectTemplate('{} is unknown in the MISP object directory.'.format(self.name)) else: # Then we have no meta-category, template_uuid, description and template_version pass + self.uuid = str(uuid.uuid4()) self.__fast_attribute_access = defaultdict(list) # Hashtable object_relation: [attributes] self.ObjectReference = [] @@ -1137,6 +1125,33 @@ class MISPObject(AbstractMISP): # Mark as non_jsonable because we need to add the references manually after the object(s) have been created self.update_not_jsonable('ObjectReference') + def _load_template_path(self, template_path): + if not os.path.exists(template_path): + return False + with open(template_path, 'rb') as f: + if OLD_PY3: + self._definition = json.loads(f.read().decode()) + else: + self._definition = json.load(f) + setattr(self, 'meta-category', self._definition['meta-category']) + self.template_uuid = self._definition['uuid'] + self.description = self._definition['description'] + self.template_version = self._definition['version'] + return True + + def force_misp_objects_path_custom(self, misp_objects_path_custom, object_name=None): + if object_name: + self.name = object_name + template_path = os.path.join(misp_objects_path_custom, self.name, 'definition.json') + + self._known_template = self._load_template_path(template_path) + if not self._known_template: + raise UnknownMISPObjectTemplate('{} is unknown in the MISP object directory ({}).'.format(self.name, template_path)) + + @property + def disable_validation(self): + self._strict = False + @property def attributes(self): return self.Attribute diff --git a/tests/mispevent_testfiles/overwrite_file/definition.json b/tests/mispevent_testfiles/overwrite_file/definition.json new file mode 100644 index 0000000..162b773 --- /dev/null +++ b/tests/mispevent_testfiles/overwrite_file/definition.json @@ -0,0 +1,457 @@ +{ + "requiredOneOf": [ + "filename", + "size-in-bytes", + "authentihash", + "ssdeep", + "md5", + "sha1", + "sha224", + "sha256", + "sha384", + "sha512", + "sha512/224", + "sha512/256", + "tlsh", + "pattern-in-file", + "certificate", + "malware-sample", + "attachment", + "path", + "fullpath" + ], + "required": [ + "test_overwrite" + ], + "attributes": { + "test_overwrite": { + "description": "Test attribute", + "misp-attribute": "text" + }, + "md5": { + "description": "[Insecure] MD5 hash (128 bits)", + "ui-priority": 1, + "misp-attribute": "md5", + "recommended": false + }, + "sha1": { + "description": "[Insecure] Secure Hash Algorithm 1 (160 bits)", + "ui-priority": 1, + "misp-attribute": "sha1", + "recommended": false + }, + "sha224": { + "description": "Secure Hash Algorithm 2 (224 bits)", + "ui-priority": 0, + "misp-attribute": "sha224", + "recommended": false + }, + "sha256": { + "description": "Secure Hash Algorithm 2 (256 bits)", + "ui-priority": 1, + "misp-attribute": "sha256" + }, + "sha384": { + "description": "Secure Hash Algorithm 2 (384 bits)", + "ui-priority": 0, + "misp-attribute": "sha384", + "recommended": false + }, + "sha512": { + "description": "Secure Hash Algorithm 2 (512 bits)", + "ui-priority": 1, + "misp-attribute": "sha512" + }, + "sha512/224": { + "description": "Secure Hash Algorithm 2 (224 bits)", + "ui-priority": 0, + "misp-attribute": "sha512/224", + "recommended": false + }, + "sha512/256": { + "description": "Secure Hash Algorithm 2 (256 bits)", + "ui-priority": 0, + "misp-attribute": "sha512/256", + "recommended": false + }, + "ssdeep": { + "description": "Fuzzy hash using context triggered piecewise hashes (CTPH)", + "ui-priority": 0, + "misp-attribute": "ssdeep" + }, + "authentihash": { + "description": "Authenticode executable signature hash", + "ui-priority": 0, + "misp-attribute": "authentihash", + "recommended": false + }, + "size-in-bytes": { + "description": "Size of the file, in bytes", + "disable_correlation": true, + "ui-priority": 0, + "misp-attribute": "size-in-bytes" + }, + "entropy": { + "description": "Entropy of the whole file", + "disable_correlation": true, + "ui-priority": 1, + "misp-attribute": "float" + }, + "pattern-in-file": { + "description": "Pattern that can be found in the file", + "categories": [ + "Artifacts dropped", + "Payload installation", + "External analysis" + ], + "ui-priority": 1, + "misp-attribute": "pattern-in-file", + "multiple": true + }, + "text": { + "description": "Free text value to attach to the file", + "disable_correlation": true, + "ui-priority": 1, + "misp-attribute": "text", + "recommended": false + }, + "malware-sample": { + "description": "The file itself (binary)", + "ui-priority": 1, + "misp-attribute": "malware-sample" + }, + "attachment": { + "description": "A non-malicious file.", + "ui-priority": 1, + "misp-attribute": "attachment" + }, + "filename": { + "description": "Filename on disk", + "disable_correlation": true, + "multiple": true, + "categories": [ + "Payload delivery", + "Artifacts dropped", + "Payload installation", + "External analysis" + ], + "ui-priority": 1, + "misp-attribute": "filename" + }, + "path": { + "description": "Path of the filename complete or partial", + "disable_correlation": true, + "multiple": true, + "ui-priority": 0, + "misp-attribute": "text" + }, + "fullpath": { + "description": "Complete path of the filename including the filename", + "multiple": true, + "ui-priority": 0, + "misp-attribute": "text" + }, + "tlsh": { + "description": "Fuzzy hash by Trend Micro: Locality Sensitive Hash", + "ui-priority": 0, + "misp-attribute": "tlsh" + }, + "certificate": { + "description": "Certificate value if the binary is signed with another authentication scheme than authenticode", + "ui-priority": 0, + "misp-attribute": "x509-fingerprint-sha1" + }, + "mimetype": { + "description": "Mime type", + "disable_correlation": true, + "ui-priority": 0, + "misp-attribute": "mime-type" + }, + "state": { + "misp-attribute": "text", + "ui-priority": 0, + "description": "State of the file", + "multiple": true, + "disable_correlation": true, + "values_list": [ + "Malicious", + "Harmless", + "Signed", + "Revoked", + "Expired", + "Trusted" + ] + }, + "file-encoding": { + "misp-attribute": "text", + "ui-priority": 0, + "description": "Encoding format of the file", + "disable_correlation": true, + "sane_default": [ + "Adobe-Standard-Encoding", + "Adobe-Symbol-Encoding", + "Amiga-1251", + "ANSI_X3.110-1983", + "ASMO_449", + "Big5", + "Big5-HKSCS", + "BOCU-1", + "BRF", + "BS_4730", + "BS_viewdata", + "CESU-8", + "CP50220", + "CP51932", + "CSA_Z243.4-1985-1", + "CSA_Z243.4-1985-2", + "CSA_Z243.4-1985-gr", + "CSN_369103", + "DEC-MCS", + "DIN_66003", + "dk-us", + "DS_2089", + "EBCDIC-AT-DE", + "EBCDIC-AT-DE-A", + "EBCDIC-CA-FR", + "EBCDIC-DK-NO", + "EBCDIC-DK-NO-A", + "EBCDIC-ES", + "EBCDIC-ES-A", + "EBCDIC-ES-S", + "EBCDIC-FI-SE", + "EBCDIC-FI-SE-A", + "EBCDIC-FR", + "EBCDIC-IT", + "EBCDIC-PT", + "EBCDIC-UK", + "EBCDIC-US", + "ECMA-cyrillic", + "ES", + "ES2", + "EUC-KR", + "Extended_UNIX_Code_Fixed_Width_for_Japanese", + "Extended_UNIX_Code_Packed_Format_for_Japanese", + "GB18030", + "GB_1988-80", + "GB2312", + "GB_2312-80", + "GBK", + "GOST_19768-74", + "greek7", + "greek7-old", + "greek-ccitt", + "HP-DeskTop", + "HP-Legal", + "HP-Math8", + "HP-Pi-font", + "hp-roman8", + "HZ-GB-2312", + "IBM00858", + "IBM00924", + "IBM01140", + "IBM01141", + "IBM01142", + "IBM01143", + "IBM01144", + "IBM01145", + "IBM01146", + "IBM01147", + "IBM01148", + "IBM01149", + "IBM037", + "IBM038", + "IBM1026", + "IBM1047", + "IBM273", + "IBM274", + "IBM275", + "IBM277", + "IBM278", + "IBM280", + "IBM281", + "IBM284", + "IBM285", + "IBM290", + "IBM297", + "IBM420", + "IBM423", + "IBM424", + "IBM437", + "IBM500", + "IBM775", + "IBM850", + "IBM851", + "IBM852", + "IBM855", + "IBM857", + "IBM860", + "IBM861", + "IBM862", + "IBM863", + "IBM864", + "IBM865", + "IBM866", + "IBM868", + "IBM869", + "IBM870", + "IBM871", + "IBM880", + "IBM891", + "IBM903", + "IBM904", + "IBM905", + "IBM918", + "IBM-Symbols", + "IBM-Thai", + "IEC_P27-1", + "INIS", + "INIS-8", + "INIS-cyrillic", + "INVARIANT", + "ISO_10367-box", + "ISO-10646-J-1", + "ISO-10646-UCS-2", + "ISO-10646-UCS-4", + "ISO-10646-UCS-Basic", + "ISO-10646-Unicode-Latin1", + "ISO-10646-UTF-1", + "ISO-11548-1", + "ISO-2022-CN", + "ISO-2022-CN-EXT", + "ISO-2022-JP", + "ISO-2022-JP-2", + "ISO-2022-KR", + "ISO_2033-1983", + "ISO_5427", + "ISO_5427:1981", + "ISO_5428:1980", + "ISO_646.basic:1983", + "ISO_646.irv:1983", + "ISO_6937-2-25", + "ISO_6937-2-add", + "ISO-8859-10", + "ISO_8859-1:1987", + "ISO-8859-13", + "ISO-8859-14", + "ISO-8859-15", + "ISO-8859-16", + "ISO-8859-1-Windows-3.0-Latin-1", + "ISO-8859-1-Windows-3.1-Latin-1", + "ISO_8859-2:1987", + "ISO-8859-2-Windows-Latin-2", + "ISO_8859-3:1988", + "ISO_8859-4:1988", + "ISO_8859-5:1988", + "ISO_8859-6:1987", + "ISO_8859-6-E", + "ISO_8859-6-I", + "ISO_8859-7:1987", + "ISO_8859-8:1988", + "ISO_8859-8-E", + "ISO_8859-8-I", + "ISO_8859-9:1989", + "ISO-8859-9-Windows-Latin-5", + "ISO_8859-supp", + "iso-ir-90", + "ISO-Unicode-IBM-1261", + "ISO-Unicode-IBM-1264", + "ISO-Unicode-IBM-1265", + "ISO-Unicode-IBM-1268", + "ISO-Unicode-IBM-1276", + "IT", + "JIS_C6220-1969-jp", + "JIS_C6220-1969-ro", + "JIS_C6226-1978", + "JIS_C6226-1983", + "JIS_C6229-1984-a", + "JIS_C6229-1984-b", + "JIS_C6229-1984-b-add", + "JIS_C6229-1984-hand", + "JIS_C6229-1984-hand-add", + "JIS_C6229-1984-kana", + "JIS_Encoding", + "JIS_X0201", + "JIS_X0212-1990", + "JUS_I.B1.002", + "JUS_I.B1.003-mac", + "JUS_I.B1.003-serb", + "KOI7-switched", + "KOI8-R", + "KOI8-U", + "KS_C_5601-1987", + "KSC5636", + "KZ-1048", + "latin-greek", + "Latin-greek-1", + "latin-lap", + "macintosh", + "Microsoft-Publishing", + "MNEM", + "MNEMONIC", + "MSZ_7795.3", + "Name", + "NATS-DANO", + "NATS-DANO-ADD", + "NATS-SEFI", + "NATS-SEFI-ADD", + "NC_NC00-10:81", + "NF_Z_62-010", + "NF_Z_62-010_(1973)", + "NS_4551-1", + "NS_4551-2", + "OSD_EBCDIC_DF03_IRV", + "OSD_EBCDIC_DF04_1", + "OSD_EBCDIC_DF04_15", + "PC8-Danish-Norwegian", + "PC8-Turkish", + "PT", + "PT2", + "PTCP154", + "SCSU", + "SEN_850200_B", + "SEN_850200_C", + "Shift_JIS", + "T.101-G2", + "T.61-7bit", + "T.61-8bit", + "TIS-620", + "TSCII", + "UNICODE-1-1", + "UNICODE-1-1-UTF-7", + "UNKNOWN-8BIT", + "US-ASCII", + "us-dk", + "UTF-16", + "UTF-16BE", + "UTF-16LE", + "UTF-32", + "UTF-32BE", + "UTF-32LE", + "UTF-7", + "UTF-8", + "Ventura-International", + "Ventura-Math", + "Ventura-US", + "videotex-suppl", + "VIQR", + "VISCII", + "windows-1250", + "windows-1251", + "windows-1252", + "windows-1253", + "windows-1254", + "windows-1255", + "windows-1256", + "windows-1257", + "windows-1258", + "Windows-31J", + "windows-874" + ] + } + }, + "version": 17, + "description": "File object describing a file with meta-information", + "meta-category": "file", + "uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "name": "file" +} diff --git a/tests/testlive_comprehensive.py b/tests/testlive_comprehensive.py index e354c0f..571976e 100644 --- a/tests/testlive_comprehensive.py +++ b/tests/testlive_comprehensive.py @@ -966,6 +966,23 @@ class TestComprehensive(unittest.TestCase): self.admin_misp_connector.delete_event(first.id) self.admin_misp_connector.delete_event(second.id) + def test_custom_template(self): + first = self.create_simple_event() + try: + with open('tests/viper-test-files/test_files/whoami.exe', 'rb') as f: + first.add_attribute('malware-sample', value='whoami.exe', data=BytesIO(f.read()), expand='binary') + first.run_expansions() + first = self.admin_misp_connector.add_event(first, pythonify=True) + self.assertEqual(len(first.objects), 7) + file_object = first.get_objects_by_name('file')[0] + file_object.force_misp_objects_path_custom('tests/mispevent_testfiles', 'overwrite_file') + file_object.add_attribute('test_overwrite', 'blah') + obj = self.admin_misp_connector.update_object(file_object, pythonify=True) + self.assertEqual(obj.get_attributes_by_relation('test_overwrite')[0].value, 'blah') + finally: + # Delete event + self.admin_misp_connector.delete_event(first.id) + def test_unknown_template(self): first = self.create_simple_event() attributeAsDict = [{'MyCoolAttribute': {'value': 'critical thing', 'type': 'text'}}, @@ -1110,10 +1127,10 @@ class TestComprehensive(unittest.TestCase): for s in sections: first.add_object(s) self.assertEqual(len(first.objects[0].references), 1) - self.assertEqual(first.objects[0].references[0].relationship_type, 'included-in') + self.assertEqual(first.objects[0].references[0].relationship_type, 'includes') first = self.user_misp_connector.update_event(first) self.assertEqual(len(first.objects[0].references), 1) - self.assertEqual(first.objects[0].references[0].relationship_type, 'included-in') + self.assertEqual(first.objects[0].references[0].relationship_type, 'includes') finally: # Delete event self.admin_misp_connector.delete_event(first.id)