MISP
/
PyMISP
mirror of https://github.com/MISP/PyMISP
2
0
Fork 0
Code Issues Releases Wiki Activity

new: Allow to change the template on an object on-the-fly 

Related: #425
pull/428/head
Raphaël Vinot 2019-07-25 14:53:23 +02:00
parent d477a3688c
commit 0957399799
3 changed files with 517 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
pymisp/mispevent.py
View File

@ -1075,35 +1075,23 @@ class MISPObject(AbstractMISP):
super(MISPObject, self).__init__(**kwargs)
self._strict = strict
self.name = name
misp_objects_path = os.path.join(
os.path.abspath(os.path.dirname(sys.modules['pymisp'].__file__)),
'data', 'misp-objects', 'objects')
misp_objects_path_custom = kwargs.get('misp_objects_path_custom')
if misp_objects_path_custom and os.path.exists(os.path.join(misp_objects_path_custom, self.name, 'definition.json')):
# Use the local object path by default if provided (allows to overwrite a default template)
template_path = os.path.join(misp_objects_path_custom, self.name, 'definition.json')
self._known_template = True
elif os.path.exists(os.path.join(misp_objects_path, self.name, 'definition.json')):
template_path = os.path.join(misp_objects_path, self.name, 'definition.json')
self._known_template = True
else:
if self._strict:
raise UnknownMISPObjectTemplate('{} is unknown in the MISP object directory.'.format(self.name))
else:
self._known_template = False
if self._known_template:
with open(template_path, 'rb') as f:
if OLD_PY3:
self._definition = json.loads(f.read().decode())
else:
self._definition = json.load(f)
setattr(self, 'meta-category', self._definition['meta-category'])
self.template_uuid = self._definition['uuid']
self.description = self._definition['description']
self.template_version = self._definition['version']
if kwargs.get('misp_objects_path_custom'):
# If misp_objects_path_custom is given, and an object with the given name exists, use that.
self._known_template = self._load_template_path(os.path.join(kwargs.get('misp_objects_path_custom'), self.name, 'definition.json'))
if not self._known_template:
# Check if the object is known in the default templates bundled in with PyMISP
misp_objects_path = os.path.join(os.path.abspath(os.path.dirname(sys.modules['pymisp'].__file__)), 'data', 'misp-objects', 'objects')
self._known_template = self._load_template_path(os.path.join(misp_objects_path, self.name, 'definition.json'))
if not self._known_template and self._strict:
raise UnknownMISPObjectTemplate('{} is unknown in the MISP object directory.'.format(self.name))
else:
# Then we have no meta-category, template_uuid, description and template_version
pass
self.uuid = str(uuid.uuid4())
self.__fast_attribute_access = defaultdict(list) # Hashtable object_relation: [attributes]
self.ObjectReference = []
@ -1137,6 +1125,33 @@ class MISPObject(AbstractMISP):
# Mark as non_jsonable because we need to add the references manually after the object(s) have been created
self.update_not_jsonable('ObjectReference')
def _load_template_path(self, template_path):
if not os.path.exists(template_path):
return False
with open(template_path, 'rb') as f:
if OLD_PY3:
self._definition = json.loads(f.read().decode())
else:
self._definition = json.load(f)
setattr(self, 'meta-category', self._definition['meta-category'])
self.template_uuid = self._definition['uuid']
self.description = self._definition['description']
self.template_version = self._definition['version']
return True
def force_misp_objects_path_custom(self, misp_objects_path_custom, object_name=None):
if object_name:
self.name = object_name
template_path = os.path.join(misp_objects_path_custom, self.name, 'definition.json')
self._known_template = self._load_template_path(template_path)
if not self._known_template:
raise UnknownMISPObjectTemplate('{} is unknown in the MISP object directory ({}).'.format(self.name, template_path))
@property
def disable_validation(self):
self._strict = False
@property
def attributes(self):
return self.Attribute

457
tests/mispevent_testfiles/overwrite_file/definition.json Normal file
View File

@ -0,0 +1,457 @@
{
"requiredOneOf": [
"filename",
"size-in-bytes",
"authentihash",
"ssdeep",
"md5",
"sha1",
"sha224",
"sha256",
"sha384",
"sha512",
"sha512/224",
"sha512/256",
"tlsh",
"pattern-in-file",
"certificate",
"malware-sample",
"attachment",
"path",
"fullpath"
],
"required": [
"test_overwrite"
],
"attributes": {
"test_overwrite": {
"description": "Test attribute",
"misp-attribute": "text"
},
"md5": {
"description": "[Insecure] MD5 hash (128 bits)",
"ui-priority": 1,
"misp-attribute": "md5",
"recommended": false
},
"sha1": {
"description": "[Insecure] Secure Hash Algorithm 1 (160 bits)",
"ui-priority": 1,
"misp-attribute": "sha1",
"recommended": false
},
"sha224": {
"description": "Secure Hash Algorithm 2 (224 bits)",
"ui-priority": 0,
"misp-attribute": "sha224",
"recommended": false
},
"sha256": {
"description": "Secure Hash Algorithm 2 (256 bits)",
"ui-priority": 1,
"misp-attribute": "sha256"
},
"sha384": {
"description": "Secure Hash Algorithm 2 (384 bits)",
"ui-priority": 0,
"misp-attribute": "sha384",
"recommended": false
},
"sha512": {
"description": "Secure Hash Algorithm 2 (512 bits)",
"ui-priority": 1,
"misp-attribute": "sha512"
},
"sha512/224": {
"description": "Secure Hash Algorithm 2 (224 bits)",
"ui-priority": 0,
"misp-attribute": "sha512/224",
"recommended": false
},
"sha512/256": {
"description": "Secure Hash Algorithm 2 (256 bits)",
"ui-priority": 0,
"misp-attribute": "sha512/256",
"recommended": false
},
"ssdeep": {
"description": "Fuzzy hash using context triggered piecewise hashes (CTPH)",
"ui-priority": 0,
"misp-attribute": "ssdeep"
},
"authentihash": {
"description": "Authenticode executable signature hash",
"ui-priority": 0,
"misp-attribute": "authentihash",
"recommended": false
},
"size-in-bytes": {
"description": "Size of the file, in bytes",
"disable_correlation": true,
"ui-priority": 0,
"misp-attribute": "size-in-bytes"
},
"entropy": {
"description": "Entropy of the whole file",
"disable_correlation": true,
"ui-priority": 1,
"misp-attribute": "float"
},
"pattern-in-file": {
"description": "Pattern that can be found in the file",
"categories": [
"Artifacts dropped",
"Payload installation",
"External analysis"
],
"ui-priority": 1,
"misp-attribute": "pattern-in-file",
"multiple": true
},
"text": {
"description": "Free text value to attach to the file",
"disable_correlation": true,
"ui-priority": 1,
"misp-attribute": "text",
"recommended": false
},
"malware-sample": {
"description": "The file itself (binary)",
"ui-priority": 1,
"misp-attribute": "malware-sample"
},
"attachment": {
"description": "A non-malicious file.",
"ui-priority": 1,
"misp-attribute": "attachment"
},
"filename": {
"description": "Filename on disk",
"disable_correlation": true,
"multiple": true,
"categories": [
"Payload delivery",
"Artifacts dropped",
"Payload installation",
"External analysis"
],
"ui-priority": 1,
"misp-attribute": "filename"
},
"path": {
"description": "Path of the filename complete or partial",
"disable_correlation": true,
"multiple": true,
"ui-priority": 0,
"misp-attribute": "text"
},
"fullpath": {
"description": "Complete path of the filename including the filename",
"multiple": true,
"ui-priority": 0,
"misp-attribute": "text"
},
"tlsh": {
"description": "Fuzzy hash by Trend Micro: Locality Sensitive Hash",
"ui-priority": 0,
"misp-attribute": "tlsh"
},
"certificate": {
"description": "Certificate value if the binary is signed with another authentication scheme than authenticode",
"ui-priority": 0,
"misp-attribute": "x509-fingerprint-sha1"
},
"mimetype": {
"description": "Mime type",
"disable_correlation": true,
"ui-priority": 0,
"misp-attribute": "mime-type"
},
"state": {
"misp-attribute": "text",
"ui-priority": 0,
"description": "State of the file",
"multiple": true,
"disable_correlation": true,
"values_list": [
"Malicious",
"Harmless",
"Signed",
"Revoked",
"Expired",
"Trusted"
]
},
"file-encoding": {
"misp-attribute": "text",
"ui-priority": 0,
"description": "Encoding format of the file",
"disable_correlation": true,
"sane_default": [
"Adobe-Standard-Encoding",
"Adobe-Symbol-Encoding",
"Amiga-1251",
"ANSI_X3.110-1983",
"ASMO_449",
"Big5",
"Big5-HKSCS",
"BOCU-1",
"BRF",
"BS_4730",
"BS_viewdata",
"CESU-8",
"CP50220",
"CP51932",
"CSA_Z243.4-1985-1",
"CSA_Z243.4-1985-2",
"CSA_Z243.4-1985-gr",
"CSN_369103",
"DEC-MCS",
"DIN_66003",
"dk-us",
"DS_2089",
"EBCDIC-AT-DE",
"EBCDIC-AT-DE-A",
"EBCDIC-CA-FR",
"EBCDIC-DK-NO",
"EBCDIC-DK-NO-A",
"EBCDIC-ES",
"EBCDIC-ES-A",
"EBCDIC-ES-S",
"EBCDIC-FI-SE",
"EBCDIC-FI-SE-A",
"EBCDIC-FR",
"EBCDIC-IT",
"EBCDIC-PT",
"EBCDIC-UK",
"EBCDIC-US",
"ECMA-cyrillic",
"ES",
"ES2",
"EUC-KR",
"Extended_UNIX_Code_Fixed_Width_for_Japanese",
"Extended_UNIX_Code_Packed_Format_for_Japanese",
"GB18030",
"GB_1988-80",
"GB2312",
"GB_2312-80",
"GBK",
"GOST_19768-74",
"greek7",
"greek7-old",
"greek-ccitt",
"HP-DeskTop",
"HP-Legal",
"HP-Math8",
"HP-Pi-font",
"hp-roman8",
"HZ-GB-2312",
"IBM00858",
"IBM00924",
"IBM01140",
"IBM01141",
"IBM01142",
"IBM01143",
"IBM01144",
"IBM01145",
"IBM01146",
"IBM01147",
"IBM01148",
"IBM01149",
"IBM037",
"IBM038",
"IBM1026",
"IBM1047",
"IBM273",
"IBM274",
"IBM275",
"IBM277",
"IBM278",
"IBM280",
"IBM281",
"IBM284",
"IBM285",
"IBM290",
"IBM297",
"IBM420",
"IBM423",
"IBM424",
"IBM437",
"IBM500",
"IBM775",
"IBM850",
"IBM851",
"IBM852",
"IBM855",
"IBM857",
"IBM860",
"IBM861",
"IBM862",
"IBM863",
"IBM864",
"IBM865",
"IBM866",
"IBM868",
"IBM869",
"IBM870",
"IBM871",
"IBM880",
"IBM891",
"IBM903",
"IBM904",
"IBM905",
"IBM918",
"IBM-Symbols",
"IBM-Thai",
"IEC_P27-1",
"INIS",
"INIS-8",
"INIS-cyrillic",
"INVARIANT",
"ISO_10367-box",
"ISO-10646-J-1",
"ISO-10646-UCS-2",
"ISO-10646-UCS-4",
"ISO-10646-UCS-Basic",
"ISO-10646-Unicode-Latin1",
"ISO-10646-UTF-1",
"ISO-11548-1",
"ISO-2022-CN",
"ISO-2022-CN-EXT",
"ISO-2022-JP",
"ISO-2022-JP-2",
"ISO-2022-KR",
"ISO_2033-1983",
"ISO_5427",
"ISO_5427:1981",
"ISO_5428:1980",
"ISO_646.basic:1983",
"ISO_646.irv:1983",
"ISO_6937-2-25",
"ISO_6937-2-add",
"ISO-8859-10",
"ISO_8859-1:1987",
"ISO-8859-13",
"ISO-8859-14",
"ISO-8859-15",
"ISO-8859-16",
"ISO-8859-1-Windows-3.0-Latin-1",
"ISO-8859-1-Windows-3.1-Latin-1",
"ISO_8859-2:1987",
"ISO-8859-2-Windows-Latin-2",
"ISO_8859-3:1988",
"ISO_8859-4:1988",
"ISO_8859-5:1988",
"ISO_8859-6:1987",
"ISO_8859-6-E",
"ISO_8859-6-I",
"ISO_8859-7:1987",
"ISO_8859-8:1988",
"ISO_8859-8-E",
"ISO_8859-8-I",
"ISO_8859-9:1989",
"ISO-8859-9-Windows-Latin-5",
"ISO_8859-supp",
"iso-ir-90",
"ISO-Unicode-IBM-1261",
"ISO-Unicode-IBM-1264",
"ISO-Unicode-IBM-1265",
"ISO-Unicode-IBM-1268",
"ISO-Unicode-IBM-1276",
"IT",
"JIS_C6220-1969-jp",
"JIS_C6220-1969-ro",
"JIS_C6226-1978",
"JIS_C6226-1983",
"JIS_C6229-1984-a",
"JIS_C6229-1984-b",
"JIS_C6229-1984-b-add",
"JIS_C6229-1984-hand",
"JIS_C6229-1984-hand-add",
"JIS_C6229-1984-kana",
"JIS_Encoding",
"JIS_X0201",
"JIS_X0212-1990",
"JUS_I.B1.002",
"JUS_I.B1.003-mac",
"JUS_I.B1.003-serb",
"KOI7-switched",
"KOI8-R",
"KOI8-U",
"KS_C_5601-1987",
"KSC5636",
"KZ-1048",
"latin-greek",
"Latin-greek-1",
"latin-lap",
"macintosh",
"Microsoft-Publishing",
"MNEM",
"MNEMONIC",
"MSZ_7795.3",
"Name",
"NATS-DANO",
"NATS-DANO-ADD",
"NATS-SEFI",
"NATS-SEFI-ADD",
"NC_NC00-10:81",
"NF_Z_62-010",
"NF_Z_62-010_(1973)",
"NS_4551-1",
"NS_4551-2",
"OSD_EBCDIC_DF03_IRV",
"OSD_EBCDIC_DF04_1",
"OSD_EBCDIC_DF04_15",
"PC8-Danish-Norwegian",
"PC8-Turkish",
"PT",
"PT2",
"PTCP154",
"SCSU",
"SEN_850200_B",
"SEN_850200_C",
"Shift_JIS",
"T.101-G2",
"T.61-7bit",
"T.61-8bit",
"TIS-620",
"TSCII",
"UNICODE-1-1",
"UNICODE-1-1-UTF-7",
"UNKNOWN-8BIT",
"US-ASCII",
"us-dk",
"UTF-16",
"UTF-16BE",
"UTF-16LE",
"UTF-32",
"UTF-32BE",
"UTF-32LE",
"UTF-7",
"UTF-8",
"Ventura-International",
"Ventura-Math",
"Ventura-US",
"videotex-suppl",
"VIQR",
"VISCII",
"windows-1250",
"windows-1251",
"windows-1252",
"windows-1253",
"windows-1254",
"windows-1255",
"windows-1256",
"windows-1257",
"windows-1258",
"Windows-31J",
"windows-874"
]
}
},
"version": 17,
"description": "File object describing a file with meta-information",
"meta-category": "file",
"uuid": "688c46fb-5edb-40a3-8273-1af7923e2215",
"name": "file"
}

21
tests/testlive_comprehensive.py
View File

@ -966,6 +966,23 @@ class TestComprehensive(unittest.TestCase):
self.admin_misp_connector.delete_event(first.id)
self.admin_misp_connector.delete_event(second.id)
def test_custom_template(self):
first = self.create_simple_event()
try:
with open('tests/viper-test-files/test_files/whoami.exe', 'rb') as f:
first.add_attribute('malware-sample', value='whoami.exe', data=BytesIO(f.read()), expand='binary')
first.run_expansions()
first = self.admin_misp_connector.add_event(first, pythonify=True)
self.assertEqual(len(first.objects), 7)
file_object = first.get_objects_by_name('file')[0]
file_object.force_misp_objects_path_custom('tests/mispevent_testfiles', 'overwrite_file')
file_object.add_attribute('test_overwrite', 'blah')
obj = self.admin_misp_connector.update_object(file_object, pythonify=True)
self.assertEqual(obj.get_attributes_by_relation('test_overwrite')[0].value, 'blah')
finally:
# Delete event
self.admin_misp_connector.delete_event(first.id)
def test_unknown_template(self):
first = self.create_simple_event()
attributeAsDict = [{'MyCoolAttribute': {'value': 'critical thing', 'type': 'text'}},
@ -1110,10 +1127,10 @@ class TestComprehensive(unittest.TestCase):
for s in sections:
first.add_object(s)
self.assertEqual(len(first.objects[0].references), 1)
self.assertEqual(first.objects[0].references[0].relationship_type, 'included-in')
self.assertEqual(first.objects[0].references[0].relationship_type, 'includes')
first = self.user_misp_connector.update_event(first)
self.assertEqual(len(first.objects[0].references), 1)
self.assertEqual(first.objects[0].references[0].relationship_type, 'included-in')
self.assertEqual(first.objects[0].references[0].relationship_type, 'includes')
finally:
# Delete event
self.admin_misp_connector.delete_event(first.id)