MISP
/
PyMISP
mirror of https://github.com/MISP/PyMISP
2
0
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'perf'

pull/475/head
Raphaël Vinot 2019-10-10 11:20:25 +02:00
parent a6ecd524d8 2785d0027d
commit c80d35fa75
20 changed files with 423 additions and 358 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Pipfile
View File

@ -10,6 +10,7 @@ codecov = "*"
requests-mock = "*" requests-mock = "*"
pymisp = {editable = true,extras = ["fileobjects", "neo", "openioc", "virustotal", "pdfexport", "docs"],path = "."} pymisp = {editable = true,extras = ["fileobjects", "neo", "openioc", "virustotal", "pdfexport", "docs"],path = "."}
docutils = "==0.15" docutils = "==0.15"
memory-profiler = "*"
[packages] [packages]
pymisp = {editable = true,extras = ["fileobjects", "openioc", "virustotal", "pdfexport"],path = "."} pymisp = {editable = true,extras = ["fileobjects", "openioc", "virustotal", "pdfexport"],path = "."}

282
Pipfile.lock generated
View File

@ -1,7 +1,7 @@
{ {
"_meta": { "_meta": {
"hash": { "hash": {
"sha256": "4b4cf20ef3242efd0c24d7cc54ba2438dee8ba853ab3b9384ad915448ce83048" "sha256": "4be7259a433785d74e1879a4a555bb669d50c5f409d0a094652c1abc9b1227c5"
}, },
"pipfile-spec": 6, "pipfile-spec": 6,
"requires": { "requires": {
@ -25,11 +25,10 @@
}, },
"beautifulsoup4": { "beautifulsoup4": {
"hashes": [ "hashes": [
"sha256:05668158c7b85b791c5abde53e50265e16f98ad601c402ba44d70f96c4159612", "sha256:5279c36b4b2ec2cb4298d723791467e3000e5384a43ea0cdf5d45207c7e97169",
"sha256:25288c9e176f354bf277c0a10aa96c782a6a18a17122dba2e8cec4a97e03343b", "sha256:dcdef580e18a76d54002088602eba453eec38ebbcafafeaabd8cab12b6155d57"
"sha256:f040590be10520f2ea4c2ae8c3dae441c7cfff5308ec9d58a0ec0c1b8f81d469"
], ],
"version": "==4.8.0" "version": "==4.8.1"
}, },
"certifi": { "certifi": {
"hashes": [ "hashes": [
@ -92,34 +91,34 @@
}, },
"pillow": { "pillow": {
"hashes": [ "hashes": [
"sha256:0804f77cb1e9b6dbd37601cee11283bba39a8d44b9ddb053400c58e0c0d7d9de", "sha256:00fdeb23820f30e43bba78eb9abb00b7a937a655de7760b2e09101d63708b64e",
"sha256:0ab7c5b5d04691bcbd570658667dd1e21ca311c62dcfd315ad2255b1cd37f64f", "sha256:01f948e8220c85eae1aa1a7f8edddcec193918f933fb07aaebe0bfbbcffefbf1",
"sha256:0b3e6cf3ea1f8cecd625f1420b931c83ce74f00c29a0ff1ce4385f99900ac7c4", "sha256:08abf39948d4b5017a137be58f1a52b7101700431f0777bec3d897c3949f74e6",
"sha256:365c06a45712cd723ec16fa4ceb32ce46ad201eb7bbf6d3c16b063c72b61a3ed", "sha256:099a61618b145ecb50c6f279666bbc398e189b8bc97544ae32b8fcb49ad6b830",
"sha256:38301fbc0af865baa4752ddae1bb3cbb24b3d8f221bf2850aad96b243306fa03", "sha256:2c1c61546e73de62747e65807d2cc4980c395d4c5600ecb1f47a650c6fa78c79",
"sha256:3aef1af1a91798536bbab35d70d35750bd2884f0832c88aeb2499aa2d1ed4992", "sha256:2ed9c4f694861642401f27dc3cb99772be67cd190e84845c749dae0a06c3bfae",
"sha256:3fe0ab49537d9330c9bba7f16a5f8b02da615b5c809cdf7124f356a0f182eccd", "sha256:338581b30b908e111be578f0297255f6b57a51358cd16fa0e6f664c9a1f88bff",
"sha256:45a619d5c1915957449264c81c008934452e3fd3604e36809212300b2a4dab68", "sha256:38c7d48a21cd06fdeee93987147b9b1c55b73b4cfcbf83240568bfbd5adee447",
"sha256:49f90f147883a0c3778fd29d3eb169d56416f25758d0f66775db9184debc8010", "sha256:43fd026f613c8e48a25eba1a92f4d2ad7f3903c95d8c33a11611a7717d2ab654",
"sha256:571b5a758baf1cb6a04233fb23d6cf1ca60b31f9f641b1700bfaab1194020555", "sha256:4548236844327a718ce3bb182ab32a16fa2050c61e334e959f554cac052fb0df",
"sha256:5ac381e8b1259925287ccc5a87d9cf6322a2dc88ae28a97fe3e196385288413f", "sha256:5090857876c58885cfa388dc649e5db30aae98a068c26f3fd0ac9d7d9a4d9572",
"sha256:6153db744a743c0c8c91b8e3b9d40e0b13a5d31dbf8a12748c6d9bfd3ddc01ad", "sha256:5bbba34f97a26a93f5e8dec469ca4ddd712451418add43da946dbaed7f7a98d2",
"sha256:6fd63afd14a16f5d6b408f623cc2142917a1f92855f0df997e09a49f0341be8a", "sha256:65a28969a025a0eb4594637b6103201dc4ed2a9508bdab56ac33e43e3081c404",
"sha256:70acbcaba2a638923c2d337e0edea210505708d7859b87c2bd81e8f9902ae826", "sha256:892bb52b70bd5ea9dbbc3ac44f38e84f5a04e9d8b1bff48159d96cb795b81159",
"sha256:70b1594d56ed32d56ed21a7fbb2a5c6fd7446cdb7b21e749c9791eac3a64d9e4", "sha256:8a9becd5cbd5062f973bcd2e7bc79483af310222de112b6541f8af1f93a3cc42",
"sha256:76638865c83b1bb33bcac2a61ce4d13c17dba2204969dedb9ab60ef62bede686", "sha256:972a7aaeb7c4a2795b52eef52ee991ef040b31009f36deca6207a986607b55f3",
"sha256:7b2ec162c87fc496aa568258ac88631a2ce0acfe681a9af40842fc55deaedc99", "sha256:97b119c436bfa96a92ac2ca525f7025836d4d4e64b1c9f9eff8dbaf3ff1d86f3",
"sha256:7cee2cef07c8d76894ebefc54e4bb707dfc7f258ad155bd61d87f6cd487a70ff", "sha256:9ba37698e242223f8053cc158f130aee046a96feacbeab65893dbe94f5530118",
"sha256:7d16d4498f8b374fc625c4037742fbdd7f9ac383fd50b06f4df00c81ef60e829", "sha256:b1b0e1f626a0f079c0d3696db70132fb1f29aa87c66aecb6501a9b8be64ce9f7",
"sha256:b50bc1780681b127e28f0075dfb81d6135c3a293e0c1d0211133c75e2179b6c0", "sha256:c14c1224fd1a5be2733530d648a316974dbbb3c946913562c6005a76f21ca042",
"sha256:bd0582f831ad5bcad6ca001deba4568573a4675437db17c4031939156ff339fa", "sha256:c79a8546c48ae6465189e54e3245a97ddf21161e33ff7eaa42787353417bb2b6",
"sha256:cfd40d8a4b59f7567620410f966bb1f32dc555b2b19f82a91b147fac296f645c", "sha256:ceb76935ac4ebdf6d7bc845482a4450b284c6ccfb281e34da51d510658ab34d8",
"sha256:e3ae410089de680e8f84c68b755b42bc42c0ceb8c03dbea88a5099747091d38e", "sha256:e22bffaad04b4d16e1c091baed7f2733fc1ebb91e0c602abf1b6834d17158b1f",
"sha256:e9046e559c299b395b39ac7dbf16005308821c2f24a63cae2ab173bd6aa11616", "sha256:ec883b8e44d877bda6f94a36313a1c6063f8b1997aa091628ae2f34c7f97c8d5",
"sha256:ef6be704ae2bc8ad0ebc5cb850ee9139493b0fc4e81abcc240fb392a63ebc808", "sha256:f1baa54d50ec031d1a9beb89974108f8f2c0706f49798f4777df879df0e1adb6",
"sha256:f8dc19d92896558f9c4317ee365729ead9d7bbcf2052a9a19a3ef17abbb8ac5b" "sha256:f53a5385932cda1e2c862d89460992911a89768c65d176ff8c50cddca4d29bed"
], ],
"version": "==6.1.0" "version": "==6.2.0"
}, },
"pydeep": { "pydeep": {
"hashes": [ "hashes": [
@ -164,31 +163,31 @@
}, },
"reportlab": { "reportlab": {
"hashes": [ "hashes": [
"sha256:035346299a556c378a57cc163f2dfd945feebd5e45c844bbd02cae0711f780b5", "sha256:06b7c7436fa6d4844c7637161f3297c7a96240f35622ab2d219e4fd8387c0ab2",
"sha256:05dafbe4cb0801fff0d0956b2d474e79d91d3b48923b6102a28cce0fcb2d8e53", "sha256:0a5acf67bd9812e38ed84be8994c07a8136b0a8f4c14a1c66c9c73a9567a9a44",
"sha256:0b28d4407caa6932539e8f9761e0430a96c6939713ee4b2f27b6c7af327c69e1", "sha256:1c8ca145d03e3c620866b06febb241b179197b58fb07454fbc8e9d6184cdcc93",
"sha256:10cda714d7da684b2332d4d435cf90472fb2ea031ebbfb7f509a31d5898c06b3", "sha256:2f8d785660ee316874c86abad345633ce8c652e88e03ae8a10f1fdadc72fd23d",
"sha256:152c321ca3caa564a5da1c33cb1937af983184eb7b3830f505770728a3f2f075", "sha256:4869d342352c92a812ce40555ef2a9cfbd722390d67fe61f1d6ec770e9ca41a3",
"sha256:1c522ef6656abaf742c0995bf4fc19b7284732c7f3a00e5cd901d48187a9e8f8", "sha256:493e0dcd9c085d46acf4fe3f00f941e562490a74b651409039a0dee2a0d76555",
"sha256:24ee58e0905d4c0ec13557212de7c36ea92f87d688814191fab19d70294afb0e", "sha256:4e606e3ee9345e68cd205022d526250ad2a1164eea8f1e29d77d6ad08631b0ba",
"sha256:43145d034423c97db1ac0db4f5e20ed6d16a1161a64e3da3f93a8a9b1eae78f1", "sha256:5bf91bae8995db91650fda658129c268515358b756fd16c0261a9dd641df1856",
"sha256:43e433fd601259f4cfdc1c076ade9537c31bd116c4f5aa48d66713a609f8ee2c", "sha256:6df0730f8f715aa12333bd6d2a72eea3a989381451861186d9b5e71889454ac7",
"sha256:5156e1e1f0b7fb8864bafc73ba02f8fa18ca46bf6702b34472f9033edf35cb3b", "sha256:7195c6ea096d10c91cc470f9f0ced3ad74470d9c0fd97923b5e764597dd13671",
"sha256:55d901dae80a30cd3b6a16accfbdf87f2fff6d379fd78ba6b87ae437391a5a2f", "sha256:7431c979e2b498e8e20abf458f360a451717d76c3c1bd49d1fc5697d3504f8e5",
"sha256:67fd4229ff0b643a0c2792ac5274ed7edaffc07052ad5dcd7d6f7a5d641a7446", "sha256:7f7f70a8d4b573d1ff65a81415b4b6ed9545630f381dff4a69307640e09d381d",
"sha256:739e81a32cd51be8d8225d316b1399d4b6ff6ee109963d3b5320b0718ce2aa0f", "sha256:9945433667a46f054d1125b4ca86fe9ee31feb254728b38242e9a6008c135efe",
"sha256:801aa0887135713ab107afd51e58dc2bb951fb3ffd773528b96924f90f82558f", "sha256:b1cdbfc1fd54ac947b9f0114e00ab94e945db679f1e03357a3c00f3a85e73eea",
"sha256:835985ac466b3ac5e0751532f135aebc50c204837ebbc781d9a6313f4b76ab5a", "sha256:bf149847a2fd8f24b788a8abbf97a2b9a73edc5b1bd719384b786eb84bcad15e",
"sha256:87b3a4cedb40c08a7708d870cd7c405effb7a6a29cedf2dde380219d7a4d3cee", "sha256:ce514bfce2bf3e302f52aba9929fe3ac7d918cfea2f5d3e30bf9dac9658bf094",
"sha256:8c1abc9a4fead2d9255dfd1d58e5e1025c1816d2c38f6f0a7f50e42d51543c1d", "sha256:d243d4c8cf1a7e78b734c03628b684ec5de25df1f02ccea2e10fbd217430cb72",
"sha256:97368b3f9622454fac3fe794664da5c09e4ec4ffa3fcbbf11a6de07e1ee35f41", "sha256:d4bee20f52b8c3c477dc780780654cafcfc0eb34d8d6960c13a34a444b431f09",
"sha256:9ca02a68b539969f9396ff485c574e6843a2433f6c165e8958a97d9366e1df0c", "sha256:e730529bd1f62034c50f70a2b05fadbf7d1402d39ff69c9dc63db066d0ef8a46",
"sha256:a43dba290c8c46e12f2cc142e4753e68e98ca088ca1dbbfce8822ca4436e7bf8", "sha256:eb54ecfbf1abe6134073b7b35fd40442c4cd81bb9a5bee1a3038b8867b721bfb",
"sha256:ac9414b875c6dc4fd46a01994612dece57a5be14b53c528f45ff85d82e332dfd", "sha256:f18ec70f5ee6a78b3bb4361e55f3a5ef34eb253f1e72fba76f29f0d680cd446f",
"sha256:b58af0f12434835efa494f081c14bf5293764c78f58e6b32d941db5fff5b1692", "sha256:f6be66f69198dcd04a79faa6052f756d35643496321858f06931c7b1ed9833ab",
"sha256:b6e4a1b2f774f6c713bece60f2c06bb662985bdefffe0bc8582d3ef912985ba5" "sha256:fc5c23a53fbd97b8aab4968c8548ce5cea4a54a26b4f8c1e6835df7adb8d0fe2"
], ],
"version": "==3.5.26" "version": "==3.5.28"
}, },
"requests": { "requests": {
"hashes": [ "hashes": [
@ -255,11 +254,10 @@
}, },
"beautifulsoup4": { "beautifulsoup4": {
"hashes": [ "hashes": [
"sha256:05668158c7b85b791c5abde53e50265e16f98ad601c402ba44d70f96c4159612", "sha256:5279c36b4b2ec2cb4298d723791467e3000e5384a43ea0cdf5d45207c7e97169",
"sha256:25288c9e176f354bf277c0a10aa96c782a6a18a17122dba2e8cec4a97e03343b", "sha256:dcdef580e18a76d54002088602eba453eec38ebbcafafeaabd8cab12b6155d57"
"sha256:f040590be10520f2ea4c2ae8c3dae441c7cfff5308ec9d58a0ec0c1b8f81d469"
], ],
"version": "==4.8.0" "version": "==4.8.1"
}, },
"certifi": { "certifi": {
"hashes": [ "hashes": [
@ -299,10 +297,10 @@
}, },
"commonmark": { "commonmark": {
"hashes": [ "hashes": [
"sha256:14c3df31e8c9c463377e287b2a1eefaa6019ab97b22dad36e2f32be59d61d68d", "sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60",
"sha256:867fc5db078ede373ab811e16b6789e9d033b15ccd7296f370ca52d1ee792ce0" "sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9"
], ],
"version": "==0.9.0" "version": "==0.9.1"
}, },
"coverage": { "coverage": {
"hashes": [ "hashes": [
@ -377,12 +375,6 @@
"index": "pypi", "index": "pypi",
"version": "==0.15" "version": "==0.15"
}, },
"future": {
"hashes": [
"sha256:67045236dcfd6816dc439556d009594abf643e5eb48992e36beac09c2ca659b8"
],
"version": "==0.17.1"
},
"idna": { "idna": {
"hashes": [ "hashes": [
"sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407", "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407",
@ -399,10 +391,10 @@
}, },
"jinja2": { "jinja2": {
"hashes": [ "hashes": [
"sha256:065c4f02ebe7f7cf559e49ee5a95fb800a9e4528727aec6f24402a5374c65013", "sha256:74320bb91f31270f9551d46522e33af46a80c3d619f4a4bf42b3164d30b5911f",
"sha256:14dd6caf1527abb21f08f86c784eac40853ba93edb79552aa1e4b8aef1b61c7b" "sha256:9fe95f19286cfefaa917656583d020be14e7859c6b0252588391e47db34527de"
], ],
"version": "==2.10.1" "version": "==2.10.3"
}, },
"jsonschema": { "jsonschema": {
"hashes": [ "hashes": [
@ -461,11 +453,18 @@
], ],
"version": "==1.1.1" "version": "==1.1.1"
}, },
"memory-profiler": {
"hashes": [
"sha256:5fa47b274c929dd2cbcd9190afb62fec110701251d2ac2d301caaf545c81afc1"
],
"index": "pypi",
"version": "==0.55.0"
},
"neobolt": { "neobolt": {
"hashes": [ "hashes": [
"sha256:fa9efe4a4defbdc63fc3f1e552d503727049586c59d8a3acf5188a2cf1a45dce" "sha256:56b86b8b2c3facdd54589e60ecd22e0234d6f40645ab2e2cf87ef0cd79df20af"
], ],
"version": "==1.7.13" "version": "==1.7.15"
}, },
"neotime": { "neotime": {
"hashes": [ "hashes": [
@ -491,42 +490,56 @@
}, },
"pillow": { "pillow": {
"hashes": [ "hashes": [
"sha256:0804f77cb1e9b6dbd37601cee11283bba39a8d44b9ddb053400c58e0c0d7d9de", "sha256:00fdeb23820f30e43bba78eb9abb00b7a937a655de7760b2e09101d63708b64e",
"sha256:0ab7c5b5d04691bcbd570658667dd1e21ca311c62dcfd315ad2255b1cd37f64f", "sha256:01f948e8220c85eae1aa1a7f8edddcec193918f933fb07aaebe0bfbbcffefbf1",
"sha256:0b3e6cf3ea1f8cecd625f1420b931c83ce74f00c29a0ff1ce4385f99900ac7c4", "sha256:08abf39948d4b5017a137be58f1a52b7101700431f0777bec3d897c3949f74e6",
"sha256:365c06a45712cd723ec16fa4ceb32ce46ad201eb7bbf6d3c16b063c72b61a3ed", "sha256:099a61618b145ecb50c6f279666bbc398e189b8bc97544ae32b8fcb49ad6b830",
"sha256:38301fbc0af865baa4752ddae1bb3cbb24b3d8f221bf2850aad96b243306fa03", "sha256:2c1c61546e73de62747e65807d2cc4980c395d4c5600ecb1f47a650c6fa78c79",
"sha256:3aef1af1a91798536bbab35d70d35750bd2884f0832c88aeb2499aa2d1ed4992", "sha256:2ed9c4f694861642401f27dc3cb99772be67cd190e84845c749dae0a06c3bfae",
"sha256:3fe0ab49537d9330c9bba7f16a5f8b02da615b5c809cdf7124f356a0f182eccd", "sha256:338581b30b908e111be578f0297255f6b57a51358cd16fa0e6f664c9a1f88bff",
"sha256:45a619d5c1915957449264c81c008934452e3fd3604e36809212300b2a4dab68", "sha256:38c7d48a21cd06fdeee93987147b9b1c55b73b4cfcbf83240568bfbd5adee447",
"sha256:49f90f147883a0c3778fd29d3eb169d56416f25758d0f66775db9184debc8010", "sha256:43fd026f613c8e48a25eba1a92f4d2ad7f3903c95d8c33a11611a7717d2ab654",
"sha256:571b5a758baf1cb6a04233fb23d6cf1ca60b31f9f641b1700bfaab1194020555", "sha256:4548236844327a718ce3bb182ab32a16fa2050c61e334e959f554cac052fb0df",
"sha256:5ac381e8b1259925287ccc5a87d9cf6322a2dc88ae28a97fe3e196385288413f", "sha256:5090857876c58885cfa388dc649e5db30aae98a068c26f3fd0ac9d7d9a4d9572",
"sha256:6153db744a743c0c8c91b8e3b9d40e0b13a5d31dbf8a12748c6d9bfd3ddc01ad", "sha256:5bbba34f97a26a93f5e8dec469ca4ddd712451418add43da946dbaed7f7a98d2",
"sha256:6fd63afd14a16f5d6b408f623cc2142917a1f92855f0df997e09a49f0341be8a", "sha256:65a28969a025a0eb4594637b6103201dc4ed2a9508bdab56ac33e43e3081c404",
"sha256:70acbcaba2a638923c2d337e0edea210505708d7859b87c2bd81e8f9902ae826", "sha256:892bb52b70bd5ea9dbbc3ac44f38e84f5a04e9d8b1bff48159d96cb795b81159",
"sha256:70b1594d56ed32d56ed21a7fbb2a5c6fd7446cdb7b21e749c9791eac3a64d9e4", "sha256:8a9becd5cbd5062f973bcd2e7bc79483af310222de112b6541f8af1f93a3cc42",
"sha256:76638865c83b1bb33bcac2a61ce4d13c17dba2204969dedb9ab60ef62bede686", "sha256:972a7aaeb7c4a2795b52eef52ee991ef040b31009f36deca6207a986607b55f3",
"sha256:7b2ec162c87fc496aa568258ac88631a2ce0acfe681a9af40842fc55deaedc99", "sha256:97b119c436bfa96a92ac2ca525f7025836d4d4e64b1c9f9eff8dbaf3ff1d86f3",
"sha256:7cee2cef07c8d76894ebefc54e4bb707dfc7f258ad155bd61d87f6cd487a70ff", "sha256:9ba37698e242223f8053cc158f130aee046a96feacbeab65893dbe94f5530118",
"sha256:7d16d4498f8b374fc625c4037742fbdd7f9ac383fd50b06f4df00c81ef60e829", "sha256:b1b0e1f626a0f079c0d3696db70132fb1f29aa87c66aecb6501a9b8be64ce9f7",
"sha256:b50bc1780681b127e28f0075dfb81d6135c3a293e0c1d0211133c75e2179b6c0", "sha256:c14c1224fd1a5be2733530d648a316974dbbb3c946913562c6005a76f21ca042",
"sha256:bd0582f831ad5bcad6ca001deba4568573a4675437db17c4031939156ff339fa", "sha256:c79a8546c48ae6465189e54e3245a97ddf21161e33ff7eaa42787353417bb2b6",
"sha256:cfd40d8a4b59f7567620410f966bb1f32dc555b2b19f82a91b147fac296f645c", "sha256:ceb76935ac4ebdf6d7bc845482a4450b284c6ccfb281e34da51d510658ab34d8",
"sha256:e3ae410089de680e8f84c68b755b42bc42c0ceb8c03dbea88a5099747091d38e", "sha256:e22bffaad04b4d16e1c091baed7f2733fc1ebb91e0c602abf1b6834d17158b1f",
"sha256:e9046e559c299b395b39ac7dbf16005308821c2f24a63cae2ab173bd6aa11616", "sha256:ec883b8e44d877bda6f94a36313a1c6063f8b1997aa091628ae2f34c7f97c8d5",
"sha256:ef6be704ae2bc8ad0ebc5cb850ee9139493b0fc4e81abcc240fb392a63ebc808", "sha256:f1baa54d50ec031d1a9beb89974108f8f2c0706f49798f4777df879df0e1adb6",
"sha256:f8dc19d92896558f9c4317ee365729ead9d7bbcf2052a9a19a3ef17abbb8ac5b" "sha256:f53a5385932cda1e2c862d89460992911a89768c65d176ff8c50cddca4d29bed"
], ],
"version": "==6.1.0" "version": "==6.2.0"
}, },
"prompt-toolkit": { "prompt-toolkit": {
"hashes": [ "hashes": [
"sha256:11adf3389a996a6d45cc277580d0d53e8a5afd281d0c9ec71b28e6f121463780", "sha256:46642344ce457641f28fc9d1c9ca939b63dadf8df128b86f1b9860e59c73a5e4",
"sha256:2519ad1d8038fd5fc8e770362237ad0364d16a7650fb5724af6997ed5515e3c1", "sha256:e7f8af9e3d70f514373bf41aa51bc33af12a6db3f71461ea47fea985defb2c31",
"sha256:977c6583ae813a37dc1c2e1b715892461fcbdaa57f6fc62f33a528c4886c8f55" "sha256:f15af68f66e664eaa559d4ac8a928111eebd5feda0c11738b5998045224829db"
], ],
"version": "==2.0.9" "version": "==2.0.10"
},
"psutil": {
"hashes": [
"sha256:028a1ec3c6197eadd11e7b46e8cc2f0720dc18ac6d7aabdb8e8c0d6c9704f000",
"sha256:503e4b20fa9d3342bcf58191bbc20a4a5ef79ca7df8972e6197cc14c5513e73d",
"sha256:863a85c1c0a5103a12c05a35e59d336e1d665747e531256e061213e2e90f63f3",
"sha256:954f782608bfef9ae9f78e660e065bd8ffcfaea780f9f2c8a133bb7cb9e826d7",
"sha256:b6e08f965a305cd84c2d07409bc16fbef4417d67b70c53b299116c5b895e3f45",
"sha256:bc96d437dfbb8865fc8828cf363450001cb04056bbdcdd6fc152c436c8a74c61",
"sha256:cf49178021075d47c61c03c0229ac0c60d5e2830f8cab19e2d88e579b18cdb76",
"sha256:d5350cb66690915d60f8b233180f1e49938756fb2d501c93c44f8fb5b970cc63",
"sha256:eba238cf1989dfff7d483c029acb0ac4fcbfc15de295d682901f0e2497e6781a"
],
"version": "==5.6.3"
}, },
"py2neo": { "py2neo": {
"hashes": [ "hashes": [
@ -586,10 +599,10 @@
}, },
"pytz": { "pytz": {
"hashes": [ "hashes": [
"sha256:26c0b32e437e54a18161324a2fca3c4b9846b74a8dccddd843113109e1116b32", "sha256:1c557d7d0e871de1f5ccd5833f60fb2550652da6be2693c1e02300743d21500d",
"sha256:c894d57500a4cd2d5c71114aaab77dbab5eabd9022308ce5ac9bb93a60a6f0c7" "sha256:b02c06db6cf09c12dd25137e563b31700d3b80fcc4ad23abb7a315f2789819be"
], ],
"version": "==2019.2" "version": "==2019.3"
}, },
"recommonmark": { "recommonmark": {
"hashes": [ "hashes": [
@ -600,31 +613,31 @@
}, },
"reportlab": { "reportlab": {
"hashes": [ "hashes": [
"sha256:035346299a556c378a57cc163f2dfd945feebd5e45c844bbd02cae0711f780b5", "sha256:06b7c7436fa6d4844c7637161f3297c7a96240f35622ab2d219e4fd8387c0ab2",
"sha256:05dafbe4cb0801fff0d0956b2d474e79d91d3b48923b6102a28cce0fcb2d8e53", "sha256:0a5acf67bd9812e38ed84be8994c07a8136b0a8f4c14a1c66c9c73a9567a9a44",
"sha256:0b28d4407caa6932539e8f9761e0430a96c6939713ee4b2f27b6c7af327c69e1", "sha256:1c8ca145d03e3c620866b06febb241b179197b58fb07454fbc8e9d6184cdcc93",
"sha256:10cda714d7da684b2332d4d435cf90472fb2ea031ebbfb7f509a31d5898c06b3", "sha256:2f8d785660ee316874c86abad345633ce8c652e88e03ae8a10f1fdadc72fd23d",
"sha256:152c321ca3caa564a5da1c33cb1937af983184eb7b3830f505770728a3f2f075", "sha256:4869d342352c92a812ce40555ef2a9cfbd722390d67fe61f1d6ec770e9ca41a3",
"sha256:1c522ef6656abaf742c0995bf4fc19b7284732c7f3a00e5cd901d48187a9e8f8", "sha256:493e0dcd9c085d46acf4fe3f00f941e562490a74b651409039a0dee2a0d76555",
"sha256:24ee58e0905d4c0ec13557212de7c36ea92f87d688814191fab19d70294afb0e", "sha256:4e606e3ee9345e68cd205022d526250ad2a1164eea8f1e29d77d6ad08631b0ba",
"sha256:43145d034423c97db1ac0db4f5e20ed6d16a1161a64e3da3f93a8a9b1eae78f1", "sha256:5bf91bae8995db91650fda658129c268515358b756fd16c0261a9dd641df1856",
"sha256:43e433fd601259f4cfdc1c076ade9537c31bd116c4f5aa48d66713a609f8ee2c", "sha256:6df0730f8f715aa12333bd6d2a72eea3a989381451861186d9b5e71889454ac7",
"sha256:5156e1e1f0b7fb8864bafc73ba02f8fa18ca46bf6702b34472f9033edf35cb3b", "sha256:7195c6ea096d10c91cc470f9f0ced3ad74470d9c0fd97923b5e764597dd13671",
"sha256:55d901dae80a30cd3b6a16accfbdf87f2fff6d379fd78ba6b87ae437391a5a2f", "sha256:7431c979e2b498e8e20abf458f360a451717d76c3c1bd49d1fc5697d3504f8e5",
"sha256:67fd4229ff0b643a0c2792ac5274ed7edaffc07052ad5dcd7d6f7a5d641a7446", "sha256:7f7f70a8d4b573d1ff65a81415b4b6ed9545630f381dff4a69307640e09d381d",
"sha256:739e81a32cd51be8d8225d316b1399d4b6ff6ee109963d3b5320b0718ce2aa0f", "sha256:9945433667a46f054d1125b4ca86fe9ee31feb254728b38242e9a6008c135efe",
"sha256:801aa0887135713ab107afd51e58dc2bb951fb3ffd773528b96924f90f82558f", "sha256:b1cdbfc1fd54ac947b9f0114e00ab94e945db679f1e03357a3c00f3a85e73eea",
"sha256:835985ac466b3ac5e0751532f135aebc50c204837ebbc781d9a6313f4b76ab5a", "sha256:bf149847a2fd8f24b788a8abbf97a2b9a73edc5b1bd719384b786eb84bcad15e",
"sha256:87b3a4cedb40c08a7708d870cd7c405effb7a6a29cedf2dde380219d7a4d3cee", "sha256:ce514bfce2bf3e302f52aba9929fe3ac7d918cfea2f5d3e30bf9dac9658bf094",
"sha256:8c1abc9a4fead2d9255dfd1d58e5e1025c1816d2c38f6f0a7f50e42d51543c1d", "sha256:d243d4c8cf1a7e78b734c03628b684ec5de25df1f02ccea2e10fbd217430cb72",
"sha256:97368b3f9622454fac3fe794664da5c09e4ec4ffa3fcbbf11a6de07e1ee35f41", "sha256:d4bee20f52b8c3c477dc780780654cafcfc0eb34d8d6960c13a34a444b431f09",
"sha256:9ca02a68b539969f9396ff485c574e6843a2433f6c165e8958a97d9366e1df0c", "sha256:e730529bd1f62034c50f70a2b05fadbf7d1402d39ff69c9dc63db066d0ef8a46",
"sha256:a43dba290c8c46e12f2cc142e4753e68e98ca088ca1dbbfce8822ca4436e7bf8", "sha256:eb54ecfbf1abe6134073b7b35fd40442c4cd81bb9a5bee1a3038b8867b721bfb",
"sha256:ac9414b875c6dc4fd46a01994612dece57a5be14b53c528f45ff85d82e332dfd", "sha256:f18ec70f5ee6a78b3bb4361e55f3a5ef34eb253f1e72fba76f29f0d680cd446f",
"sha256:b58af0f12434835efa494f081c14bf5293764c78f58e6b32d941db5fff5b1692", "sha256:f6be66f69198dcd04a79faa6052f756d35643496321858f06931c7b1ed9833ab",
"sha256:b6e4a1b2f774f6c713bece60f2c06bb662985bdefffe0bc8582d3ef912985ba5" "sha256:fc5c23a53fbd97b8aab4968c8548ce5cea4a54a26b4f8c1e6835df7adb8d0fe2"
], ],
"version": "==3.5.26" "version": "==3.5.28"
}, },
"requests": { "requests": {
"hashes": [ "hashes": [
@ -650,9 +663,10 @@
}, },
"snowballstemmer": { "snowballstemmer": {
"hashes": [ "hashes": [
"sha256:713e53b79cbcf97bc5245a06080a33d54a77e7cce2f789c835a143bcdb5c033e" "sha256:209f257d7533fdb3cb73bdbd24f436239ca3b2fa67d56f6ff88e86be08cc5ef0",
"sha256:df3bac3df4c2c01363f3dd2cfa78cce2840a79b9f1c2d2de9ce8d31683992f52"
], ],
"version": "==1.9.1" "version": "==2.0.0"
}, },
"soupsieve": { "soupsieve": {
"hashes": [ "hashes": [

5
examples/generate_file_objects.py
View File

@ -5,7 +5,7 @@ import argparse
import json import json
try: try:
from pymisp import MISPEncode, AbstractMISP from pymisp import pymisp_json_default, AbstractMISP
from pymisp.tools import make_binary_objects from pymisp.tools import make_binary_objects
except ImportError: except ImportError:
pass pass
@ -51,7 +51,8 @@ def make_objects(path):
to_return['objects'].append(fo) to_return['objects'].append(fo)
if fo.ObjectReference: if fo.ObjectReference:
to_return['references'] += fo.ObjectReference to_return['references'] += fo.ObjectReference
return json.dumps(to_return, cls=MISPEncode) return json.dumps(to_return, default=pymisp_json_default)
if __name__ == '__main__': if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Extract indicators out of binaries and returns MISP objects.') parser = argparse.ArgumentParser(description='Extract indicators out of binaries and returns MISP objects.')

2
pymisp/__init__.py
View File

@ -31,7 +31,7 @@ try:
warning_2020() warning_2020()
from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey, InvalidMISPObject, UnknownMISPObjectTemplate, PyMISPInvalidFormat, MISPServerError, PyMISPNotImplementedYet, PyMISPUnexpectedResponse, PyMISPEmptyResponse # noqa from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey, InvalidMISPObject, UnknownMISPObjectTemplate, PyMISPInvalidFormat, MISPServerError, PyMISPNotImplementedYet, PyMISPUnexpectedResponse, PyMISPEmptyResponse # noqa
from .api import PyMISP # noqa from .api import PyMISP # noqa
from .abstract import AbstractMISP, MISPEncode, MISPTag, Distribution, ThreatLevel, Analysis # noqa from .abstract import AbstractMISP, MISPEncode, pymisp_json_default, MISPTag, Distribution, ThreatLevel, Analysis # noqa
from .mispevent import MISPEvent, MISPAttribute, MISPObjectReference, MISPObjectAttribute, MISPObject, MISPUser, MISPOrganisation, MISPSighting, MISPLog, MISPShadowAttribute, MISPWarninglist, MISPTaxonomy, MISPNoticelist, MISPObjectTemplate, MISPSharingGroup, MISPRole, MISPServer, MISPFeed, MISPEventDelegation # noqa from .mispevent import MISPEvent, MISPAttribute, MISPObjectReference, MISPObjectAttribute, MISPObject, MISPUser, MISPOrganisation, MISPSighting, MISPLog, MISPShadowAttribute, MISPWarninglist, MISPTaxonomy, MISPNoticelist, MISPObjectTemplate, MISPSharingGroup, MISPRole, MISPServer, MISPFeed, MISPEventDelegation # noqa
from .tools import AbstractMISPObjectGenerator # noqa from .tools import AbstractMISPObjectGenerator # noqa
from .tools import Neo4j # noqa from .tools import Neo4j # noqa

153
pymisp/abstract.py
View File

@ -3,24 +3,40 @@
import sys import sys
import datetime import datetime
import json
from deprecated import deprecated
from json import JSONEncoder from json import JSONEncoder
try:
from rapidjson import load
from rapidjson import loads
from rapidjson import dumps
import rapidjson
HAS_RAPIDJSON = True
except ImportError:
from json import load
from json import loads
from json import dumps
import json
HAS_RAPIDJSON = False
import logging import logging
from enum import Enum from enum import Enum
from .exceptions import PyMISPInvalidFormat from .exceptions import PyMISPInvalidFormat
# Try to import MutableMapping the python 3.3+ way
try:
from collections.abc import MutableMapping
except Exception:
pass
logger = logging.getLogger('pymisp') logger = logging.getLogger('pymisp')
if sys.version_info < (3, 0): if sys.version_info < (3, 0):
from collections import MutableMapping from collections import MutableMapping
import os
from cachetools import cached, LRUCache
resources_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data')
misp_objects_path = os.path.join(resources_path, 'misp-objects', 'objects')
with open(os.path.join(resources_path, 'describeTypes.json'), 'r') as f:
describe_types = load(f)['result']
# This is required because Python 2 is a pain. # This is required because Python 2 is a pain.
from datetime import tzinfo, timedelta from datetime import tzinfo, timedelta
@ -37,6 +53,36 @@ if sys.version_info < (3, 0):
def dst(self, dt): def dst(self, dt):
return timedelta(0) return timedelta(0)
class MISPFileCache(object):
# cache up to 150 JSON structures in class attribute
@staticmethod
@cached(cache=LRUCache(maxsize=150))
def _load_json(path):
with open(path, 'r') as f:
data = load(f)
return data
else:
from collections.abc import MutableMapping
from functools import lru_cache
from pathlib import Path
resources_path = Path(__file__).parent / 'data'
misp_objects_path = resources_path / 'misp-objects' / 'objects'
with (resources_path / 'describeTypes.json').open('r') as f:
describe_types = load(f)['result']
class MISPFileCache(object):
# cache up to 150 JSON structures in class attribute
@staticmethod
@lru_cache(maxsize=150)
def _load_json(path):
with path.open('r') as f:
data = load(f)
return data
class Distribution(Enum): class Distribution(Enum):
your_organisation_only = 0 your_organisation_only = 0
@ -68,8 +114,8 @@ def _int_to_str(d):
return d return d
@deprecated(reason=" Use method default=pymisp_json_default instead of cls=MISPEncode", version='2.4.117', action='default')
class MISPEncode(JSONEncoder): class MISPEncode(JSONEncoder):
def default(self, obj): def default(self, obj):
if isinstance(obj, AbstractMISP): if isinstance(obj, AbstractMISP):
return obj.jsonable() return obj.jsonable()
@ -80,13 +126,37 @@ class MISPEncode(JSONEncoder):
return JSONEncoder.default(self, obj) return JSONEncoder.default(self, obj)
class AbstractMISP(MutableMapping): if HAS_RAPIDJSON:
def pymisp_json_default(obj):
if isinstance(obj, AbstractMISP):
return obj.jsonable()
elif isinstance(obj, (datetime.datetime, datetime.date)):
return obj.isoformat()
elif isinstance(obj, Enum):
return obj.value
return rapidjson.default(obj)
else:
def pymisp_json_default(obj):
if isinstance(obj, AbstractMISP):
return obj.jsonable()
elif isinstance(obj, (datetime.datetime, datetime.date)):
return obj.isoformat()
elif isinstance(obj, Enum):
return obj.value
return json.default(obj)
class AbstractMISP(MutableMapping, MISPFileCache):
__resources_path = resources_path
__misp_objects_path = misp_objects_path
__describe_types = describe_types
def __init__(self, **kwargs): def __init__(self, **kwargs):
"""Abstract class for all the MISP objects""" """Abstract class for all the MISP objects"""
super(AbstractMISP, self).__init__() super(AbstractMISP, self).__init__()
self.__edited = True # As we create a new object, we assume it is edited self.__edited = True # As we create a new object, we assume it is edited
self.__not_jsonable = [] self.__not_jsonable = []
self.__self_defined_describe_types = None
if kwargs.get('force_timestamps') is not None: if kwargs.get('force_timestamps') is not None:
# Ignore the edited objects and keep the timestamps. # Ignore the edited objects and keep the timestamps.
@ -103,16 +173,28 @@ class AbstractMISP(MutableMapping):
setattr(AbstractMISP, 'tags', property(AbstractMISP.__get_tags, AbstractMISP.__set_tags)) setattr(AbstractMISP, 'tags', property(AbstractMISP.__get_tags, AbstractMISP.__set_tags))
@property @property
def properties(self): def describe_types(self):
"""All the class public properties that will be dumped in the dictionary, and the JSON export. if self.__self_defined_describe_types:
Note: all the properties starting with a `_` (private), or listed in __not_jsonable will be skipped. return self.__self_defined_describe_types
""" return self.__describe_types
to_return = []
for prop, value in vars(self).items(): @describe_types.setter
if prop.startswith('_') or prop in self.__not_jsonable: def describe_types(self, describe_types):
continue self.__self_defined_describe_types = describe_types
to_return.append(prop)
return to_return @property
def resources_path(self):
return self.__resources_path
@property
def misp_objects_path(self):
return self.__misp_objects_path
@misp_objects_path.setter
def misp_objects_path(self, misp_objects_path):
if sys.version_info >= (3, 0) and isinstance(misp_objects_path, str):
misp_objects_path = Path(misp_objects_path)
self.__misp_objects_path = misp_objects_path
def from_dict(self, **kwargs): def from_dict(self, **kwargs):
"""Loading all the parameters as class properties, if they aren't `None`. """Loading all the parameters as class properties, if they aren't `None`.
@ -137,21 +219,21 @@ class AbstractMISP(MutableMapping):
def from_json(self, json_string): def from_json(self, json_string):
"""Load a JSON string""" """Load a JSON string"""
self.from_dict(**json.loads(json_string)) self.from_dict(**loads(json_string))
def to_dict(self): def to_dict(self):
"""Dump the lass to a dictionary. """Dump the class to a dictionary.
This method automatically removes the timestamp recursively in every object This method automatically removes the timestamp recursively in every object
that has been edited is order to let MISP update the event accordingly.""" that has been edited is order to let MISP update the event accordingly."""
is_edited = self.edited
to_return = {} to_return = {}
for attribute in self.properties: for attribute, val in self.items():
val = getattr(self, attribute, None)
if val is None: if val is None:
continue continue
elif isinstance(val, list) and len(val) == 0: elif isinstance(val, list) and len(val) == 0:
continue continue
if attribute == 'timestamp': if attribute == 'timestamp':
if not self.__force_timestamps and self.edited: if not self.__force_timestamps and is_edited:
# In order to be accepted by MISP, the timestamp of an object # In order to be accepted by MISP, the timestamp of an object
# needs to be either newer, or None. # needs to be either newer, or None.
# If the current object is marked as edited, the easiest is to # If the current object is marked as edited, the easiest is to
@ -167,13 +249,15 @@ class AbstractMISP(MutableMapping):
"""This method is used by the JSON encoder""" """This method is used by the JSON encoder"""
return self.to_dict() return self.to_dict()
def to_json(self): def to_json(self, sort_keys=False, indent=None):
"""Dump recursively any class of type MISPAbstract to a json string""" """Dump recursively any class of type MISPAbstract to a json string"""
return json.dumps(self, cls=MISPEncode, sort_keys=True, indent=2) return dumps(self, default=pymisp_json_default, sort_keys=sort_keys, indent=indent)
def __getitem__(self, key): def __getitem__(self, key):
try: try:
return getattr(self, key) if key[0] != '_' and key not in self.__not_jsonable:
return self.__dict__[key]
raise KeyError
except AttributeError: except AttributeError:
# Expected by pop and other dict-related methods # Expected by pop and other dict-related methods
raise KeyError raise KeyError
@ -185,10 +269,10 @@ class AbstractMISP(MutableMapping):
delattr(self, key) delattr(self, key)
def __iter__(self): def __iter__(self):
return iter(self.to_dict()) return iter({k: v for k, v in self.__dict__.items() if not (k[0] == '_' or k in self.__not_jsonable)})
def __len__(self): def __len__(self):
return len(self.to_dict()) return len([k for k in self.__dict__.keys() if not (k[0] == '_' or k in self.__not_jsonable)])
@property @property
def edited(self): def edited(self):
@ -196,15 +280,14 @@ class AbstractMISP(MutableMapping):
to the parent objects""" to the parent objects"""
if self.__edited: if self.__edited:
return self.__edited return self.__edited
for p in self.properties: for p, val in self.items():
if self.__edited:
break
val = getattr(self, p)
if isinstance(val, AbstractMISP) and val.edited: if isinstance(val, AbstractMISP) and val.edited:
self.__edited = True self.__edited = True
break
elif isinstance(val, list) and all(isinstance(a, AbstractMISP) for a in val): elif isinstance(val, list) and all(isinstance(a, AbstractMISP) for a in val):
if any(a.edited for a in val): if any(a.edited for a in val):
self.__edited = True self.__edited = True
break
return self.__edited return self.__edited
@edited.setter @edited.setter
@ -216,7 +299,9 @@ class AbstractMISP(MutableMapping):
raise Exception('edited can only be True or False') raise Exception('edited can only be True or False')
def __setattr__(self, name, value): def __setattr__(self, name, value):
if name in self.properties: if name[0] != '_' and not self.__edited and name in self.keys():
# The private members don't matter
# If we already have a key with that name, we're modifying it.
self.__edited = True self.__edited = True
super(AbstractMISP, self).__setattr__(name, value) super(AbstractMISP, self).__setattr__(name, value)

34
pymisp/api.py
View File

@ -19,7 +19,7 @@ from deprecated import deprecated
from . import __version__, warning_2020 from . import __version__, warning_2020
from .exceptions import PyMISPError, SearchError, NoURL, NoKey, PyMISPEmptyResponse from .exceptions import PyMISPError, SearchError, NoURL, NoKey, PyMISPEmptyResponse
from .mispevent import MISPEvent, MISPAttribute, MISPUser, MISPOrganisation, MISPSighting, MISPFeed, MISPObject, MISPSharingGroup from .mispevent import MISPEvent, MISPAttribute, MISPUser, MISPOrganisation, MISPSighting, MISPFeed, MISPObject, MISPSharingGroup
from .abstract import AbstractMISP, MISPEncode from .abstract import AbstractMISP, pymisp_json_default, MISPFileCache, describe_types
logger = logging.getLogger('pymisp') logger = logging.getLogger('pymisp')
@ -37,11 +37,6 @@ try:
except ImportError: except ImportError:
HAVE_REQUESTS = False HAVE_REQUESTS = False
if (3, 0) <= sys.version_info < (3, 6):
OLD_PY3 = True
else:
OLD_PY3 = False
try: try:
from requests_futures.sessions import FuturesSession from requests_futures.sessions import FuturesSession
ASYNC_OK = True ASYNC_OK = True
@ -58,7 +53,7 @@ Response (if any):
{}''' {}'''
class PyMISP(object): # pragma: no cover class PyMISP(MISPFileCache): # pragma: no cover
"""Python API for MISP """Python API for MISP
:param url: URL of the MISP instance you want to connect to :param url: URL of the MISP instance you want to connect to
@ -140,24 +135,19 @@ class PyMISP(object): # pragma: no cover
@deprecated(reason="Use ExpandedPyMISP.describe_types_local", version='2.4.110', action='default') @deprecated(reason="Use ExpandedPyMISP.describe_types_local", version='2.4.110', action='default')
def get_local_describe_types(self): def get_local_describe_types(self):
with open(os.path.join(self.resources_path, 'describeTypes.json'), 'rb') as f: return describe_types
if OLD_PY3:
describe_types = json.loads(f.read().decode())
else:
describe_types = json.load(f)
return describe_types['result']
@deprecated(reason="Use ExpandedPyMISP.describe_types_remote", version='2.4.110', action='default') @deprecated(reason="Use ExpandedPyMISP.describe_types_remote", version='2.4.110', action='default')
def get_live_describe_types(self): def get_live_describe_types(self):
response = self._prepare_request('GET', urljoin(self.root_url, 'attributes/describeTypes.json')) response = self._prepare_request('GET', urljoin(self.root_url, 'attributes/describeTypes.json'))
describe_types = self._check_response(response) remote_describe_types = self._check_response(response)
if describe_types.get('error'): if remote_describe_types.get('error'):
for e in describe_types.get('error'): for e in remote_describe_types.get('error'):
raise PyMISPError('Failed: {}'.format(e)) raise PyMISPError('Failed: {}'.format(e))
describe_types = describe_types['result'] remote_describe_types = describe_types
if not describe_types.get('sane_defaults'): if not remote_describe_types.get('sane_defaults'):
raise PyMISPError('The MISP server your are trying to reach is outdated (<2.4.52). Please use PyMISP v2.4.51.1 (pip install -I PyMISP==v2.4.51.1) and/or contact your administrator.') raise PyMISPError('The MISP server your are trying to reach is outdated (<2.4.52). Please use PyMISP v2.4.51.1 (pip install -I PyMISP==v2.4.51.1) and/or contact your administrator.')
return describe_types return remote_describe_types
def _prepare_request(self, request_type, url, data=None, def _prepare_request(self, request_type, url, data=None,
background_callback=None, output_type='json'): background_callback=None, output_type='json'):
@ -172,7 +162,7 @@ class PyMISP(object): # pragma: no cover
if isinstance(data, dict): if isinstance(data, dict):
# Remove None values. # Remove None values.
data = {k: v for k, v in data.items() if v is not None} data = {k: v for k, v in data.items() if v is not None}
data = json.dumps(data, cls=MISPEncode) data = json.dumps(data, default=pymisp_json_default)
req = requests.Request(request_type, url, data=data) req = requests.Request(request_type, url, data=data)
if self.asynch and background_callback is not None: if self.asynch and background_callback is not None:
local_session = FuturesSession local_session = FuturesSession
@ -614,7 +604,7 @@ class PyMISP(object): # pragma: no cover
else: else:
data = attributes.to_json() data = attributes.to_json()
# _prepare_request(...) returns a requests.Response Object # _prepare_request(...) returns a requests.Response Object
resp = self._prepare_request('POST', url, json.dumps(data, cls=MISPEncode)) resp = self._prepare_request('POST', url, json.dumps(data, default=pymisp_json_default))
try: try:
responses.append(resp.json()) responses.append(resp.json())
except Exception: except Exception:
@ -1068,7 +1058,7 @@ class PyMISP(object): # pragma: no cover
url = urljoin(self.root_url, 'shadow_attributes/{}/{}'.format(path, id)) url = urljoin(self.root_url, 'shadow_attributes/{}/{}'.format(path, id))
if path in ['add', 'edit']: if path in ['add', 'edit']:
query = {'request': {'ShadowAttribute': attribute}} query = {'request': {'ShadowAttribute': attribute}}
response = self._prepare_request('POST', url, json.dumps(query, cls=MISPEncode)) response = self._prepare_request('POST', url, json.dumps(query, default=pymisp_json_default))
elif path == 'view': elif path == 'view':
response = self._prepare_request('GET', url) response = self._prepare_request('GET', url)
else: # accept or discard else: # accept or discard

12
pymisp/aping.py
View File

@ -19,7 +19,7 @@ from . import __version__
from .exceptions import MISPServerError, PyMISPUnexpectedResponse, PyMISPNotImplementedYet, PyMISPError, NoURL, NoKey from .exceptions import MISPServerError, PyMISPUnexpectedResponse, PyMISPNotImplementedYet, PyMISPError, NoURL, NoKey
from .api import everything_broken, PyMISP from .api import everything_broken, PyMISP
from .mispevent import MISPEvent, MISPAttribute, MISPSighting, MISPLog, MISPObject, MISPUser, MISPOrganisation, MISPShadowAttribute, MISPWarninglist, MISPTaxonomy, MISPGalaxy, MISPNoticelist, MISPObjectReference, MISPObjectTemplate, MISPSharingGroup, MISPRole, MISPServer, MISPFeed, MISPEventDelegation, MISPCommunity from .mispevent import MISPEvent, MISPAttribute, MISPSighting, MISPLog, MISPObject, MISPUser, MISPOrganisation, MISPShadowAttribute, MISPWarninglist, MISPTaxonomy, MISPGalaxy, MISPNoticelist, MISPObjectReference, MISPObjectTemplate, MISPSharingGroup, MISPRole, MISPServer, MISPFeed, MISPEventDelegation, MISPCommunity
from .abstract import MISPEncode, MISPTag, AbstractMISP from .abstract import pymisp_json_default, MISPTag, AbstractMISP, describe_types
SearchType = TypeVar('SearchType', str, int) SearchType = TypeVar('SearchType', str, int)
# str: string to search / list: values to search (OR) / dict: {'OR': [list], 'NOT': [list], 'AND': [list]} # str: string to search / list: values to search (OR) / dict: {'OR': [list], 'NOT': [list], 'AND': [list]}
@ -106,16 +106,14 @@ class ExpandedPyMISP(PyMISP):
@property @property
def describe_types_local(self): def describe_types_local(self):
'''Returns the content of describe types from the package''' '''Returns the content of describe types from the package'''
with (self.resources_path / 'describeTypes.json').open() as f: return describe_types
describe_types = json.load(f)
return describe_types['result']
@property @property
def describe_types_remote(self): def describe_types_remote(self):
'''Returns the content of describe types from the remote instance''' '''Returns the content of describe types from the remote instance'''
response = self._prepare_request('GET', 'attributes/describeTypes.json') response = self._prepare_request('GET', 'attributes/describeTypes.json')
describe_types = self._check_response(response, expect_json=True) remote_describe_types = self._check_response(response, expect_json=True)
return describe_types['result'] return remote_describe_types['result']
@property @property
def recommended_pymisp_version(self): def recommended_pymisp_version(self):
@ -2111,7 +2109,7 @@ class ExpandedPyMISP(PyMISP):
if isinstance(data, dict): # Else, we can directly json encode. if isinstance(data, dict): # Else, we can directly json encode.
# Remove None values. # Remove None values.
data = {k: v for k, v in data.items() if v is not None} data = {k: v for k, v in data.items() if v is not None}
data = json.dumps(data, cls=MISPEncode) data = json.dumps(data, default=pymisp_json_default)
if kw_params: if kw_params:
# CakePHP params in URL # CakePHP params in URL

203
pymisp/mispevent.py
View File

@ -17,6 +17,7 @@ from deprecated import deprecated
from .abstract import AbstractMISP from .abstract import AbstractMISP
from .exceptions import UnknownMISPObjectTemplate, InvalidMISPObject, PyMISPError, NewEventError, NewAttributeError from .exceptions import UnknownMISPObjectTemplate, InvalidMISPObject, PyMISPError, NewEventError, NewAttributeError
logger = logging.getLogger('pymisp') logger = logging.getLogger('pymisp')
if sys.version_info < (3, 0): if sys.version_info < (3, 0):
@ -109,18 +110,11 @@ class MISPAttribute(AbstractMISP):
:strict: If false, fallback to sane defaults for the attribute type if the ones passed by the user are incorrect :strict: If false, fallback to sane defaults for the attribute type if the ones passed by the user are incorrect
""" """
super(MISPAttribute, self).__init__() super(MISPAttribute, self).__init__()
if not describe_types: if describe_types:
ressources_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data') self.describe_types = describe_types
with open(os.path.join(ressources_path, 'describeTypes.json'), 'rb') as f: self.__categories = self.describe_types['categories']
if OLD_PY3: self.__category_type_mapping = self.describe_types['category_type_mappings']
t = json.loads(f.read().decode()) self.__sane_default = self.describe_types['sane_defaults']
else:
t = json.load(f)
describe_types = t['result']
self.__categories = describe_types['categories']
self._types = describe_types['types']
self.__category_type_mapping = describe_types['category_type_mappings']
self.__sane_default = describe_types['sane_defaults']
self.__strict = strict self.__strict = strict
self._data = None self._data = None
self.uuid = str(uuid.uuid4()) self.uuid = str(uuid.uuid4())
@ -130,7 +124,7 @@ class MISPAttribute(AbstractMISP):
@property @property
def known_types(self): def known_types(self):
"""Returns a list of all the known MISP attributes types""" """Returns a list of all the known MISP attributes types"""
return self._types return self.describe_types['types']
@property @property
def malware_binary(self): def malware_binary(self):
@ -204,8 +198,8 @@ class MISPAttribute(AbstractMISP):
return misp_sighting return misp_sighting
def from_dict(self, **kwargs): def from_dict(self, **kwargs):
if kwargs.get('Attribute'): if 'Attribute' in kwargs:
kwargs = kwargs.get('Attribute') kwargs = kwargs['Attribute']
if kwargs.get('type') and kwargs.get('category'): if kwargs.get('type') and kwargs.get('category'):
if kwargs['type'] not in self.__category_type_mapping[kwargs['category']]: if kwargs['type'] not in self.__category_type_mapping[kwargs['category']]:
if self.__strict: if self.__strict:
@ -218,7 +212,7 @@ class MISPAttribute(AbstractMISP):
if self.type is None: if self.type is None:
raise NewAttributeError('The type of the attribute is required.') raise NewAttributeError('The type of the attribute is required.')
if self.type not in self.known_types: if self.type not in self.known_types:
raise NewAttributeError('{} is invalid, type has to be in {}'.format(self.type, (', '.join(self._types)))) raise NewAttributeError('{} is invalid, type has to be in {}'.format(self.type, (', '.join(self.known_types))))
type_defaults = self.__sane_default[self.type] type_defaults = self.__sane_default[self.type]
@ -278,14 +272,11 @@ class MISPAttribute(AbstractMISP):
raise NewAttributeError('If the distribution is set to sharing group, a sharing group ID is required (cannot be {}).'.format(self.sharing_group_id)) raise NewAttributeError('If the distribution is set to sharing group, a sharing group ID is required (cannot be {}).'.format(self.sharing_group_id))
if kwargs.get('Tag'): if kwargs.get('Tag'):
for tag in kwargs.pop('Tag'): [self.add_tag(tag) for tag in kwargs.pop('Tag')]
self.add_tag(tag)
if kwargs.get('Sighting'): if kwargs.get('Sighting'):
for sighting in kwargs.pop('Sighting'): [self.add_sighting(sighting) for sighting in kwargs.pop('Sighting')]
self.add_sighting(sighting)
if kwargs.get('ShadowAttribute'): if kwargs.get('ShadowAttribute'):
for s_attr in kwargs.pop('ShadowAttribute'): [self.add_shadow_attribute(s_attr) for s_attr in kwargs.pop('ShadowAttribute')]
self.add_shadow_attribute(s_attr)
# If the user wants to disable correlation, let them. Defaults to False. # If the user wants to disable correlation, let them. Defaults to False.
self.disable_correlation = kwargs.pop("disable_correlation", False) self.disable_correlation = kwargs.pop("disable_correlation", False)
@ -425,31 +416,18 @@ class MISPEvent(AbstractMISP):
def __init__(self, describe_types=None, strict_validation=False, **kwargs): def __init__(self, describe_types=None, strict_validation=False, **kwargs):
super(MISPEvent, self).__init__(**kwargs) super(MISPEvent, self).__init__(**kwargs)
ressources_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data')
if strict_validation: if strict_validation:
with open(os.path.join(ressources_path, 'schema.json'), 'rb') as f: schema_file = 'schema.json'
if OLD_PY3:
self.__json_schema = json.loads(f.read().decode())
else: else:
self.__json_schema = json.load(f) schema_file = 'schema-lax.json'
if sys.version_info >= (3, 6):
self.__json_schema = self._load_json(self.resources_path / schema_file)
else: else:
with open(os.path.join(ressources_path, 'schema-lax.json'), 'rb') as f: self.__json_schema = self._load_json(os.path.join(self.resources_path, schema_file))
if OLD_PY3:
self.__json_schema = json.loads(f.read().decode())
else:
self.__json_schema = json.load(f)
if describe_types: if describe_types:
# This variable is used in add_attribute in order to avoid duplicating the structure # This variable is used in add_attribute in order to avoid duplicating the structure
self._describe_types = describe_types self.describe_types = describe_types
else:
with open(os.path.join(ressources_path, 'describeTypes.json'), 'rb') as f:
if OLD_PY3:
t = json.loads(f.read().decode())
else:
t = json.load(f)
self._describe_types = t['result']
self._types = self._describe_types['types']
self.Attribute = [] self.Attribute = []
self.Object = [] self.Object = []
self.RelatedEvent = [] self.RelatedEvent = []
@ -457,7 +435,7 @@ class MISPEvent(AbstractMISP):
@property @property
def known_types(self): def known_types(self):
return self._types return self.describe_types['types']
@property @property
def org(self): def org(self):
@ -554,8 +532,8 @@ class MISPEvent(AbstractMISP):
raise NewEventError('Invalid format for the date: {} - {}'.format(date, type(date))) raise NewEventError('Invalid format for the date: {} - {}'.format(date, type(date)))
def from_dict(self, **kwargs): def from_dict(self, **kwargs):
if kwargs.get('Event'): if 'Event' in kwargs:
kwargs = kwargs.get('Event') kwargs = kwargs['Event']
# Required value # Required value
self.info = kwargs.pop('info', None) self.info = kwargs.pop('info', None)
if self.info is None: if self.info is None:
@ -587,8 +565,7 @@ class MISPEvent(AbstractMISP):
if kwargs.get('date'): if kwargs.get('date'):
self.set_date(kwargs.pop('date')) self.set_date(kwargs.pop('date'))
if kwargs.get('Attribute'): if kwargs.get('Attribute'):
for a in kwargs.pop('Attribute'): [self.add_attribute(**a) for a in kwargs.pop('Attribute')]
self.add_attribute(**a)
# All other keys # All other keys
if kwargs.get('id'): if kwargs.get('id'):
@ -615,11 +592,9 @@ class MISPEvent(AbstractMISP):
sub_event.load(rel_event) sub_event.load(rel_event)
self.RelatedEvent.append({'Event': sub_event}) self.RelatedEvent.append({'Event': sub_event})
if kwargs.get('Tag'): if kwargs.get('Tag'):
for tag in kwargs.pop('Tag'): [self.add_tag(tag) for tag in kwargs.pop('Tag')]
self.add_tag(tag)
if kwargs.get('Object'): if kwargs.get('Object'):
for obj in kwargs.pop('Object'): [self.add_object(obj) for obj in kwargs.pop('Object')]
self.add_object(obj)
if kwargs.get('Org'): if kwargs.get('Org'):
self.Org = MISPOrganisation() self.Org = MISPOrganisation()
self.Org.from_dict(**kwargs.pop('Org')) self.Org.from_dict(**kwargs.pop('Org'))
@ -720,7 +695,7 @@ class MISPEvent(AbstractMISP):
if isinstance(value, list): if isinstance(value, list):
attr_list = [self.add_attribute(type=type, value=a, **kwargs) for a in value] attr_list = [self.add_attribute(type=type, value=a, **kwargs) for a in value]
else: else:
attribute = MISPAttribute(describe_types=self._describe_types) attribute = MISPAttribute(describe_types=self.describe_types)
attribute.from_dict(type=type, value=value, **kwargs) attribute.from_dict(type=type, value=value, **kwargs)
self.attributes.append(attribute) self.attributes.append(attribute)
self.edited = True self.edited = True
@ -879,8 +854,8 @@ class MISPObjectReference(AbstractMISP):
super(MISPObjectReference, self).__init__() super(MISPObjectReference, self).__init__()
def from_dict(self, **kwargs): def from_dict(self, **kwargs):
if kwargs.get('ObjectReference'): if 'ObjectReference' in kwargs:
kwargs = kwargs.get('ObjectReference') kwargs = kwargs['ObjectReference']
super(MISPObjectReference, self).from_dict(**kwargs) super(MISPObjectReference, self).from_dict(**kwargs)
def __repr__(self): def __repr__(self):
@ -895,8 +870,8 @@ class MISPObjectTemplate(AbstractMISP):
super(MISPObjectTemplate, self).__init__() super(MISPObjectTemplate, self).__init__()
def from_dict(self, **kwargs): def from_dict(self, **kwargs):
if kwargs.get('ObjectTemplate'): if 'ObjectTemplate' in kwargs:
kwargs = kwargs.get('ObjectTemplate') kwargs = kwargs['ObjectTemplate']
super(MISPObjectTemplate, self).from_dict(**kwargs) super(MISPObjectTemplate, self).from_dict(**kwargs)
@ -906,8 +881,8 @@ class MISPUser(AbstractMISP):
super(MISPUser, self).__init__() super(MISPUser, self).__init__()
def from_dict(self, **kwargs): def from_dict(self, **kwargs):
if kwargs.get('User'): if 'User' in kwargs:
kwargs = kwargs.get('User') kwargs = kwargs['User']
super(MISPUser, self).from_dict(**kwargs) super(MISPUser, self).from_dict(**kwargs)
def __repr__(self): def __repr__(self):
@ -922,8 +897,8 @@ class MISPOrganisation(AbstractMISP):
super(MISPOrganisation, self).__init__() super(MISPOrganisation, self).__init__()
def from_dict(self, **kwargs): def from_dict(self, **kwargs):
if kwargs.get('Organisation'): if 'Organisation' in kwargs:
kwargs = kwargs.get('Organisation') kwargs = kwargs['Organisation']
super(MISPOrganisation, self).from_dict(**kwargs) super(MISPOrganisation, self).from_dict(**kwargs)
@ -933,8 +908,8 @@ class MISPFeed(AbstractMISP):
super(MISPFeed, self).__init__() super(MISPFeed, self).__init__()
def from_dict(self, **kwargs): def from_dict(self, **kwargs):
if kwargs.get('Feed'): if 'Feed' in kwargs:
kwargs = kwargs.get('Feed') kwargs = kwargs['Feed']
super(MISPFeed, self).from_dict(**kwargs) super(MISPFeed, self).from_dict(**kwargs)
@ -944,8 +919,8 @@ class MISPWarninglist(AbstractMISP):
super(MISPWarninglist, self).__init__() super(MISPWarninglist, self).__init__()
def from_dict(self, **kwargs): def from_dict(self, **kwargs):
if kwargs.get('Warninglist'): if 'Warninglist' in kwargs:
kwargs = kwargs.get('Warninglist') kwargs = kwargs['Warninglist']
super(MISPWarninglist, self).from_dict(**kwargs) super(MISPWarninglist, self).from_dict(**kwargs)
@ -955,8 +930,8 @@ class MISPTaxonomy(AbstractMISP):
super(MISPTaxonomy, self).__init__() super(MISPTaxonomy, self).__init__()
def from_dict(self, **kwargs): def from_dict(self, **kwargs):
if kwargs.get('Taxonomy'): if 'Taxonomy' in kwargs:
kwargs = kwargs.get('Taxonomy') kwargs = kwargs['Taxonomy']
super(MISPTaxonomy, self).from_dict(**kwargs) super(MISPTaxonomy, self).from_dict(**kwargs)
@ -966,8 +941,8 @@ class MISPGalaxy(AbstractMISP):
super(MISPGalaxy, self).__init__() super(MISPGalaxy, self).__init__()
def from_dict(self, **kwargs): def from_dict(self, **kwargs):
if kwargs.get('Galaxy'): if 'Galaxy' in kwargs:
kwargs = kwargs.get('Galaxy') kwargs = kwargs['Galaxy']
super(MISPGalaxy, self).from_dict(**kwargs) super(MISPGalaxy, self).from_dict(**kwargs)
@ -977,8 +952,8 @@ class MISPNoticelist(AbstractMISP):
super(MISPNoticelist, self).__init__() super(MISPNoticelist, self).__init__()
def from_dict(self, **kwargs): def from_dict(self, **kwargs):
if kwargs.get('Noticelist'): if 'Noticelist' in kwargs:
kwargs = kwargs.get('Noticelist') kwargs = kwargs['Noticelist']
super(MISPNoticelist, self).from_dict(**kwargs) super(MISPNoticelist, self).from_dict(**kwargs)
@ -988,8 +963,8 @@ class MISPRole(AbstractMISP):
super(MISPRole, self).__init__() super(MISPRole, self).__init__()
def from_dict(self, **kwargs): def from_dict(self, **kwargs):
if kwargs.get('Role'): if 'Role' in kwargs:
kwargs = kwargs.get('Role') kwargs = kwargs['Role']
super(MISPRole, self).from_dict(**kwargs) super(MISPRole, self).from_dict(**kwargs)
@ -999,8 +974,8 @@ class MISPServer(AbstractMISP):
super(MISPServer, self).__init__() super(MISPServer, self).__init__()
def from_dict(self, **kwargs): def from_dict(self, **kwargs):
if kwargs.get('Server'): if 'Server' in kwargs:
kwargs = kwargs.get('Server') kwargs = kwargs['Server']
super(MISPServer, self).from_dict(**kwargs) super(MISPServer, self).from_dict(**kwargs)
@ -1010,8 +985,8 @@ class MISPSharingGroup(AbstractMISP):
super(MISPSharingGroup, self).__init__() super(MISPSharingGroup, self).__init__()
def from_dict(self, **kwargs): def from_dict(self, **kwargs):
if kwargs.get('SharingGroup'): if 'SharingGroup' in kwargs:
kwargs = kwargs.get('SharingGroup') kwargs = kwargs['SharingGroup']
super(MISPSharingGroup, self).from_dict(**kwargs) super(MISPSharingGroup, self).from_dict(**kwargs)
@ -1021,8 +996,8 @@ class MISPLog(AbstractMISP):
super(MISPLog, self).__init__() super(MISPLog, self).__init__()
def from_dict(self, **kwargs): def from_dict(self, **kwargs):
if kwargs.get('Log'): if 'Log' in kwargs:
kwargs = kwargs.get('Log') kwargs = kwargs['Log']
super(MISPLog, self).from_dict(**kwargs) super(MISPLog, self).from_dict(**kwargs)
def __repr__(self): def __repr__(self):
@ -1035,8 +1010,8 @@ class MISPEventDelegation(AbstractMISP):
super(MISPEventDelegation, self).__init__() super(MISPEventDelegation, self).__init__()
def from_dict(self, **kwargs): def from_dict(self, **kwargs):
if kwargs.get('EventDelegation'): if 'EventDelegation' in kwargs:
kwargs = kwargs.get('EventDelegation') kwargs = kwargs['EventDelegation']
super(MISPEventDelegation, self).from_dict(**kwargs) super(MISPEventDelegation, self).from_dict(**kwargs)
def __repr__(self): def __repr__(self):
@ -1058,8 +1033,8 @@ class MISPSighting(AbstractMISP):
:type: Type of the sighting :type: Type of the sighting
:timestamp: Timestamp associated to the sighting :timestamp: Timestamp associated to the sighting
""" """
if kwargs.get('Sighting'): if 'Sighting' in kwargs:
kwargs = kwargs.get('Sighting') kwargs = kwargs['Sighting']
super(MISPSighting, self).from_dict(**kwargs) super(MISPSighting, self).from_dict(**kwargs)
def __repr__(self): def __repr__(self):
@ -1081,6 +1056,8 @@ class MISPObjectAttribute(MISPAttribute):
def from_dict(self, object_relation, value, **kwargs): def from_dict(self, object_relation, value, **kwargs):
self.object_relation = object_relation self.object_relation = object_relation
self.value = value self.value = value
if 'Attribute' in kwargs:
kwargs = kwargs['Attribute']
# Initialize the new MISPAttribute # Initialize the new MISPAttribute
# Get the misp attribute type from the definition # Get the misp attribute type from the definition
self.type = kwargs.pop('type', None) self.type = kwargs.pop('type', None)
@ -1098,6 +1075,8 @@ class MISPObjectAttribute(MISPAttribute):
if not self.type: if not self.type:
raise NewAttributeError("The type of the attribute is required. Is the object template missing?") raise NewAttributeError("The type of the attribute is required. Is the object template missing?")
super(MISPObjectAttribute, self).from_dict(**dict(self, **kwargs)) super(MISPObjectAttribute, self).from_dict(**dict(self, **kwargs))
# FIXME New syntax python3 only, keep for later.
# super(MISPObjectAttribute, self).from_dict(**{**self, **kwargs})
def __repr__(self): def __repr__(self):
if hasattr(self, 'value'): if hasattr(self, 'value'):
@ -1111,8 +1090,8 @@ class MISPShadowAttribute(AbstractMISP):
super(MISPShadowAttribute, self).__init__() super(MISPShadowAttribute, self).__init__()
def from_dict(self, **kwargs): def from_dict(self, **kwargs):
if kwargs.get('ShadowAttribute'): if 'ShadowAttribute' in kwargs:
kwargs = kwargs.get('ShadowAttribute') kwargs = kwargs['ShadowAttribute']
super(MISPShadowAttribute, self).from_dict(**kwargs) super(MISPShadowAttribute, self).from_dict(**kwargs)
def __repr__(self): def __repr__(self):
@ -1127,8 +1106,8 @@ class MISPCommunity(AbstractMISP):
super(MISPCommunity, self).__init__() super(MISPCommunity, self).__init__()
def from_dict(self, **kwargs): def from_dict(self, **kwargs):
if kwargs.get('Community'): if 'Community' in kwargs:
kwargs = kwargs.get('Community') kwargs = kwargs['Community']
super(MISPCommunity, self).from_dict(**kwargs) super(MISPCommunity, self).from_dict(**kwargs)
def __repr__(self): def __repr__(self):
@ -1155,20 +1134,7 @@ class MISPObject(AbstractMISP):
self.name = name self.name = name
self._known_template = False self._known_template = False
if kwargs.get('misp_objects_path_custom'): self._set_template(kwargs.get('misp_objects_path_custom'))
# If misp_objects_path_custom is given, and an object with the given name exists, use that.
self._known_template = self._load_template_path(os.path.join(kwargs.get('misp_objects_path_custom'), self.name, 'definition.json'))
if not self._known_template:
# Check if the object is known in the default templates bundled in with PyMISP
misp_objects_path = os.path.join(os.path.abspath(os.path.dirname(sys.modules['pymisp'].__file__)), 'data', 'misp-objects', 'objects')
self._known_template = self._load_template_path(os.path.join(misp_objects_path, self.name, 'definition.json'))
if not self._known_template and self._strict:
raise UnknownMISPObjectTemplate('{} is unknown in the MISP object directory.'.format(self.name))
else:
# Then we have no meta-category, template_uuid, description and template_version
pass
self.uuid = str(uuid.uuid4()) self.uuid = str(uuid.uuid4())
self.__fast_attribute_access = defaultdict(list) # Hashtable object_relation: [attributes] self.__fast_attribute_access = defaultdict(list) # Hashtable object_relation: [attributes]
@ -1206,11 +1172,7 @@ class MISPObject(AbstractMISP):
def _load_template_path(self, template_path): def _load_template_path(self, template_path):
if not os.path.exists(template_path): if not os.path.exists(template_path):
return False return False
with open(template_path, 'rb') as f: self._definition = self._load_json(template_path)
if OLD_PY3:
self._definition = json.loads(f.read().decode())
else:
self._definition = json.load(f)
setattr(self, 'meta-category', self._definition['meta-category']) setattr(self, 'meta-category', self._definition['meta-category'])
self.template_uuid = self._definition['uuid'] self.template_uuid = self._definition['uuid']
self.description = self._definition['description'] self.description = self._definition['description']
@ -1220,11 +1182,24 @@ class MISPObject(AbstractMISP):
def force_misp_objects_path_custom(self, misp_objects_path_custom, object_name=None): def force_misp_objects_path_custom(self, misp_objects_path_custom, object_name=None):
if object_name: if object_name:
self.name = object_name self.name = object_name
template_path = os.path.join(misp_objects_path_custom, self.name, 'definition.json') self._set_template(misp_objects_path_custom)
self._known_template = self._load_template_path(template_path) def _set_template(self, misp_objects_path_custom=None):
if not self._known_template: if misp_objects_path_custom:
raise UnknownMISPObjectTemplate('{} is unknown in the MISP object directory ({}).'.format(self.name, template_path)) # If misp_objects_path_custom is given, and an object with the given name exists, use that.
self.misp_objects_path = misp_objects_path_custom
# Try to get the template
if sys.version_info >= (3, 6):
self._known_template = self._load_template_path(self.misp_objects_path / self.name / 'definition.json')
else:
self._known_template = self._load_template_path(os.path.join(self.misp_objects_path, self.name, 'definition.json'))
if not self._known_template and self._strict:
raise UnknownMISPObjectTemplate('{} is unknown in the MISP object directory.'.format(self.name))
else:
# Then we have no meta-category, template_uuid, description and template_version
pass
@property @property
def disable_validation(self): def disable_validation(self):
@ -1254,8 +1229,8 @@ class MISPObject(AbstractMISP):
raise PyMISPError('All the attributes have to be of type MISPObjectReference.') raise PyMISPError('All the attributes have to be of type MISPObjectReference.')
def from_dict(self, **kwargs): def from_dict(self, **kwargs):
if kwargs.get('Object'): if 'Object' in kwargs:
kwargs = kwargs.get('Object') kwargs = kwargs['Object']
if self._known_template: if self._known_template:
if kwargs.get('template_uuid') and kwargs['template_uuid'] != self.template_uuid: if kwargs.get('template_uuid') and kwargs['template_uuid'] != self.template_uuid:
if self._strict: if self._strict:
@ -1283,11 +1258,9 @@ class MISPObject(AbstractMISP):
else: else:
self.timestamp = datetime.datetime.fromtimestamp(int(ts), UTC()) self.timestamp = datetime.datetime.fromtimestamp(int(ts), UTC())
if kwargs.get('Attribute'): if kwargs.get('Attribute'):
for a in kwargs.pop('Attribute'): [self.add_attribute(**a) for a in kwargs.pop('Attribute')]
self.add_attribute(**a)
if kwargs.get('ObjectReference'): if kwargs.get('ObjectReference'):
for r in kwargs.pop('ObjectReference'): [self.add_reference(**r) for r in kwargs.pop('ObjectReference')]
self.add_reference(**r)
# Not supported yet - https://github.com/MISP/PyMISP/issues/168 # Not supported yet - https://github.com/MISP/PyMISP/issues/168
# if kwargs.get('Tag'): # if kwargs.get('Tag'):
@ -1347,6 +1320,8 @@ class MISPObject(AbstractMISP):
attribute = MISPObjectAttribute({}) attribute = MISPObjectAttribute({})
# Overwrite the parameters of self._default_attributes_parameters with the ones of value # Overwrite the parameters of self._default_attributes_parameters with the ones of value
attribute.from_dict(object_relation=object_relation, **dict(self._default_attributes_parameters, **value)) attribute.from_dict(object_relation=object_relation, **dict(self._default_attributes_parameters, **value))
# FIXME New syntax python3 only, keep for later.
# attribute.from_dict(object_relation=object_relation, **{**self._default_attributes_parameters, **value})
self.__fast_attribute_access[object_relation].append(attribute) self.__fast_attribute_access[object_relation].append(attribute)
self.Attribute.append(attribute) self.Attribute.append(attribute)
self.edited = True self.edited = True

2
pymisp/tools/elfobject.py
View File

@ -25,6 +25,7 @@ except ImportError:
class ELFObject(AbstractMISPObjectGenerator): class ELFObject(AbstractMISPObjectGenerator):
def __init__(self, parsed=None, filepath=None, pseudofile=None, standalone=True, **kwargs): def __init__(self, parsed=None, filepath=None, pseudofile=None, standalone=True, **kwargs):
super(ELFObject, self).__init__('elf', standalone=standalone, **kwargs)
if not HAS_PYDEEP: if not HAS_PYDEEP:
logger.warning("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git") logger.warning("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git")
if not HAS_LIEF: if not HAS_LIEF:
@ -44,7 +45,6 @@ class ELFObject(AbstractMISPObjectGenerator):
self.__elf = parsed self.__elf = parsed
else: else:
raise InvalidMISPObject('Not a lief.ELF.Binary: {}'.format(type(parsed))) raise InvalidMISPObject('Not a lief.ELF.Binary: {}'.format(type(parsed)))
super(ELFObject, self).__init__('elf', standalone=standalone, **kwargs)
self.generate_attributes() self.generate_attributes()
def generate_attributes(self): def generate_attributes(self):

6
pymisp/tools/emailobject.py
View File

@ -13,6 +13,9 @@ logger = logging.getLogger('pymisp')
class EMailObject(AbstractMISPObjectGenerator): class EMailObject(AbstractMISPObjectGenerator):
def __init__(self, filepath=None, pseudofile=None, attach_original_email=True, standalone=True, **kwargs): def __init__(self, filepath=None, pseudofile=None, attach_original_email=True, standalone=True, **kwargs):
# PY3 way:
# super().__init__('file')
super(EMailObject, self).__init__('email', standalone=standalone, **kwargs)
if filepath: if filepath:
with open(filepath, 'rb') as f: with open(filepath, 'rb') as f:
self.__pseudofile = BytesIO(f.read()) self.__pseudofile = BytesIO(f.read())
@ -20,9 +23,6 @@ class EMailObject(AbstractMISPObjectGenerator):
self.__pseudofile = pseudofile self.__pseudofile = pseudofile
else: else:
raise InvalidMISPObject('File buffer (BytesIO) or a path is required.') raise InvalidMISPObject('File buffer (BytesIO) or a path is required.')
# PY3 way:
# super().__init__('file')
super(EMailObject, self).__init__('email', standalone=standalone, **kwargs)
self.__email = message_from_bytes(self.__pseudofile.getvalue(), policy=policy.default) self.__email = message_from_bytes(self.__pseudofile.getvalue(), policy=policy.default)
if attach_original_email: if attach_original_email:
self.add_attribute('eml', value='Full email.eml', data=self.__pseudofile) self.add_attribute('eml', value='Full email.eml', data=self.__pseudofile)

6
pymisp/tools/fileobject.py
View File

@ -29,6 +29,9 @@ except ImportError:
class FileObject(AbstractMISPObjectGenerator): class FileObject(AbstractMISPObjectGenerator):
def __init__(self, filepath=None, pseudofile=None, filename=None, standalone=True, **kwargs): def __init__(self, filepath=None, pseudofile=None, filename=None, standalone=True, **kwargs):
# PY3 way:
# super().__init__('file')
super(FileObject, self).__init__('file', standalone=standalone, **kwargs)
if not HAS_PYDEEP: if not HAS_PYDEEP:
logger.warning("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git") logger.warning("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git")
if not HAS_MAGIC: if not HAS_MAGIC:
@ -49,9 +52,6 @@ class FileObject(AbstractMISPObjectGenerator):
self.__pseudofile = pseudofile self.__pseudofile = pseudofile
else: else:
raise InvalidMISPObject('File buffer (BytesIO) or a path is required.') raise InvalidMISPObject('File buffer (BytesIO) or a path is required.')
# PY3 way:
# super().__init__('file')
super(FileObject, self).__init__('file', standalone=standalone, **kwargs)
self.__data = self.__pseudofile.getvalue() self.__data = self.__pseudofile.getvalue()
self.generate_attributes() self.generate_attributes()

6
pymisp/tools/machoobject.py
View File

@ -26,6 +26,9 @@ except ImportError:
class MachOObject(AbstractMISPObjectGenerator): class MachOObject(AbstractMISPObjectGenerator):
def __init__(self, parsed=None, filepath=None, pseudofile=None, standalone=True, **kwargs): def __init__(self, parsed=None, filepath=None, pseudofile=None, standalone=True, **kwargs):
# Python3 way
# super().__init__('elf')
super(MachOObject, self).__init__('macho', standalone=standalone, **kwargs)
if not HAS_PYDEEP: if not HAS_PYDEEP:
logger.warning("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git") logger.warning("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git")
if not HAS_LIEF: if not HAS_LIEF:
@ -45,9 +48,6 @@ class MachOObject(AbstractMISPObjectGenerator):
self.__macho = parsed self.__macho = parsed
else: else:
raise InvalidMISPObject('Not a lief.MachO.Binary: {}'.format(type(parsed))) raise InvalidMISPObject('Not a lief.MachO.Binary: {}'.format(type(parsed)))
# Python3 way
# super().__init__('elf')
super(MachOObject, self).__init__('macho', standalone=standalone, **kwargs)
self.generate_attributes() self.generate_attributes()
def generate_attributes(self): def generate_attributes(self):

6
pymisp/tools/peobject.py
View File

@ -26,6 +26,9 @@ except ImportError:
class PEObject(AbstractMISPObjectGenerator): class PEObject(AbstractMISPObjectGenerator):
def __init__(self, parsed=None, filepath=None, pseudofile=None, standalone=True, **kwargs): def __init__(self, parsed=None, filepath=None, pseudofile=None, standalone=True, **kwargs):
# Python3 way
# super().__init__('pe')
super(PEObject, self).__init__('pe', standalone=standalone, **kwargs)
if not HAS_PYDEEP: if not HAS_PYDEEP:
logger.warning("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git") logger.warning("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git")
if not HAS_LIEF: if not HAS_LIEF:
@ -45,9 +48,6 @@ class PEObject(AbstractMISPObjectGenerator):
self.__pe = parsed self.__pe = parsed
else: else:
raise InvalidMISPObject('Not a lief.PE.Binary: {}'.format(type(parsed))) raise InvalidMISPObject('Not a lief.PE.Binary: {}'.format(type(parsed)))
# Python3 way
# super().__init__('pe')
super(PEObject, self).__init__('pe', standalone=standalone, **kwargs)
self.generate_attributes() self.generate_attributes()
def _is_exe(self): def _is_exe(self):

6
pymisp/tools/sshauthkeyobject.py
View File

@ -12,6 +12,9 @@ logger = logging.getLogger('pymisp')
class SSHAuthorizedKeysObject(AbstractMISPObjectGenerator): class SSHAuthorizedKeysObject(AbstractMISPObjectGenerator):
def __init__(self, authorized_keys_path=None, authorized_keys_pseudofile=None, standalone=True, **kwargs): def __init__(self, authorized_keys_path=None, authorized_keys_pseudofile=None, standalone=True, **kwargs):
# PY3 way:
# super().__init__('file')
super(SSHAuthorizedKeysObject, self).__init__('ssh-authorized-keys', standalone=standalone, **kwargs)
if authorized_keys_path: if authorized_keys_path:
with open(authorized_keys_path, 'r') as f: with open(authorized_keys_path, 'r') as f:
self.__pseudofile = StringIO(f.read()) self.__pseudofile = StringIO(f.read())
@ -19,9 +22,6 @@ class SSHAuthorizedKeysObject(AbstractMISPObjectGenerator):
self.__pseudofile = authorized_keys_path self.__pseudofile = authorized_keys_path
else: else:
raise InvalidMISPObject('File buffer (StringIO) or a path is required.') raise InvalidMISPObject('File buffer (StringIO) or a path is required.')
# PY3 way:
# super().__init__('file')
super(SSHAuthorizedKeysObject, self).__init__('ssh-authorized-keys', standalone=standalone, **kwargs)
self.__data = self.__pseudofile.getvalue() self.__data = self.__pseudofile.getvalue()
self.generate_attributes() self.generate_attributes()

2
setup.py
View File

@ -41,7 +41,7 @@ setup(
], ],
install_requires=['six', 'requests', 'python-dateutil', 'jsonschema', install_requires=['six', 'requests', 'python-dateutil', 'jsonschema',
'python-dateutil', 'enum34;python_version<"3.4"', 'python-dateutil', 'enum34;python_version<"3.4"',
'functools32;python_version<"3.0"', 'deprecated'], 'functools32;python_version<"3.0"', 'deprecated', 'cachetools;python_version<"3.0"'],
extras_require={'fileobjects': ['lief>=0.8,<0.10;python_version<"3.5"', 'lief>=0.10.0.dev0;python_version>"3.5"', 'python-magic', 'pydeep'], extras_require={'fileobjects': ['lief>=0.8,<0.10;python_version<"3.5"', 'lief>=0.10.0.dev0;python_version>"3.5"', 'python-magic', 'pydeep'],
'neo': ['py2neo'], 'neo': ['py2neo'],
'openioc': ['beautifulsoup4'], 'openioc': ['beautifulsoup4'],

4
tests/mispevent_testfiles/existing_event.json
View File

@ -3919,7 +3919,7 @@
"date": "2017-12-14", "date": "2017-12-14",
"distribution": "3", "distribution": "3",
"id": "9616", "id": "9616",
"info": "OSINT - Attackers Deploy New ICS Attack Framework “TRITON” and Cause Operational Disruption to Critical Infrastructure", "info": "OSINT - Attackers Deploy New ICS Attack Framework \"TRITON\" and Cause Operational Disruption to Critical Infrastructure",
"org_id": "2", "org_id": "2",
"orgc_id": "2", "orgc_id": "2",
"published": false, "published": false,
@ -4019,7 +4019,7 @@
"date": "2017-10-23", "date": "2017-10-23",
"distribution": "3", "distribution": "3",
"id": "9208", "id": "9208",
"info": "Talos: “Cyber Conflict” Decoy Document Used In Real Cyber Conflict", "info": "Talos: \"Cyber Conflict\" Decoy Document Used In Real Cyber Conflict",
"org_id": "291", "org_id": "291",
"orgc_id": "291", "orgc_id": "291",
"published": true, "published": true,

4
tests/mispevent_testfiles/existing_event_edited.json
View File

@ -3922,7 +3922,7 @@
"date": "2017-12-14", "date": "2017-12-14",
"distribution": "3", "distribution": "3",
"id": "9616", "id": "9616",
"info": "OSINT - Attackers Deploy New ICS Attack Framework “TRITON” and Cause Operational Disruption to Critical Infrastructure", "info": "OSINT - Attackers Deploy New ICS Attack Framework \"TRITON\" and Cause Operational Disruption to Critical Infrastructure",
"org_id": "2", "org_id": "2",
"orgc_id": "2", "orgc_id": "2",
"published": false, "published": false,
@ -4022,7 +4022,7 @@
"date": "2017-10-23", "date": "2017-10-23",
"distribution": "3", "distribution": "3",
"id": "9208", "id": "9208",
"info": "Talos: “Cyber Conflict” Decoy Document Used In Real Cyber Conflict", "info": "Talos: \"Cyber Conflict\" Decoy Document Used In Real Cyber Conflict",
"org_id": "291", "org_id": "291",
"orgc_id": "291", "orgc_id": "291",
"published": true, "published": true,

2
tests/test.py
View File

@ -6,7 +6,7 @@ try:
except ImportError as e: except ImportError as e:
print(e) print(e)
url = 'https://localhost:8443' url = 'https://localhost:8443'
key = 'K5yV0CcxdnklzDfCKlnPniIxrMX41utQ2dG13zZ3' key = 'd6OmdDFvU3Seau3UjwvHS1y3tFQbaRNhJhDX0tjh'
import time import time

41
tests/test_mispevent.py
View File

@ -26,20 +26,20 @@ class TestMISPEvent(unittest.TestCase):
def test_simple(self): def test_simple(self):
with open('tests/mispevent_testfiles/simple.json', 'r') as f: with open('tests/mispevent_testfiles/simple.json', 'r') as f:
ref_json = json.load(f) ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2))
def test_event(self): def test_event(self):
self.init_event() self.init_event()
self.mispevent.publish() self.mispevent.publish()
with open('tests/mispevent_testfiles/event.json', 'r') as f: with open('tests/mispevent_testfiles/event.json', 'r') as f:
ref_json = json.load(f) ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2))
def test_loadfile(self): def test_loadfile(self):
self.mispevent.load_file('tests/mispevent_testfiles/event.json') self.mispevent.load_file('tests/mispevent_testfiles/event.json')
with open('tests/mispevent_testfiles/event.json', 'r') as f: with open('tests/mispevent_testfiles/event.json', 'r') as f:
ref_json = json.load(f) ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2))
def test_event_tag(self): def test_event_tag(self):
self.init_event() self.init_event()
@ -50,7 +50,7 @@ class TestMISPEvent(unittest.TestCase):
self.mispevent.add_tag(new_tag) self.mispevent.add_tag(new_tag)
with open('tests/mispevent_testfiles/event_tags.json', 'r') as f: with open('tests/mispevent_testfiles/event_tags.json', 'r') as f:
ref_json = json.load(f) ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2))
def test_attribute(self): def test_attribute(self):
self.init_event() self.init_event()
@ -62,13 +62,13 @@ class TestMISPEvent(unittest.TestCase):
self.assertEqual(attr_tags[0].name, 'osint') self.assertEqual(attr_tags[0].name, 'osint')
with open('tests/mispevent_testfiles/attribute.json', 'r') as f: with open('tests/mispevent_testfiles/attribute.json', 'r') as f:
ref_json = json.load(f) ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2))
# Fake setting an attribute ID for testing # Fake setting an attribute ID for testing
self.mispevent.attributes[0].id = 42 self.mispevent.attributes[0].id = 42
self.mispevent.delete_attribute(42) self.mispevent.delete_attribute(42)
with open('tests/mispevent_testfiles/attribute_del.json', 'r') as f: with open('tests/mispevent_testfiles/attribute_del.json', 'r') as f:
ref_json = json.load(f) ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2))
def test_object_tag(self): def test_object_tag(self):
self.mispevent.add_object(name='file', strict=True) self.mispevent.add_object(name='file', strict=True)
@ -90,7 +90,7 @@ class TestMISPEvent(unittest.TestCase):
self.assertEqual(self.mispevent.objects[0].references[0].relationship_type, 'baz') self.assertEqual(self.mispevent.objects[0].references[0].relationship_type, 'baz')
with open('tests/mispevent_testfiles/event_obj_attr_tag.json', 'r') as f: with open('tests/mispevent_testfiles/event_obj_attr_tag.json', 'r') as f:
ref_json = json.load(f) ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2))
@unittest.skip("Not supported on MISP: https://github.com/MISP/MISP/issues/2638 - https://github.com/MISP/PyMISP/issues/168") @unittest.skip("Not supported on MISP: https://github.com/MISP/MISP/issues/2638 - https://github.com/MISP/PyMISP/issues/168")
def test_object_level_tag(self): def test_object_level_tag(self):
@ -100,7 +100,7 @@ class TestMISPEvent(unittest.TestCase):
self.mispevent.objects[0].uuid = 'a' self.mispevent.objects[0].uuid = 'a'
with open('tests/mispevent_testfiles/event_obj_tag.json', 'r') as f: with open('tests/mispevent_testfiles/event_obj_tag.json', 'r') as f:
ref_json = json.load(f) ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2))
def test_malware(self): def test_malware(self):
with open('tests/mispevent_testfiles/simple.json', 'rb') as f: with open('tests/mispevent_testfiles/simple.json', 'rb') as f:
@ -112,7 +112,7 @@ class TestMISPEvent(unittest.TestCase):
self.assertEqual(attribute.malware_binary, pseudofile) self.assertEqual(attribute.malware_binary, pseudofile)
with open('tests/mispevent_testfiles/malware.json', 'r') as f: with open('tests/mispevent_testfiles/malware.json', 'r') as f:
ref_json = json.load(f) ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2))
def test_existing_malware(self): def test_existing_malware(self):
self.mispevent.load_file('tests/mispevent_testfiles/malware_exist.json') self.mispevent.load_file('tests/mispevent_testfiles/malware_exist.json')
@ -127,19 +127,20 @@ class TestMISPEvent(unittest.TestCase):
sighting.from_dict(value='1', type='bar', timestamp=11111111) sighting.from_dict(value='1', type='bar', timestamp=11111111)
with open('tests/mispevent_testfiles/sighting.json', 'r') as f: with open('tests/mispevent_testfiles/sighting.json', 'r') as f:
ref_json = json.load(f) ref_json = json.load(f)
self.assertEqual(sighting.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) self.assertEqual(sighting.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2))
def test_existing_event(self): def test_existing_event(self):
self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
with open('tests/mispevent_testfiles/existing_event.json', 'r') as f: with open('tests/mispevent_testfiles/existing_event.json', 'r') as f:
ref_json = json.load(f) ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2))
def test_shadow_attributes_existing(self): def test_shadow_attributes_existing(self):
self.mispevent.load_file('tests/mispevent_testfiles/shadow.json') self.mispevent.load_file('tests/mispevent_testfiles/shadow.json')
with open('tests/mispevent_testfiles/shadow.json', 'r') as f: with open('tests/mispevent_testfiles/shadow.json', 'r') as f:
ref_json = json.load(f) ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2))
@unittest.skip("Not supported on MISP.") @unittest.skip("Not supported on MISP.")
def test_shadow_attributes(self): def test_shadow_attributes(self):
@ -152,7 +153,7 @@ class TestMISPEvent(unittest.TestCase):
del p.uuid del p.uuid
with open('tests/mispevent_testfiles/proposals.json', 'r') as f: with open('tests/mispevent_testfiles/proposals.json', 'r') as f:
ref_json = json.load(f) ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2))
def test_default_attributes(self): def test_default_attributes(self):
self.mispevent.add_object(name='file', strict=True) self.mispevent.add_object(name='file', strict=True)
@ -165,7 +166,7 @@ class TestMISPEvent(unittest.TestCase):
self.mispevent.objects[1].uuid = 'b' self.mispevent.objects[1].uuid = 'b'
with open('tests/mispevent_testfiles/event_obj_def_param.json', 'r') as f: with open('tests/mispevent_testfiles/event_obj_def_param.json', 'r') as f:
ref_json = json.load(f) ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2))
def test_obj_default_values(self): def test_obj_default_values(self):
self.init_event() self.init_event()
@ -181,7 +182,7 @@ class TestMISPEvent(unittest.TestCase):
self.mispevent.objects[0].uuid = 'a' self.mispevent.objects[0].uuid = 'a'
with open('tests/mispevent_testfiles/def_param.json', 'r') as f: with open('tests/mispevent_testfiles/def_param.json', 'r') as f:
ref_json = json.load(f) ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2))
def test_event_not_edited(self): def test_event_not_edited(self):
self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
@ -246,7 +247,7 @@ class TestMISPEvent(unittest.TestCase):
self.assertTrue(self.mispevent.edited) self.assertTrue(self.mispevent.edited)
with open('tests/mispevent_testfiles/existing_event_edited.json', 'r') as f: with open('tests/mispevent_testfiles/existing_event_edited.json', 'r') as f:
ref_json = json.load(f) ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2))
def test_obj_by_id(self): def test_obj_by_id(self):
self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
@ -258,7 +259,7 @@ class TestMISPEvent(unittest.TestCase):
self.mispevent.add_object(name='test_object_template', strict=True, misp_objects_path_custom='tests/mispevent_testfiles') self.mispevent.add_object(name='test_object_template', strict=True, misp_objects_path_custom='tests/mispevent_testfiles')
with self.assertRaises(InvalidMISPObject) as e: with self.assertRaises(InvalidMISPObject) as e:
# Fail on required # Fail on required
self.mispevent.to_json() self.mispevent.to_json(sort_keys=True, indent=2)
if sys.version_info >= (3, ): if sys.version_info >= (3, ):
self.assertEqual(e.exception.message, '{\'member3\'} are required.') self.assertEqual(e.exception.message, '{\'member3\'} are required.')
else: else:
@ -269,7 +270,7 @@ class TestMISPEvent(unittest.TestCase):
del a.uuid del a.uuid
with self.assertRaises(InvalidMISPObject) as e: with self.assertRaises(InvalidMISPObject) as e:
# Fail on requiredOneOf # Fail on requiredOneOf
self.mispevent.to_json() self.mispevent.to_json(sort_keys=True, indent=2)
self.assertEqual(e.exception.message, 'At least one of the following attributes is required: member1, member2') self.assertEqual(e.exception.message, 'At least one of the following attributes is required: member1, member2')
a = self.mispevent.objects[0].add_attribute('member1', value='bar') a = self.mispevent.objects[0].add_attribute('member1', value='bar')
@ -278,14 +279,14 @@ class TestMISPEvent(unittest.TestCase):
del a.uuid del a.uuid
with self.assertRaises(InvalidMISPObject) as e: with self.assertRaises(InvalidMISPObject) as e:
# member1 is not a multiple # member1 is not a multiple
self.mispevent.to_json() self.mispevent.to_json(sort_keys=True, indent=2)
self.assertEqual(e.exception.message, 'Multiple occurrences of member1 is not allowed') self.assertEqual(e.exception.message, 'Multiple occurrences of member1 is not allowed')
self.mispevent.objects[0].attributes = self.mispevent.objects[0].attributes[:2] self.mispevent.objects[0].attributes = self.mispevent.objects[0].attributes[:2]
self.mispevent.objects[0].uuid = 'a' self.mispevent.objects[0].uuid = 'a'
with open('tests/mispevent_testfiles/misp_custom_obj.json', 'r') as f: with open('tests/mispevent_testfiles/misp_custom_obj.json', 'r') as f:
ref_json = json.load(f) ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2))
if __name__ == '__main__': if __name__ == '__main__':

2
tests/testlive_comprehensive.py
View File

@ -1577,7 +1577,7 @@ class TestComprehensive(unittest.TestCase):
remote_types = remote.pop('types') remote_types = remote.pop('types')
remote_categories = remote.pop('categories') remote_categories = remote.pop('categories')
remote_category_type_mappings = remote.pop('category_type_mappings') remote_category_type_mappings = remote.pop('category_type_mappings')
local = self.admin_misp_connector.describe_types_local local = dict(self.admin_misp_connector.describe_types_local)
local_types = local.pop('types') local_types = local.pop('types')
local_categories = local.pop('categories') local_categories = local.pop('categories')
local_category_type_mappings = local.pop('category_type_mappings') local_category_type_mappings = local.pop('category_type_mappings')