diff --git a/Pipfile b/Pipfile index 0ae5f3d..c5d5c0f 100644 --- a/Pipfile +++ b/Pipfile @@ -10,6 +10,7 @@ codecov = "*" requests-mock = "*" pymisp = {editable = true,extras = ["fileobjects", "neo", "openioc", "virustotal", "pdfexport", "docs"],path = "."} docutils = "==0.15" +memory-profiler = "*" [packages] pymisp = {editable = true,extras = ["fileobjects", "openioc", "virustotal", "pdfexport"],path = "."} diff --git a/Pipfile.lock b/Pipfile.lock index 60f7bd7..becc122 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "4b4cf20ef3242efd0c24d7cc54ba2438dee8ba853ab3b9384ad915448ce83048" + "sha256": "4be7259a433785d74e1879a4a555bb669d50c5f409d0a094652c1abc9b1227c5" }, "pipfile-spec": 6, "requires": { @@ -25,11 +25,10 @@ }, "beautifulsoup4": { "hashes": [ - "sha256:05668158c7b85b791c5abde53e50265e16f98ad601c402ba44d70f96c4159612", - "sha256:25288c9e176f354bf277c0a10aa96c782a6a18a17122dba2e8cec4a97e03343b", - "sha256:f040590be10520f2ea4c2ae8c3dae441c7cfff5308ec9d58a0ec0c1b8f81d469" + "sha256:5279c36b4b2ec2cb4298d723791467e3000e5384a43ea0cdf5d45207c7e97169", + "sha256:dcdef580e18a76d54002088602eba453eec38ebbcafafeaabd8cab12b6155d57" ], - "version": "==4.8.0" + "version": "==4.8.1" }, "certifi": { "hashes": [ @@ -92,34 +91,34 @@ }, "pillow": { "hashes": [ - "sha256:0804f77cb1e9b6dbd37601cee11283bba39a8d44b9ddb053400c58e0c0d7d9de", - "sha256:0ab7c5b5d04691bcbd570658667dd1e21ca311c62dcfd315ad2255b1cd37f64f", - "sha256:0b3e6cf3ea1f8cecd625f1420b931c83ce74f00c29a0ff1ce4385f99900ac7c4", - "sha256:365c06a45712cd723ec16fa4ceb32ce46ad201eb7bbf6d3c16b063c72b61a3ed", - "sha256:38301fbc0af865baa4752ddae1bb3cbb24b3d8f221bf2850aad96b243306fa03", - "sha256:3aef1af1a91798536bbab35d70d35750bd2884f0832c88aeb2499aa2d1ed4992", - "sha256:3fe0ab49537d9330c9bba7f16a5f8b02da615b5c809cdf7124f356a0f182eccd", - "sha256:45a619d5c1915957449264c81c008934452e3fd3604e36809212300b2a4dab68", - "sha256:49f90f147883a0c3778fd29d3eb169d56416f25758d0f66775db9184debc8010", - "sha256:571b5a758baf1cb6a04233fb23d6cf1ca60b31f9f641b1700bfaab1194020555", - "sha256:5ac381e8b1259925287ccc5a87d9cf6322a2dc88ae28a97fe3e196385288413f", - "sha256:6153db744a743c0c8c91b8e3b9d40e0b13a5d31dbf8a12748c6d9bfd3ddc01ad", - "sha256:6fd63afd14a16f5d6b408f623cc2142917a1f92855f0df997e09a49f0341be8a", - "sha256:70acbcaba2a638923c2d337e0edea210505708d7859b87c2bd81e8f9902ae826", - "sha256:70b1594d56ed32d56ed21a7fbb2a5c6fd7446cdb7b21e749c9791eac3a64d9e4", - "sha256:76638865c83b1bb33bcac2a61ce4d13c17dba2204969dedb9ab60ef62bede686", - "sha256:7b2ec162c87fc496aa568258ac88631a2ce0acfe681a9af40842fc55deaedc99", - "sha256:7cee2cef07c8d76894ebefc54e4bb707dfc7f258ad155bd61d87f6cd487a70ff", - "sha256:7d16d4498f8b374fc625c4037742fbdd7f9ac383fd50b06f4df00c81ef60e829", - "sha256:b50bc1780681b127e28f0075dfb81d6135c3a293e0c1d0211133c75e2179b6c0", - "sha256:bd0582f831ad5bcad6ca001deba4568573a4675437db17c4031939156ff339fa", - "sha256:cfd40d8a4b59f7567620410f966bb1f32dc555b2b19f82a91b147fac296f645c", - "sha256:e3ae410089de680e8f84c68b755b42bc42c0ceb8c03dbea88a5099747091d38e", - "sha256:e9046e559c299b395b39ac7dbf16005308821c2f24a63cae2ab173bd6aa11616", - "sha256:ef6be704ae2bc8ad0ebc5cb850ee9139493b0fc4e81abcc240fb392a63ebc808", - "sha256:f8dc19d92896558f9c4317ee365729ead9d7bbcf2052a9a19a3ef17abbb8ac5b" + "sha256:00fdeb23820f30e43bba78eb9abb00b7a937a655de7760b2e09101d63708b64e", + "sha256:01f948e8220c85eae1aa1a7f8edddcec193918f933fb07aaebe0bfbbcffefbf1", + "sha256:08abf39948d4b5017a137be58f1a52b7101700431f0777bec3d897c3949f74e6", + "sha256:099a61618b145ecb50c6f279666bbc398e189b8bc97544ae32b8fcb49ad6b830", + "sha256:2c1c61546e73de62747e65807d2cc4980c395d4c5600ecb1f47a650c6fa78c79", + "sha256:2ed9c4f694861642401f27dc3cb99772be67cd190e84845c749dae0a06c3bfae", + "sha256:338581b30b908e111be578f0297255f6b57a51358cd16fa0e6f664c9a1f88bff", + "sha256:38c7d48a21cd06fdeee93987147b9b1c55b73b4cfcbf83240568bfbd5adee447", + "sha256:43fd026f613c8e48a25eba1a92f4d2ad7f3903c95d8c33a11611a7717d2ab654", + "sha256:4548236844327a718ce3bb182ab32a16fa2050c61e334e959f554cac052fb0df", + "sha256:5090857876c58885cfa388dc649e5db30aae98a068c26f3fd0ac9d7d9a4d9572", + "sha256:5bbba34f97a26a93f5e8dec469ca4ddd712451418add43da946dbaed7f7a98d2", + "sha256:65a28969a025a0eb4594637b6103201dc4ed2a9508bdab56ac33e43e3081c404", + "sha256:892bb52b70bd5ea9dbbc3ac44f38e84f5a04e9d8b1bff48159d96cb795b81159", + "sha256:8a9becd5cbd5062f973bcd2e7bc79483af310222de112b6541f8af1f93a3cc42", + "sha256:972a7aaeb7c4a2795b52eef52ee991ef040b31009f36deca6207a986607b55f3", + "sha256:97b119c436bfa96a92ac2ca525f7025836d4d4e64b1c9f9eff8dbaf3ff1d86f3", + "sha256:9ba37698e242223f8053cc158f130aee046a96feacbeab65893dbe94f5530118", + "sha256:b1b0e1f626a0f079c0d3696db70132fb1f29aa87c66aecb6501a9b8be64ce9f7", + "sha256:c14c1224fd1a5be2733530d648a316974dbbb3c946913562c6005a76f21ca042", + "sha256:c79a8546c48ae6465189e54e3245a97ddf21161e33ff7eaa42787353417bb2b6", + "sha256:ceb76935ac4ebdf6d7bc845482a4450b284c6ccfb281e34da51d510658ab34d8", + "sha256:e22bffaad04b4d16e1c091baed7f2733fc1ebb91e0c602abf1b6834d17158b1f", + "sha256:ec883b8e44d877bda6f94a36313a1c6063f8b1997aa091628ae2f34c7f97c8d5", + "sha256:f1baa54d50ec031d1a9beb89974108f8f2c0706f49798f4777df879df0e1adb6", + "sha256:f53a5385932cda1e2c862d89460992911a89768c65d176ff8c50cddca4d29bed" ], - "version": "==6.1.0" + "version": "==6.2.0" }, "pydeep": { "hashes": [ @@ -164,31 +163,31 @@ }, "reportlab": { "hashes": [ - "sha256:035346299a556c378a57cc163f2dfd945feebd5e45c844bbd02cae0711f780b5", - "sha256:05dafbe4cb0801fff0d0956b2d474e79d91d3b48923b6102a28cce0fcb2d8e53", - "sha256:0b28d4407caa6932539e8f9761e0430a96c6939713ee4b2f27b6c7af327c69e1", - "sha256:10cda714d7da684b2332d4d435cf90472fb2ea031ebbfb7f509a31d5898c06b3", - "sha256:152c321ca3caa564a5da1c33cb1937af983184eb7b3830f505770728a3f2f075", - "sha256:1c522ef6656abaf742c0995bf4fc19b7284732c7f3a00e5cd901d48187a9e8f8", - "sha256:24ee58e0905d4c0ec13557212de7c36ea92f87d688814191fab19d70294afb0e", - "sha256:43145d034423c97db1ac0db4f5e20ed6d16a1161a64e3da3f93a8a9b1eae78f1", - "sha256:43e433fd601259f4cfdc1c076ade9537c31bd116c4f5aa48d66713a609f8ee2c", - "sha256:5156e1e1f0b7fb8864bafc73ba02f8fa18ca46bf6702b34472f9033edf35cb3b", - "sha256:55d901dae80a30cd3b6a16accfbdf87f2fff6d379fd78ba6b87ae437391a5a2f", - "sha256:67fd4229ff0b643a0c2792ac5274ed7edaffc07052ad5dcd7d6f7a5d641a7446", - "sha256:739e81a32cd51be8d8225d316b1399d4b6ff6ee109963d3b5320b0718ce2aa0f", - "sha256:801aa0887135713ab107afd51e58dc2bb951fb3ffd773528b96924f90f82558f", - "sha256:835985ac466b3ac5e0751532f135aebc50c204837ebbc781d9a6313f4b76ab5a", - "sha256:87b3a4cedb40c08a7708d870cd7c405effb7a6a29cedf2dde380219d7a4d3cee", - "sha256:8c1abc9a4fead2d9255dfd1d58e5e1025c1816d2c38f6f0a7f50e42d51543c1d", - "sha256:97368b3f9622454fac3fe794664da5c09e4ec4ffa3fcbbf11a6de07e1ee35f41", - "sha256:9ca02a68b539969f9396ff485c574e6843a2433f6c165e8958a97d9366e1df0c", - "sha256:a43dba290c8c46e12f2cc142e4753e68e98ca088ca1dbbfce8822ca4436e7bf8", - "sha256:ac9414b875c6dc4fd46a01994612dece57a5be14b53c528f45ff85d82e332dfd", - "sha256:b58af0f12434835efa494f081c14bf5293764c78f58e6b32d941db5fff5b1692", - "sha256:b6e4a1b2f774f6c713bece60f2c06bb662985bdefffe0bc8582d3ef912985ba5" + "sha256:06b7c7436fa6d4844c7637161f3297c7a96240f35622ab2d219e4fd8387c0ab2", + "sha256:0a5acf67bd9812e38ed84be8994c07a8136b0a8f4c14a1c66c9c73a9567a9a44", + "sha256:1c8ca145d03e3c620866b06febb241b179197b58fb07454fbc8e9d6184cdcc93", + "sha256:2f8d785660ee316874c86abad345633ce8c652e88e03ae8a10f1fdadc72fd23d", + "sha256:4869d342352c92a812ce40555ef2a9cfbd722390d67fe61f1d6ec770e9ca41a3", + "sha256:493e0dcd9c085d46acf4fe3f00f941e562490a74b651409039a0dee2a0d76555", + "sha256:4e606e3ee9345e68cd205022d526250ad2a1164eea8f1e29d77d6ad08631b0ba", + "sha256:5bf91bae8995db91650fda658129c268515358b756fd16c0261a9dd641df1856", + "sha256:6df0730f8f715aa12333bd6d2a72eea3a989381451861186d9b5e71889454ac7", + "sha256:7195c6ea096d10c91cc470f9f0ced3ad74470d9c0fd97923b5e764597dd13671", + "sha256:7431c979e2b498e8e20abf458f360a451717d76c3c1bd49d1fc5697d3504f8e5", + "sha256:7f7f70a8d4b573d1ff65a81415b4b6ed9545630f381dff4a69307640e09d381d", + "sha256:9945433667a46f054d1125b4ca86fe9ee31feb254728b38242e9a6008c135efe", + "sha256:b1cdbfc1fd54ac947b9f0114e00ab94e945db679f1e03357a3c00f3a85e73eea", + "sha256:bf149847a2fd8f24b788a8abbf97a2b9a73edc5b1bd719384b786eb84bcad15e", + "sha256:ce514bfce2bf3e302f52aba9929fe3ac7d918cfea2f5d3e30bf9dac9658bf094", + "sha256:d243d4c8cf1a7e78b734c03628b684ec5de25df1f02ccea2e10fbd217430cb72", + "sha256:d4bee20f52b8c3c477dc780780654cafcfc0eb34d8d6960c13a34a444b431f09", + "sha256:e730529bd1f62034c50f70a2b05fadbf7d1402d39ff69c9dc63db066d0ef8a46", + "sha256:eb54ecfbf1abe6134073b7b35fd40442c4cd81bb9a5bee1a3038b8867b721bfb", + "sha256:f18ec70f5ee6a78b3bb4361e55f3a5ef34eb253f1e72fba76f29f0d680cd446f", + "sha256:f6be66f69198dcd04a79faa6052f756d35643496321858f06931c7b1ed9833ab", + "sha256:fc5c23a53fbd97b8aab4968c8548ce5cea4a54a26b4f8c1e6835df7adb8d0fe2" ], - "version": "==3.5.26" + "version": "==3.5.28" }, "requests": { "hashes": [ @@ -255,11 +254,10 @@ }, "beautifulsoup4": { "hashes": [ - "sha256:05668158c7b85b791c5abde53e50265e16f98ad601c402ba44d70f96c4159612", - "sha256:25288c9e176f354bf277c0a10aa96c782a6a18a17122dba2e8cec4a97e03343b", - "sha256:f040590be10520f2ea4c2ae8c3dae441c7cfff5308ec9d58a0ec0c1b8f81d469" + "sha256:5279c36b4b2ec2cb4298d723791467e3000e5384a43ea0cdf5d45207c7e97169", + "sha256:dcdef580e18a76d54002088602eba453eec38ebbcafafeaabd8cab12b6155d57" ], - "version": "==4.8.0" + "version": "==4.8.1" }, "certifi": { "hashes": [ @@ -299,10 +297,10 @@ }, "commonmark": { "hashes": [ - "sha256:14c3df31e8c9c463377e287b2a1eefaa6019ab97b22dad36e2f32be59d61d68d", - "sha256:867fc5db078ede373ab811e16b6789e9d033b15ccd7296f370ca52d1ee792ce0" + "sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60", + "sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9" ], - "version": "==0.9.0" + "version": "==0.9.1" }, "coverage": { "hashes": [ @@ -377,12 +375,6 @@ "index": "pypi", "version": "==0.15" }, - "future": { - "hashes": [ - "sha256:67045236dcfd6816dc439556d009594abf643e5eb48992e36beac09c2ca659b8" - ], - "version": "==0.17.1" - }, "idna": { "hashes": [ "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407", @@ -399,10 +391,10 @@ }, "jinja2": { "hashes": [ - "sha256:065c4f02ebe7f7cf559e49ee5a95fb800a9e4528727aec6f24402a5374c65013", - "sha256:14dd6caf1527abb21f08f86c784eac40853ba93edb79552aa1e4b8aef1b61c7b" + "sha256:74320bb91f31270f9551d46522e33af46a80c3d619f4a4bf42b3164d30b5911f", + "sha256:9fe95f19286cfefaa917656583d020be14e7859c6b0252588391e47db34527de" ], - "version": "==2.10.1" + "version": "==2.10.3" }, "jsonschema": { "hashes": [ @@ -461,11 +453,18 @@ ], "version": "==1.1.1" }, + "memory-profiler": { + "hashes": [ + "sha256:5fa47b274c929dd2cbcd9190afb62fec110701251d2ac2d301caaf545c81afc1" + ], + "index": "pypi", + "version": "==0.55.0" + }, "neobolt": { "hashes": [ - "sha256:fa9efe4a4defbdc63fc3f1e552d503727049586c59d8a3acf5188a2cf1a45dce" + "sha256:56b86b8b2c3facdd54589e60ecd22e0234d6f40645ab2e2cf87ef0cd79df20af" ], - "version": "==1.7.13" + "version": "==1.7.15" }, "neotime": { "hashes": [ @@ -491,42 +490,56 @@ }, "pillow": { "hashes": [ - "sha256:0804f77cb1e9b6dbd37601cee11283bba39a8d44b9ddb053400c58e0c0d7d9de", - "sha256:0ab7c5b5d04691bcbd570658667dd1e21ca311c62dcfd315ad2255b1cd37f64f", - "sha256:0b3e6cf3ea1f8cecd625f1420b931c83ce74f00c29a0ff1ce4385f99900ac7c4", - "sha256:365c06a45712cd723ec16fa4ceb32ce46ad201eb7bbf6d3c16b063c72b61a3ed", - "sha256:38301fbc0af865baa4752ddae1bb3cbb24b3d8f221bf2850aad96b243306fa03", - "sha256:3aef1af1a91798536bbab35d70d35750bd2884f0832c88aeb2499aa2d1ed4992", - "sha256:3fe0ab49537d9330c9bba7f16a5f8b02da615b5c809cdf7124f356a0f182eccd", - "sha256:45a619d5c1915957449264c81c008934452e3fd3604e36809212300b2a4dab68", - "sha256:49f90f147883a0c3778fd29d3eb169d56416f25758d0f66775db9184debc8010", - "sha256:571b5a758baf1cb6a04233fb23d6cf1ca60b31f9f641b1700bfaab1194020555", - "sha256:5ac381e8b1259925287ccc5a87d9cf6322a2dc88ae28a97fe3e196385288413f", - "sha256:6153db744a743c0c8c91b8e3b9d40e0b13a5d31dbf8a12748c6d9bfd3ddc01ad", - "sha256:6fd63afd14a16f5d6b408f623cc2142917a1f92855f0df997e09a49f0341be8a", - "sha256:70acbcaba2a638923c2d337e0edea210505708d7859b87c2bd81e8f9902ae826", - "sha256:70b1594d56ed32d56ed21a7fbb2a5c6fd7446cdb7b21e749c9791eac3a64d9e4", - "sha256:76638865c83b1bb33bcac2a61ce4d13c17dba2204969dedb9ab60ef62bede686", - "sha256:7b2ec162c87fc496aa568258ac88631a2ce0acfe681a9af40842fc55deaedc99", - "sha256:7cee2cef07c8d76894ebefc54e4bb707dfc7f258ad155bd61d87f6cd487a70ff", - "sha256:7d16d4498f8b374fc625c4037742fbdd7f9ac383fd50b06f4df00c81ef60e829", - "sha256:b50bc1780681b127e28f0075dfb81d6135c3a293e0c1d0211133c75e2179b6c0", - "sha256:bd0582f831ad5bcad6ca001deba4568573a4675437db17c4031939156ff339fa", - "sha256:cfd40d8a4b59f7567620410f966bb1f32dc555b2b19f82a91b147fac296f645c", - "sha256:e3ae410089de680e8f84c68b755b42bc42c0ceb8c03dbea88a5099747091d38e", - "sha256:e9046e559c299b395b39ac7dbf16005308821c2f24a63cae2ab173bd6aa11616", - "sha256:ef6be704ae2bc8ad0ebc5cb850ee9139493b0fc4e81abcc240fb392a63ebc808", - "sha256:f8dc19d92896558f9c4317ee365729ead9d7bbcf2052a9a19a3ef17abbb8ac5b" + "sha256:00fdeb23820f30e43bba78eb9abb00b7a937a655de7760b2e09101d63708b64e", + "sha256:01f948e8220c85eae1aa1a7f8edddcec193918f933fb07aaebe0bfbbcffefbf1", + "sha256:08abf39948d4b5017a137be58f1a52b7101700431f0777bec3d897c3949f74e6", + "sha256:099a61618b145ecb50c6f279666bbc398e189b8bc97544ae32b8fcb49ad6b830", + "sha256:2c1c61546e73de62747e65807d2cc4980c395d4c5600ecb1f47a650c6fa78c79", + "sha256:2ed9c4f694861642401f27dc3cb99772be67cd190e84845c749dae0a06c3bfae", + "sha256:338581b30b908e111be578f0297255f6b57a51358cd16fa0e6f664c9a1f88bff", + "sha256:38c7d48a21cd06fdeee93987147b9b1c55b73b4cfcbf83240568bfbd5adee447", + "sha256:43fd026f613c8e48a25eba1a92f4d2ad7f3903c95d8c33a11611a7717d2ab654", + "sha256:4548236844327a718ce3bb182ab32a16fa2050c61e334e959f554cac052fb0df", + "sha256:5090857876c58885cfa388dc649e5db30aae98a068c26f3fd0ac9d7d9a4d9572", + "sha256:5bbba34f97a26a93f5e8dec469ca4ddd712451418add43da946dbaed7f7a98d2", + "sha256:65a28969a025a0eb4594637b6103201dc4ed2a9508bdab56ac33e43e3081c404", + "sha256:892bb52b70bd5ea9dbbc3ac44f38e84f5a04e9d8b1bff48159d96cb795b81159", + "sha256:8a9becd5cbd5062f973bcd2e7bc79483af310222de112b6541f8af1f93a3cc42", + "sha256:972a7aaeb7c4a2795b52eef52ee991ef040b31009f36deca6207a986607b55f3", + "sha256:97b119c436bfa96a92ac2ca525f7025836d4d4e64b1c9f9eff8dbaf3ff1d86f3", + "sha256:9ba37698e242223f8053cc158f130aee046a96feacbeab65893dbe94f5530118", + "sha256:b1b0e1f626a0f079c0d3696db70132fb1f29aa87c66aecb6501a9b8be64ce9f7", + "sha256:c14c1224fd1a5be2733530d648a316974dbbb3c946913562c6005a76f21ca042", + "sha256:c79a8546c48ae6465189e54e3245a97ddf21161e33ff7eaa42787353417bb2b6", + "sha256:ceb76935ac4ebdf6d7bc845482a4450b284c6ccfb281e34da51d510658ab34d8", + "sha256:e22bffaad04b4d16e1c091baed7f2733fc1ebb91e0c602abf1b6834d17158b1f", + "sha256:ec883b8e44d877bda6f94a36313a1c6063f8b1997aa091628ae2f34c7f97c8d5", + "sha256:f1baa54d50ec031d1a9beb89974108f8f2c0706f49798f4777df879df0e1adb6", + "sha256:f53a5385932cda1e2c862d89460992911a89768c65d176ff8c50cddca4d29bed" ], - "version": "==6.1.0" + "version": "==6.2.0" }, "prompt-toolkit": { "hashes": [ - "sha256:11adf3389a996a6d45cc277580d0d53e8a5afd281d0c9ec71b28e6f121463780", - "sha256:2519ad1d8038fd5fc8e770362237ad0364d16a7650fb5724af6997ed5515e3c1", - "sha256:977c6583ae813a37dc1c2e1b715892461fcbdaa57f6fc62f33a528c4886c8f55" + "sha256:46642344ce457641f28fc9d1c9ca939b63dadf8df128b86f1b9860e59c73a5e4", + "sha256:e7f8af9e3d70f514373bf41aa51bc33af12a6db3f71461ea47fea985defb2c31", + "sha256:f15af68f66e664eaa559d4ac8a928111eebd5feda0c11738b5998045224829db" ], - "version": "==2.0.9" + "version": "==2.0.10" + }, + "psutil": { + "hashes": [ + "sha256:028a1ec3c6197eadd11e7b46e8cc2f0720dc18ac6d7aabdb8e8c0d6c9704f000", + "sha256:503e4b20fa9d3342bcf58191bbc20a4a5ef79ca7df8972e6197cc14c5513e73d", + "sha256:863a85c1c0a5103a12c05a35e59d336e1d665747e531256e061213e2e90f63f3", + "sha256:954f782608bfef9ae9f78e660e065bd8ffcfaea780f9f2c8a133bb7cb9e826d7", + "sha256:b6e08f965a305cd84c2d07409bc16fbef4417d67b70c53b299116c5b895e3f45", + "sha256:bc96d437dfbb8865fc8828cf363450001cb04056bbdcdd6fc152c436c8a74c61", + "sha256:cf49178021075d47c61c03c0229ac0c60d5e2830f8cab19e2d88e579b18cdb76", + "sha256:d5350cb66690915d60f8b233180f1e49938756fb2d501c93c44f8fb5b970cc63", + "sha256:eba238cf1989dfff7d483c029acb0ac4fcbfc15de295d682901f0e2497e6781a" + ], + "version": "==5.6.3" }, "py2neo": { "hashes": [ @@ -586,10 +599,10 @@ }, "pytz": { "hashes": [ - "sha256:26c0b32e437e54a18161324a2fca3c4b9846b74a8dccddd843113109e1116b32", - "sha256:c894d57500a4cd2d5c71114aaab77dbab5eabd9022308ce5ac9bb93a60a6f0c7" + "sha256:1c557d7d0e871de1f5ccd5833f60fb2550652da6be2693c1e02300743d21500d", + "sha256:b02c06db6cf09c12dd25137e563b31700d3b80fcc4ad23abb7a315f2789819be" ], - "version": "==2019.2" + "version": "==2019.3" }, "recommonmark": { "hashes": [ @@ -600,31 +613,31 @@ }, "reportlab": { "hashes": [ - "sha256:035346299a556c378a57cc163f2dfd945feebd5e45c844bbd02cae0711f780b5", - "sha256:05dafbe4cb0801fff0d0956b2d474e79d91d3b48923b6102a28cce0fcb2d8e53", - "sha256:0b28d4407caa6932539e8f9761e0430a96c6939713ee4b2f27b6c7af327c69e1", - "sha256:10cda714d7da684b2332d4d435cf90472fb2ea031ebbfb7f509a31d5898c06b3", - "sha256:152c321ca3caa564a5da1c33cb1937af983184eb7b3830f505770728a3f2f075", - "sha256:1c522ef6656abaf742c0995bf4fc19b7284732c7f3a00e5cd901d48187a9e8f8", - "sha256:24ee58e0905d4c0ec13557212de7c36ea92f87d688814191fab19d70294afb0e", - "sha256:43145d034423c97db1ac0db4f5e20ed6d16a1161a64e3da3f93a8a9b1eae78f1", - "sha256:43e433fd601259f4cfdc1c076ade9537c31bd116c4f5aa48d66713a609f8ee2c", - "sha256:5156e1e1f0b7fb8864bafc73ba02f8fa18ca46bf6702b34472f9033edf35cb3b", - "sha256:55d901dae80a30cd3b6a16accfbdf87f2fff6d379fd78ba6b87ae437391a5a2f", - "sha256:67fd4229ff0b643a0c2792ac5274ed7edaffc07052ad5dcd7d6f7a5d641a7446", - "sha256:739e81a32cd51be8d8225d316b1399d4b6ff6ee109963d3b5320b0718ce2aa0f", - "sha256:801aa0887135713ab107afd51e58dc2bb951fb3ffd773528b96924f90f82558f", - "sha256:835985ac466b3ac5e0751532f135aebc50c204837ebbc781d9a6313f4b76ab5a", - "sha256:87b3a4cedb40c08a7708d870cd7c405effb7a6a29cedf2dde380219d7a4d3cee", - "sha256:8c1abc9a4fead2d9255dfd1d58e5e1025c1816d2c38f6f0a7f50e42d51543c1d", - "sha256:97368b3f9622454fac3fe794664da5c09e4ec4ffa3fcbbf11a6de07e1ee35f41", - "sha256:9ca02a68b539969f9396ff485c574e6843a2433f6c165e8958a97d9366e1df0c", - "sha256:a43dba290c8c46e12f2cc142e4753e68e98ca088ca1dbbfce8822ca4436e7bf8", - "sha256:ac9414b875c6dc4fd46a01994612dece57a5be14b53c528f45ff85d82e332dfd", - "sha256:b58af0f12434835efa494f081c14bf5293764c78f58e6b32d941db5fff5b1692", - "sha256:b6e4a1b2f774f6c713bece60f2c06bb662985bdefffe0bc8582d3ef912985ba5" + "sha256:06b7c7436fa6d4844c7637161f3297c7a96240f35622ab2d219e4fd8387c0ab2", + "sha256:0a5acf67bd9812e38ed84be8994c07a8136b0a8f4c14a1c66c9c73a9567a9a44", + "sha256:1c8ca145d03e3c620866b06febb241b179197b58fb07454fbc8e9d6184cdcc93", + "sha256:2f8d785660ee316874c86abad345633ce8c652e88e03ae8a10f1fdadc72fd23d", + "sha256:4869d342352c92a812ce40555ef2a9cfbd722390d67fe61f1d6ec770e9ca41a3", + "sha256:493e0dcd9c085d46acf4fe3f00f941e562490a74b651409039a0dee2a0d76555", + "sha256:4e606e3ee9345e68cd205022d526250ad2a1164eea8f1e29d77d6ad08631b0ba", + "sha256:5bf91bae8995db91650fda658129c268515358b756fd16c0261a9dd641df1856", + "sha256:6df0730f8f715aa12333bd6d2a72eea3a989381451861186d9b5e71889454ac7", + "sha256:7195c6ea096d10c91cc470f9f0ced3ad74470d9c0fd97923b5e764597dd13671", + "sha256:7431c979e2b498e8e20abf458f360a451717d76c3c1bd49d1fc5697d3504f8e5", + "sha256:7f7f70a8d4b573d1ff65a81415b4b6ed9545630f381dff4a69307640e09d381d", + "sha256:9945433667a46f054d1125b4ca86fe9ee31feb254728b38242e9a6008c135efe", + "sha256:b1cdbfc1fd54ac947b9f0114e00ab94e945db679f1e03357a3c00f3a85e73eea", + "sha256:bf149847a2fd8f24b788a8abbf97a2b9a73edc5b1bd719384b786eb84bcad15e", + "sha256:ce514bfce2bf3e302f52aba9929fe3ac7d918cfea2f5d3e30bf9dac9658bf094", + "sha256:d243d4c8cf1a7e78b734c03628b684ec5de25df1f02ccea2e10fbd217430cb72", + "sha256:d4bee20f52b8c3c477dc780780654cafcfc0eb34d8d6960c13a34a444b431f09", + "sha256:e730529bd1f62034c50f70a2b05fadbf7d1402d39ff69c9dc63db066d0ef8a46", + "sha256:eb54ecfbf1abe6134073b7b35fd40442c4cd81bb9a5bee1a3038b8867b721bfb", + "sha256:f18ec70f5ee6a78b3bb4361e55f3a5ef34eb253f1e72fba76f29f0d680cd446f", + "sha256:f6be66f69198dcd04a79faa6052f756d35643496321858f06931c7b1ed9833ab", + "sha256:fc5c23a53fbd97b8aab4968c8548ce5cea4a54a26b4f8c1e6835df7adb8d0fe2" ], - "version": "==3.5.26" + "version": "==3.5.28" }, "requests": { "hashes": [ @@ -650,9 +663,10 @@ }, "snowballstemmer": { "hashes": [ - "sha256:713e53b79cbcf97bc5245a06080a33d54a77e7cce2f789c835a143bcdb5c033e" + "sha256:209f257d7533fdb3cb73bdbd24f436239ca3b2fa67d56f6ff88e86be08cc5ef0", + "sha256:df3bac3df4c2c01363f3dd2cfa78cce2840a79b9f1c2d2de9ce8d31683992f52" ], - "version": "==1.9.1" + "version": "==2.0.0" }, "soupsieve": { "hashes": [ diff --git a/examples/generate_file_objects.py b/examples/generate_file_objects.py index c3eda36..c187b9d 100755 --- a/examples/generate_file_objects.py +++ b/examples/generate_file_objects.py @@ -5,7 +5,7 @@ import argparse import json try: - from pymisp import MISPEncode, AbstractMISP + from pymisp import pymisp_json_default, AbstractMISP from pymisp.tools import make_binary_objects except ImportError: pass @@ -51,7 +51,8 @@ def make_objects(path): to_return['objects'].append(fo) if fo.ObjectReference: to_return['references'] += fo.ObjectReference - return json.dumps(to_return, cls=MISPEncode) + return json.dumps(to_return, default=pymisp_json_default) + if __name__ == '__main__': parser = argparse.ArgumentParser(description='Extract indicators out of binaries and returns MISP objects.') diff --git a/pymisp/__init__.py b/pymisp/__init__.py index f302be8..f134cce 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -31,7 +31,7 @@ try: warning_2020() from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey, InvalidMISPObject, UnknownMISPObjectTemplate, PyMISPInvalidFormat, MISPServerError, PyMISPNotImplementedYet, PyMISPUnexpectedResponse, PyMISPEmptyResponse # noqa from .api import PyMISP # noqa - from .abstract import AbstractMISP, MISPEncode, MISPTag, Distribution, ThreatLevel, Analysis # noqa + from .abstract import AbstractMISP, MISPEncode, pymisp_json_default, MISPTag, Distribution, ThreatLevel, Analysis # noqa from .mispevent import MISPEvent, MISPAttribute, MISPObjectReference, MISPObjectAttribute, MISPObject, MISPUser, MISPOrganisation, MISPSighting, MISPLog, MISPShadowAttribute, MISPWarninglist, MISPTaxonomy, MISPNoticelist, MISPObjectTemplate, MISPSharingGroup, MISPRole, MISPServer, MISPFeed, MISPEventDelegation # noqa from .tools import AbstractMISPObjectGenerator # noqa from .tools import Neo4j # noqa diff --git a/pymisp/abstract.py b/pymisp/abstract.py index c85ef3b..0e2d486 100644 --- a/pymisp/abstract.py +++ b/pymisp/abstract.py @@ -3,24 +3,40 @@ import sys import datetime -import json + +from deprecated import deprecated from json import JSONEncoder + +try: + from rapidjson import load + from rapidjson import loads + from rapidjson import dumps + import rapidjson + HAS_RAPIDJSON = True +except ImportError: + from json import load + from json import loads + from json import dumps + import json + HAS_RAPIDJSON = False + import logging from enum import Enum from .exceptions import PyMISPInvalidFormat -# Try to import MutableMapping the python 3.3+ way -try: - from collections.abc import MutableMapping -except Exception: - pass - logger = logging.getLogger('pymisp') if sys.version_info < (3, 0): from collections import MutableMapping + import os + from cachetools import cached, LRUCache + + resources_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data') + misp_objects_path = os.path.join(resources_path, 'misp-objects', 'objects') + with open(os.path.join(resources_path, 'describeTypes.json'), 'r') as f: + describe_types = load(f)['result'] # This is required because Python 2 is a pain. from datetime import tzinfo, timedelta @@ -37,6 +53,36 @@ if sys.version_info < (3, 0): def dst(self, dt): return timedelta(0) + class MISPFileCache(object): + # cache up to 150 JSON structures in class attribute + + @staticmethod + @cached(cache=LRUCache(maxsize=150)) + def _load_json(path): + with open(path, 'r') as f: + data = load(f) + return data + +else: + from collections.abc import MutableMapping + from functools import lru_cache + from pathlib import Path + + resources_path = Path(__file__).parent / 'data' + misp_objects_path = resources_path / 'misp-objects' / 'objects' + with (resources_path / 'describeTypes.json').open('r') as f: + describe_types = load(f)['result'] + + class MISPFileCache(object): + # cache up to 150 JSON structures in class attribute + + @staticmethod + @lru_cache(maxsize=150) + def _load_json(path): + with path.open('r') as f: + data = load(f) + return data + class Distribution(Enum): your_organisation_only = 0 @@ -68,8 +114,8 @@ def _int_to_str(d): return d +@deprecated(reason=" Use method default=pymisp_json_default instead of cls=MISPEncode", version='2.4.117', action='default') class MISPEncode(JSONEncoder): - def default(self, obj): if isinstance(obj, AbstractMISP): return obj.jsonable() @@ -80,13 +126,37 @@ class MISPEncode(JSONEncoder): return JSONEncoder.default(self, obj) -class AbstractMISP(MutableMapping): +if HAS_RAPIDJSON: + def pymisp_json_default(obj): + if isinstance(obj, AbstractMISP): + return obj.jsonable() + elif isinstance(obj, (datetime.datetime, datetime.date)): + return obj.isoformat() + elif isinstance(obj, Enum): + return obj.value + return rapidjson.default(obj) +else: + def pymisp_json_default(obj): + if isinstance(obj, AbstractMISP): + return obj.jsonable() + elif isinstance(obj, (datetime.datetime, datetime.date)): + return obj.isoformat() + elif isinstance(obj, Enum): + return obj.value + return json.default(obj) + + +class AbstractMISP(MutableMapping, MISPFileCache): + __resources_path = resources_path + __misp_objects_path = misp_objects_path + __describe_types = describe_types def __init__(self, **kwargs): """Abstract class for all the MISP objects""" super(AbstractMISP, self).__init__() self.__edited = True # As we create a new object, we assume it is edited self.__not_jsonable = [] + self.__self_defined_describe_types = None if kwargs.get('force_timestamps') is not None: # Ignore the edited objects and keep the timestamps. @@ -103,16 +173,28 @@ class AbstractMISP(MutableMapping): setattr(AbstractMISP, 'tags', property(AbstractMISP.__get_tags, AbstractMISP.__set_tags)) @property - def properties(self): - """All the class public properties that will be dumped in the dictionary, and the JSON export. - Note: all the properties starting with a `_` (private), or listed in __not_jsonable will be skipped. - """ - to_return = [] - for prop, value in vars(self).items(): - if prop.startswith('_') or prop in self.__not_jsonable: - continue - to_return.append(prop) - return to_return + def describe_types(self): + if self.__self_defined_describe_types: + return self.__self_defined_describe_types + return self.__describe_types + + @describe_types.setter + def describe_types(self, describe_types): + self.__self_defined_describe_types = describe_types + + @property + def resources_path(self): + return self.__resources_path + + @property + def misp_objects_path(self): + return self.__misp_objects_path + + @misp_objects_path.setter + def misp_objects_path(self, misp_objects_path): + if sys.version_info >= (3, 0) and isinstance(misp_objects_path, str): + misp_objects_path = Path(misp_objects_path) + self.__misp_objects_path = misp_objects_path def from_dict(self, **kwargs): """Loading all the parameters as class properties, if they aren't `None`. @@ -137,21 +219,21 @@ class AbstractMISP(MutableMapping): def from_json(self, json_string): """Load a JSON string""" - self.from_dict(**json.loads(json_string)) + self.from_dict(**loads(json_string)) def to_dict(self): - """Dump the lass to a dictionary. + """Dump the class to a dictionary. This method automatically removes the timestamp recursively in every object that has been edited is order to let MISP update the event accordingly.""" + is_edited = self.edited to_return = {} - for attribute in self.properties: - val = getattr(self, attribute, None) + for attribute, val in self.items(): if val is None: continue elif isinstance(val, list) and len(val) == 0: continue if attribute == 'timestamp': - if not self.__force_timestamps and self.edited: + if not self.__force_timestamps and is_edited: # In order to be accepted by MISP, the timestamp of an object # needs to be either newer, or None. # If the current object is marked as edited, the easiest is to @@ -167,13 +249,15 @@ class AbstractMISP(MutableMapping): """This method is used by the JSON encoder""" return self.to_dict() - def to_json(self): + def to_json(self, sort_keys=False, indent=None): """Dump recursively any class of type MISPAbstract to a json string""" - return json.dumps(self, cls=MISPEncode, sort_keys=True, indent=2) + return dumps(self, default=pymisp_json_default, sort_keys=sort_keys, indent=indent) def __getitem__(self, key): try: - return getattr(self, key) + if key[0] != '_' and key not in self.__not_jsonable: + return self.__dict__[key] + raise KeyError except AttributeError: # Expected by pop and other dict-related methods raise KeyError @@ -185,10 +269,10 @@ class AbstractMISP(MutableMapping): delattr(self, key) def __iter__(self): - return iter(self.to_dict()) + return iter({k: v for k, v in self.__dict__.items() if not (k[0] == '_' or k in self.__not_jsonable)}) def __len__(self): - return len(self.to_dict()) + return len([k for k in self.__dict__.keys() if not (k[0] == '_' or k in self.__not_jsonable)]) @property def edited(self): @@ -196,15 +280,14 @@ class AbstractMISP(MutableMapping): to the parent objects""" if self.__edited: return self.__edited - for p in self.properties: - if self.__edited: - break - val = getattr(self, p) + for p, val in self.items(): if isinstance(val, AbstractMISP) and val.edited: self.__edited = True + break elif isinstance(val, list) and all(isinstance(a, AbstractMISP) for a in val): if any(a.edited for a in val): self.__edited = True + break return self.__edited @edited.setter @@ -216,7 +299,9 @@ class AbstractMISP(MutableMapping): raise Exception('edited can only be True or False') def __setattr__(self, name, value): - if name in self.properties: + if name[0] != '_' and not self.__edited and name in self.keys(): + # The private members don't matter + # If we already have a key with that name, we're modifying it. self.__edited = True super(AbstractMISP, self).__setattr__(name, value) diff --git a/pymisp/api.py b/pymisp/api.py index b15424b..a5c0670 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -19,7 +19,7 @@ from deprecated import deprecated from . import __version__, warning_2020 from .exceptions import PyMISPError, SearchError, NoURL, NoKey, PyMISPEmptyResponse from .mispevent import MISPEvent, MISPAttribute, MISPUser, MISPOrganisation, MISPSighting, MISPFeed, MISPObject, MISPSharingGroup -from .abstract import AbstractMISP, MISPEncode +from .abstract import AbstractMISP, pymisp_json_default, MISPFileCache, describe_types logger = logging.getLogger('pymisp') @@ -37,11 +37,6 @@ try: except ImportError: HAVE_REQUESTS = False -if (3, 0) <= sys.version_info < (3, 6): - OLD_PY3 = True -else: - OLD_PY3 = False - try: from requests_futures.sessions import FuturesSession ASYNC_OK = True @@ -58,7 +53,7 @@ Response (if any): {}''' -class PyMISP(object): # pragma: no cover +class PyMISP(MISPFileCache): # pragma: no cover """Python API for MISP :param url: URL of the MISP instance you want to connect to @@ -140,24 +135,19 @@ class PyMISP(object): # pragma: no cover @deprecated(reason="Use ExpandedPyMISP.describe_types_local", version='2.4.110', action='default') def get_local_describe_types(self): - with open(os.path.join(self.resources_path, 'describeTypes.json'), 'rb') as f: - if OLD_PY3: - describe_types = json.loads(f.read().decode()) - else: - describe_types = json.load(f) - return describe_types['result'] + return describe_types @deprecated(reason="Use ExpandedPyMISP.describe_types_remote", version='2.4.110', action='default') def get_live_describe_types(self): response = self._prepare_request('GET', urljoin(self.root_url, 'attributes/describeTypes.json')) - describe_types = self._check_response(response) - if describe_types.get('error'): - for e in describe_types.get('error'): + remote_describe_types = self._check_response(response) + if remote_describe_types.get('error'): + for e in remote_describe_types.get('error'): raise PyMISPError('Failed: {}'.format(e)) - describe_types = describe_types['result'] - if not describe_types.get('sane_defaults'): + remote_describe_types = describe_types + if not remote_describe_types.get('sane_defaults'): raise PyMISPError('The MISP server your are trying to reach is outdated (<2.4.52). Please use PyMISP v2.4.51.1 (pip install -I PyMISP==v2.4.51.1) and/or contact your administrator.') - return describe_types + return remote_describe_types def _prepare_request(self, request_type, url, data=None, background_callback=None, output_type='json'): @@ -172,7 +162,7 @@ class PyMISP(object): # pragma: no cover if isinstance(data, dict): # Remove None values. data = {k: v for k, v in data.items() if v is not None} - data = json.dumps(data, cls=MISPEncode) + data = json.dumps(data, default=pymisp_json_default) req = requests.Request(request_type, url, data=data) if self.asynch and background_callback is not None: local_session = FuturesSession @@ -614,7 +604,7 @@ class PyMISP(object): # pragma: no cover else: data = attributes.to_json() # _prepare_request(...) returns a requests.Response Object - resp = self._prepare_request('POST', url, json.dumps(data, cls=MISPEncode)) + resp = self._prepare_request('POST', url, json.dumps(data, default=pymisp_json_default)) try: responses.append(resp.json()) except Exception: @@ -1068,7 +1058,7 @@ class PyMISP(object): # pragma: no cover url = urljoin(self.root_url, 'shadow_attributes/{}/{}'.format(path, id)) if path in ['add', 'edit']: query = {'request': {'ShadowAttribute': attribute}} - response = self._prepare_request('POST', url, json.dumps(query, cls=MISPEncode)) + response = self._prepare_request('POST', url, json.dumps(query, default=pymisp_json_default)) elif path == 'view': response = self._prepare_request('GET', url) else: # accept or discard diff --git a/pymisp/aping.py b/pymisp/aping.py index a685936..152612a 100644 --- a/pymisp/aping.py +++ b/pymisp/aping.py @@ -19,7 +19,7 @@ from . import __version__ from .exceptions import MISPServerError, PyMISPUnexpectedResponse, PyMISPNotImplementedYet, PyMISPError, NoURL, NoKey from .api import everything_broken, PyMISP from .mispevent import MISPEvent, MISPAttribute, MISPSighting, MISPLog, MISPObject, MISPUser, MISPOrganisation, MISPShadowAttribute, MISPWarninglist, MISPTaxonomy, MISPGalaxy, MISPNoticelist, MISPObjectReference, MISPObjectTemplate, MISPSharingGroup, MISPRole, MISPServer, MISPFeed, MISPEventDelegation, MISPCommunity -from .abstract import MISPEncode, MISPTag, AbstractMISP +from .abstract import pymisp_json_default, MISPTag, AbstractMISP, describe_types SearchType = TypeVar('SearchType', str, int) # str: string to search / list: values to search (OR) / dict: {'OR': [list], 'NOT': [list], 'AND': [list]} @@ -106,16 +106,14 @@ class ExpandedPyMISP(PyMISP): @property def describe_types_local(self): '''Returns the content of describe types from the package''' - with (self.resources_path / 'describeTypes.json').open() as f: - describe_types = json.load(f) - return describe_types['result'] + return describe_types @property def describe_types_remote(self): '''Returns the content of describe types from the remote instance''' response = self._prepare_request('GET', 'attributes/describeTypes.json') - describe_types = self._check_response(response, expect_json=True) - return describe_types['result'] + remote_describe_types = self._check_response(response, expect_json=True) + return remote_describe_types['result'] @property def recommended_pymisp_version(self): @@ -2111,7 +2109,7 @@ class ExpandedPyMISP(PyMISP): if isinstance(data, dict): # Else, we can directly json encode. # Remove None values. data = {k: v for k, v in data.items() if v is not None} - data = json.dumps(data, cls=MISPEncode) + data = json.dumps(data, default=pymisp_json_default) if kw_params: # CakePHP params in URL diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index d5ef5ab..e3a10e8 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -17,6 +17,7 @@ from deprecated import deprecated from .abstract import AbstractMISP from .exceptions import UnknownMISPObjectTemplate, InvalidMISPObject, PyMISPError, NewEventError, NewAttributeError + logger = logging.getLogger('pymisp') if sys.version_info < (3, 0): @@ -109,18 +110,11 @@ class MISPAttribute(AbstractMISP): :strict: If false, fallback to sane defaults for the attribute type if the ones passed by the user are incorrect """ super(MISPAttribute, self).__init__() - if not describe_types: - ressources_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data') - with open(os.path.join(ressources_path, 'describeTypes.json'), 'rb') as f: - if OLD_PY3: - t = json.loads(f.read().decode()) - else: - t = json.load(f) - describe_types = t['result'] - self.__categories = describe_types['categories'] - self._types = describe_types['types'] - self.__category_type_mapping = describe_types['category_type_mappings'] - self.__sane_default = describe_types['sane_defaults'] + if describe_types: + self.describe_types = describe_types + self.__categories = self.describe_types['categories'] + self.__category_type_mapping = self.describe_types['category_type_mappings'] + self.__sane_default = self.describe_types['sane_defaults'] self.__strict = strict self._data = None self.uuid = str(uuid.uuid4()) @@ -130,7 +124,7 @@ class MISPAttribute(AbstractMISP): @property def known_types(self): """Returns a list of all the known MISP attributes types""" - return self._types + return self.describe_types['types'] @property def malware_binary(self): @@ -204,8 +198,8 @@ class MISPAttribute(AbstractMISP): return misp_sighting def from_dict(self, **kwargs): - if kwargs.get('Attribute'): - kwargs = kwargs.get('Attribute') + if 'Attribute' in kwargs: + kwargs = kwargs['Attribute'] if kwargs.get('type') and kwargs.get('category'): if kwargs['type'] not in self.__category_type_mapping[kwargs['category']]: if self.__strict: @@ -218,7 +212,7 @@ class MISPAttribute(AbstractMISP): if self.type is None: raise NewAttributeError('The type of the attribute is required.') if self.type not in self.known_types: - raise NewAttributeError('{} is invalid, type has to be in {}'.format(self.type, (', '.join(self._types)))) + raise NewAttributeError('{} is invalid, type has to be in {}'.format(self.type, (', '.join(self.known_types)))) type_defaults = self.__sane_default[self.type] @@ -278,14 +272,11 @@ class MISPAttribute(AbstractMISP): raise NewAttributeError('If the distribution is set to sharing group, a sharing group ID is required (cannot be {}).'.format(self.sharing_group_id)) if kwargs.get('Tag'): - for tag in kwargs.pop('Tag'): - self.add_tag(tag) + [self.add_tag(tag) for tag in kwargs.pop('Tag')] if kwargs.get('Sighting'): - for sighting in kwargs.pop('Sighting'): - self.add_sighting(sighting) + [self.add_sighting(sighting) for sighting in kwargs.pop('Sighting')] if kwargs.get('ShadowAttribute'): - for s_attr in kwargs.pop('ShadowAttribute'): - self.add_shadow_attribute(s_attr) + [self.add_shadow_attribute(s_attr) for s_attr in kwargs.pop('ShadowAttribute')] # If the user wants to disable correlation, let them. Defaults to False. self.disable_correlation = kwargs.pop("disable_correlation", False) @@ -425,31 +416,18 @@ class MISPEvent(AbstractMISP): def __init__(self, describe_types=None, strict_validation=False, **kwargs): super(MISPEvent, self).__init__(**kwargs) - ressources_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data') if strict_validation: - with open(os.path.join(ressources_path, 'schema.json'), 'rb') as f: - if OLD_PY3: - self.__json_schema = json.loads(f.read().decode()) - else: - self.__json_schema = json.load(f) + schema_file = 'schema.json' else: - with open(os.path.join(ressources_path, 'schema-lax.json'), 'rb') as f: - if OLD_PY3: - self.__json_schema = json.loads(f.read().decode()) - else: - self.__json_schema = json.load(f) + schema_file = 'schema-lax.json' + if sys.version_info >= (3, 6): + self.__json_schema = self._load_json(self.resources_path / schema_file) + else: + self.__json_schema = self._load_json(os.path.join(self.resources_path, schema_file)) if describe_types: # This variable is used in add_attribute in order to avoid duplicating the structure - self._describe_types = describe_types - else: - with open(os.path.join(ressources_path, 'describeTypes.json'), 'rb') as f: - if OLD_PY3: - t = json.loads(f.read().decode()) - else: - t = json.load(f) - self._describe_types = t['result'] + self.describe_types = describe_types - self._types = self._describe_types['types'] self.Attribute = [] self.Object = [] self.RelatedEvent = [] @@ -457,7 +435,7 @@ class MISPEvent(AbstractMISP): @property def known_types(self): - return self._types + return self.describe_types['types'] @property def org(self): @@ -554,8 +532,8 @@ class MISPEvent(AbstractMISP): raise NewEventError('Invalid format for the date: {} - {}'.format(date, type(date))) def from_dict(self, **kwargs): - if kwargs.get('Event'): - kwargs = kwargs.get('Event') + if 'Event' in kwargs: + kwargs = kwargs['Event'] # Required value self.info = kwargs.pop('info', None) if self.info is None: @@ -587,8 +565,7 @@ class MISPEvent(AbstractMISP): if kwargs.get('date'): self.set_date(kwargs.pop('date')) if kwargs.get('Attribute'): - for a in kwargs.pop('Attribute'): - self.add_attribute(**a) + [self.add_attribute(**a) for a in kwargs.pop('Attribute')] # All other keys if kwargs.get('id'): @@ -615,11 +592,9 @@ class MISPEvent(AbstractMISP): sub_event.load(rel_event) self.RelatedEvent.append({'Event': sub_event}) if kwargs.get('Tag'): - for tag in kwargs.pop('Tag'): - self.add_tag(tag) + [self.add_tag(tag) for tag in kwargs.pop('Tag')] if kwargs.get('Object'): - for obj in kwargs.pop('Object'): - self.add_object(obj) + [self.add_object(obj) for obj in kwargs.pop('Object')] if kwargs.get('Org'): self.Org = MISPOrganisation() self.Org.from_dict(**kwargs.pop('Org')) @@ -720,7 +695,7 @@ class MISPEvent(AbstractMISP): if isinstance(value, list): attr_list = [self.add_attribute(type=type, value=a, **kwargs) for a in value] else: - attribute = MISPAttribute(describe_types=self._describe_types) + attribute = MISPAttribute(describe_types=self.describe_types) attribute.from_dict(type=type, value=value, **kwargs) self.attributes.append(attribute) self.edited = True @@ -879,8 +854,8 @@ class MISPObjectReference(AbstractMISP): super(MISPObjectReference, self).__init__() def from_dict(self, **kwargs): - if kwargs.get('ObjectReference'): - kwargs = kwargs.get('ObjectReference') + if 'ObjectReference' in kwargs: + kwargs = kwargs['ObjectReference'] super(MISPObjectReference, self).from_dict(**kwargs) def __repr__(self): @@ -895,8 +870,8 @@ class MISPObjectTemplate(AbstractMISP): super(MISPObjectTemplate, self).__init__() def from_dict(self, **kwargs): - if kwargs.get('ObjectTemplate'): - kwargs = kwargs.get('ObjectTemplate') + if 'ObjectTemplate' in kwargs: + kwargs = kwargs['ObjectTemplate'] super(MISPObjectTemplate, self).from_dict(**kwargs) @@ -906,8 +881,8 @@ class MISPUser(AbstractMISP): super(MISPUser, self).__init__() def from_dict(self, **kwargs): - if kwargs.get('User'): - kwargs = kwargs.get('User') + if 'User' in kwargs: + kwargs = kwargs['User'] super(MISPUser, self).from_dict(**kwargs) def __repr__(self): @@ -922,8 +897,8 @@ class MISPOrganisation(AbstractMISP): super(MISPOrganisation, self).__init__() def from_dict(self, **kwargs): - if kwargs.get('Organisation'): - kwargs = kwargs.get('Organisation') + if 'Organisation' in kwargs: + kwargs = kwargs['Organisation'] super(MISPOrganisation, self).from_dict(**kwargs) @@ -933,8 +908,8 @@ class MISPFeed(AbstractMISP): super(MISPFeed, self).__init__() def from_dict(self, **kwargs): - if kwargs.get('Feed'): - kwargs = kwargs.get('Feed') + if 'Feed' in kwargs: + kwargs = kwargs['Feed'] super(MISPFeed, self).from_dict(**kwargs) @@ -944,8 +919,8 @@ class MISPWarninglist(AbstractMISP): super(MISPWarninglist, self).__init__() def from_dict(self, **kwargs): - if kwargs.get('Warninglist'): - kwargs = kwargs.get('Warninglist') + if 'Warninglist' in kwargs: + kwargs = kwargs['Warninglist'] super(MISPWarninglist, self).from_dict(**kwargs) @@ -955,8 +930,8 @@ class MISPTaxonomy(AbstractMISP): super(MISPTaxonomy, self).__init__() def from_dict(self, **kwargs): - if kwargs.get('Taxonomy'): - kwargs = kwargs.get('Taxonomy') + if 'Taxonomy' in kwargs: + kwargs = kwargs['Taxonomy'] super(MISPTaxonomy, self).from_dict(**kwargs) @@ -966,8 +941,8 @@ class MISPGalaxy(AbstractMISP): super(MISPGalaxy, self).__init__() def from_dict(self, **kwargs): - if kwargs.get('Galaxy'): - kwargs = kwargs.get('Galaxy') + if 'Galaxy' in kwargs: + kwargs = kwargs['Galaxy'] super(MISPGalaxy, self).from_dict(**kwargs) @@ -977,8 +952,8 @@ class MISPNoticelist(AbstractMISP): super(MISPNoticelist, self).__init__() def from_dict(self, **kwargs): - if kwargs.get('Noticelist'): - kwargs = kwargs.get('Noticelist') + if 'Noticelist' in kwargs: + kwargs = kwargs['Noticelist'] super(MISPNoticelist, self).from_dict(**kwargs) @@ -988,8 +963,8 @@ class MISPRole(AbstractMISP): super(MISPRole, self).__init__() def from_dict(self, **kwargs): - if kwargs.get('Role'): - kwargs = kwargs.get('Role') + if 'Role' in kwargs: + kwargs = kwargs['Role'] super(MISPRole, self).from_dict(**kwargs) @@ -999,8 +974,8 @@ class MISPServer(AbstractMISP): super(MISPServer, self).__init__() def from_dict(self, **kwargs): - if kwargs.get('Server'): - kwargs = kwargs.get('Server') + if 'Server' in kwargs: + kwargs = kwargs['Server'] super(MISPServer, self).from_dict(**kwargs) @@ -1010,8 +985,8 @@ class MISPSharingGroup(AbstractMISP): super(MISPSharingGroup, self).__init__() def from_dict(self, **kwargs): - if kwargs.get('SharingGroup'): - kwargs = kwargs.get('SharingGroup') + if 'SharingGroup' in kwargs: + kwargs = kwargs['SharingGroup'] super(MISPSharingGroup, self).from_dict(**kwargs) @@ -1021,8 +996,8 @@ class MISPLog(AbstractMISP): super(MISPLog, self).__init__() def from_dict(self, **kwargs): - if kwargs.get('Log'): - kwargs = kwargs.get('Log') + if 'Log' in kwargs: + kwargs = kwargs['Log'] super(MISPLog, self).from_dict(**kwargs) def __repr__(self): @@ -1035,8 +1010,8 @@ class MISPEventDelegation(AbstractMISP): super(MISPEventDelegation, self).__init__() def from_dict(self, **kwargs): - if kwargs.get('EventDelegation'): - kwargs = kwargs.get('EventDelegation') + if 'EventDelegation' in kwargs: + kwargs = kwargs['EventDelegation'] super(MISPEventDelegation, self).from_dict(**kwargs) def __repr__(self): @@ -1058,8 +1033,8 @@ class MISPSighting(AbstractMISP): :type: Type of the sighting :timestamp: Timestamp associated to the sighting """ - if kwargs.get('Sighting'): - kwargs = kwargs.get('Sighting') + if 'Sighting' in kwargs: + kwargs = kwargs['Sighting'] super(MISPSighting, self).from_dict(**kwargs) def __repr__(self): @@ -1081,6 +1056,8 @@ class MISPObjectAttribute(MISPAttribute): def from_dict(self, object_relation, value, **kwargs): self.object_relation = object_relation self.value = value + if 'Attribute' in kwargs: + kwargs = kwargs['Attribute'] # Initialize the new MISPAttribute # Get the misp attribute type from the definition self.type = kwargs.pop('type', None) @@ -1098,6 +1075,8 @@ class MISPObjectAttribute(MISPAttribute): if not self.type: raise NewAttributeError("The type of the attribute is required. Is the object template missing?") super(MISPObjectAttribute, self).from_dict(**dict(self, **kwargs)) + # FIXME New syntax python3 only, keep for later. + # super(MISPObjectAttribute, self).from_dict(**{**self, **kwargs}) def __repr__(self): if hasattr(self, 'value'): @@ -1111,8 +1090,8 @@ class MISPShadowAttribute(AbstractMISP): super(MISPShadowAttribute, self).__init__() def from_dict(self, **kwargs): - if kwargs.get('ShadowAttribute'): - kwargs = kwargs.get('ShadowAttribute') + if 'ShadowAttribute' in kwargs: + kwargs = kwargs['ShadowAttribute'] super(MISPShadowAttribute, self).from_dict(**kwargs) def __repr__(self): @@ -1127,8 +1106,8 @@ class MISPCommunity(AbstractMISP): super(MISPCommunity, self).__init__() def from_dict(self, **kwargs): - if kwargs.get('Community'): - kwargs = kwargs.get('Community') + if 'Community' in kwargs: + kwargs = kwargs['Community'] super(MISPCommunity, self).from_dict(**kwargs) def __repr__(self): @@ -1155,20 +1134,7 @@ class MISPObject(AbstractMISP): self.name = name self._known_template = False - if kwargs.get('misp_objects_path_custom'): - # If misp_objects_path_custom is given, and an object with the given name exists, use that. - self._known_template = self._load_template_path(os.path.join(kwargs.get('misp_objects_path_custom'), self.name, 'definition.json')) - - if not self._known_template: - # Check if the object is known in the default templates bundled in with PyMISP - misp_objects_path = os.path.join(os.path.abspath(os.path.dirname(sys.modules['pymisp'].__file__)), 'data', 'misp-objects', 'objects') - self._known_template = self._load_template_path(os.path.join(misp_objects_path, self.name, 'definition.json')) - - if not self._known_template and self._strict: - raise UnknownMISPObjectTemplate('{} is unknown in the MISP object directory.'.format(self.name)) - else: - # Then we have no meta-category, template_uuid, description and template_version - pass + self._set_template(kwargs.get('misp_objects_path_custom')) self.uuid = str(uuid.uuid4()) self.__fast_attribute_access = defaultdict(list) # Hashtable object_relation: [attributes] @@ -1206,11 +1172,7 @@ class MISPObject(AbstractMISP): def _load_template_path(self, template_path): if not os.path.exists(template_path): return False - with open(template_path, 'rb') as f: - if OLD_PY3: - self._definition = json.loads(f.read().decode()) - else: - self._definition = json.load(f) + self._definition = self._load_json(template_path) setattr(self, 'meta-category', self._definition['meta-category']) self.template_uuid = self._definition['uuid'] self.description = self._definition['description'] @@ -1220,11 +1182,24 @@ class MISPObject(AbstractMISP): def force_misp_objects_path_custom(self, misp_objects_path_custom, object_name=None): if object_name: self.name = object_name - template_path = os.path.join(misp_objects_path_custom, self.name, 'definition.json') + self._set_template(misp_objects_path_custom) - self._known_template = self._load_template_path(template_path) - if not self._known_template: - raise UnknownMISPObjectTemplate('{} is unknown in the MISP object directory ({}).'.format(self.name, template_path)) + def _set_template(self, misp_objects_path_custom=None): + if misp_objects_path_custom: + # If misp_objects_path_custom is given, and an object with the given name exists, use that. + self.misp_objects_path = misp_objects_path_custom + + # Try to get the template + if sys.version_info >= (3, 6): + self._known_template = self._load_template_path(self.misp_objects_path / self.name / 'definition.json') + else: + self._known_template = self._load_template_path(os.path.join(self.misp_objects_path, self.name, 'definition.json')) + + if not self._known_template and self._strict: + raise UnknownMISPObjectTemplate('{} is unknown in the MISP object directory.'.format(self.name)) + else: + # Then we have no meta-category, template_uuid, description and template_version + pass @property def disable_validation(self): @@ -1254,8 +1229,8 @@ class MISPObject(AbstractMISP): raise PyMISPError('All the attributes have to be of type MISPObjectReference.') def from_dict(self, **kwargs): - if kwargs.get('Object'): - kwargs = kwargs.get('Object') + if 'Object' in kwargs: + kwargs = kwargs['Object'] if self._known_template: if kwargs.get('template_uuid') and kwargs['template_uuid'] != self.template_uuid: if self._strict: @@ -1283,11 +1258,9 @@ class MISPObject(AbstractMISP): else: self.timestamp = datetime.datetime.fromtimestamp(int(ts), UTC()) if kwargs.get('Attribute'): - for a in kwargs.pop('Attribute'): - self.add_attribute(**a) + [self.add_attribute(**a) for a in kwargs.pop('Attribute')] if kwargs.get('ObjectReference'): - for r in kwargs.pop('ObjectReference'): - self.add_reference(**r) + [self.add_reference(**r) for r in kwargs.pop('ObjectReference')] # Not supported yet - https://github.com/MISP/PyMISP/issues/168 # if kwargs.get('Tag'): @@ -1347,6 +1320,8 @@ class MISPObject(AbstractMISP): attribute = MISPObjectAttribute({}) # Overwrite the parameters of self._default_attributes_parameters with the ones of value attribute.from_dict(object_relation=object_relation, **dict(self._default_attributes_parameters, **value)) + # FIXME New syntax python3 only, keep for later. + # attribute.from_dict(object_relation=object_relation, **{**self._default_attributes_parameters, **value}) self.__fast_attribute_access[object_relation].append(attribute) self.Attribute.append(attribute) self.edited = True diff --git a/pymisp/tools/elfobject.py b/pymisp/tools/elfobject.py index 2dcdeb0..e57654b 100644 --- a/pymisp/tools/elfobject.py +++ b/pymisp/tools/elfobject.py @@ -25,6 +25,7 @@ except ImportError: class ELFObject(AbstractMISPObjectGenerator): def __init__(self, parsed=None, filepath=None, pseudofile=None, standalone=True, **kwargs): + super(ELFObject, self).__init__('elf', standalone=standalone, **kwargs) if not HAS_PYDEEP: logger.warning("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git") if not HAS_LIEF: @@ -44,7 +45,6 @@ class ELFObject(AbstractMISPObjectGenerator): self.__elf = parsed else: raise InvalidMISPObject('Not a lief.ELF.Binary: {}'.format(type(parsed))) - super(ELFObject, self).__init__('elf', standalone=standalone, **kwargs) self.generate_attributes() def generate_attributes(self): diff --git a/pymisp/tools/emailobject.py b/pymisp/tools/emailobject.py index 4a5b54f..0984336 100644 --- a/pymisp/tools/emailobject.py +++ b/pymisp/tools/emailobject.py @@ -13,6 +13,9 @@ logger = logging.getLogger('pymisp') class EMailObject(AbstractMISPObjectGenerator): def __init__(self, filepath=None, pseudofile=None, attach_original_email=True, standalone=True, **kwargs): + # PY3 way: + # super().__init__('file') + super(EMailObject, self).__init__('email', standalone=standalone, **kwargs) if filepath: with open(filepath, 'rb') as f: self.__pseudofile = BytesIO(f.read()) @@ -20,9 +23,6 @@ class EMailObject(AbstractMISPObjectGenerator): self.__pseudofile = pseudofile else: raise InvalidMISPObject('File buffer (BytesIO) or a path is required.') - # PY3 way: - # super().__init__('file') - super(EMailObject, self).__init__('email', standalone=standalone, **kwargs) self.__email = message_from_bytes(self.__pseudofile.getvalue(), policy=policy.default) if attach_original_email: self.add_attribute('eml', value='Full email.eml', data=self.__pseudofile) diff --git a/pymisp/tools/fileobject.py b/pymisp/tools/fileobject.py index e9b05cd..55e946c 100644 --- a/pymisp/tools/fileobject.py +++ b/pymisp/tools/fileobject.py @@ -29,6 +29,9 @@ except ImportError: class FileObject(AbstractMISPObjectGenerator): def __init__(self, filepath=None, pseudofile=None, filename=None, standalone=True, **kwargs): + # PY3 way: + # super().__init__('file') + super(FileObject, self).__init__('file', standalone=standalone, **kwargs) if not HAS_PYDEEP: logger.warning("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git") if not HAS_MAGIC: @@ -49,9 +52,6 @@ class FileObject(AbstractMISPObjectGenerator): self.__pseudofile = pseudofile else: raise InvalidMISPObject('File buffer (BytesIO) or a path is required.') - # PY3 way: - # super().__init__('file') - super(FileObject, self).__init__('file', standalone=standalone, **kwargs) self.__data = self.__pseudofile.getvalue() self.generate_attributes() diff --git a/pymisp/tools/machoobject.py b/pymisp/tools/machoobject.py index f0f11fb..984798b 100644 --- a/pymisp/tools/machoobject.py +++ b/pymisp/tools/machoobject.py @@ -26,6 +26,9 @@ except ImportError: class MachOObject(AbstractMISPObjectGenerator): def __init__(self, parsed=None, filepath=None, pseudofile=None, standalone=True, **kwargs): + # Python3 way + # super().__init__('elf') + super(MachOObject, self).__init__('macho', standalone=standalone, **kwargs) if not HAS_PYDEEP: logger.warning("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git") if not HAS_LIEF: @@ -45,9 +48,6 @@ class MachOObject(AbstractMISPObjectGenerator): self.__macho = parsed else: raise InvalidMISPObject('Not a lief.MachO.Binary: {}'.format(type(parsed))) - # Python3 way - # super().__init__('elf') - super(MachOObject, self).__init__('macho', standalone=standalone, **kwargs) self.generate_attributes() def generate_attributes(self): diff --git a/pymisp/tools/peobject.py b/pymisp/tools/peobject.py index b24d4cc..fa1b4e5 100644 --- a/pymisp/tools/peobject.py +++ b/pymisp/tools/peobject.py @@ -26,6 +26,9 @@ except ImportError: class PEObject(AbstractMISPObjectGenerator): def __init__(self, parsed=None, filepath=None, pseudofile=None, standalone=True, **kwargs): + # Python3 way + # super().__init__('pe') + super(PEObject, self).__init__('pe', standalone=standalone, **kwargs) if not HAS_PYDEEP: logger.warning("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git") if not HAS_LIEF: @@ -45,9 +48,6 @@ class PEObject(AbstractMISPObjectGenerator): self.__pe = parsed else: raise InvalidMISPObject('Not a lief.PE.Binary: {}'.format(type(parsed))) - # Python3 way - # super().__init__('pe') - super(PEObject, self).__init__('pe', standalone=standalone, **kwargs) self.generate_attributes() def _is_exe(self): diff --git a/pymisp/tools/sshauthkeyobject.py b/pymisp/tools/sshauthkeyobject.py index 30f675d..95bb937 100644 --- a/pymisp/tools/sshauthkeyobject.py +++ b/pymisp/tools/sshauthkeyobject.py @@ -12,6 +12,9 @@ logger = logging.getLogger('pymisp') class SSHAuthorizedKeysObject(AbstractMISPObjectGenerator): def __init__(self, authorized_keys_path=None, authorized_keys_pseudofile=None, standalone=True, **kwargs): + # PY3 way: + # super().__init__('file') + super(SSHAuthorizedKeysObject, self).__init__('ssh-authorized-keys', standalone=standalone, **kwargs) if authorized_keys_path: with open(authorized_keys_path, 'r') as f: self.__pseudofile = StringIO(f.read()) @@ -19,9 +22,6 @@ class SSHAuthorizedKeysObject(AbstractMISPObjectGenerator): self.__pseudofile = authorized_keys_path else: raise InvalidMISPObject('File buffer (StringIO) or a path is required.') - # PY3 way: - # super().__init__('file') - super(SSHAuthorizedKeysObject, self).__init__('ssh-authorized-keys', standalone=standalone, **kwargs) self.__data = self.__pseudofile.getvalue() self.generate_attributes() diff --git a/setup.py b/setup.py index 83055d9..61819bd 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ setup( ], install_requires=['six', 'requests', 'python-dateutil', 'jsonschema', 'python-dateutil', 'enum34;python_version<"3.4"', - 'functools32;python_version<"3.0"', 'deprecated'], + 'functools32;python_version<"3.0"', 'deprecated', 'cachetools;python_version<"3.0"'], extras_require={'fileobjects': ['lief>=0.8,<0.10;python_version<"3.5"', 'lief>=0.10.0.dev0;python_version>"3.5"', 'python-magic', 'pydeep'], 'neo': ['py2neo'], 'openioc': ['beautifulsoup4'], diff --git a/tests/mispevent_testfiles/existing_event.json b/tests/mispevent_testfiles/existing_event.json index 6587dab..40453af 100644 --- a/tests/mispevent_testfiles/existing_event.json +++ b/tests/mispevent_testfiles/existing_event.json @@ -3919,7 +3919,7 @@ "date": "2017-12-14", "distribution": "3", "id": "9616", - "info": "OSINT - Attackers Deploy New ICS Attack Framework “TRITON” and Cause Operational Disruption to Critical Infrastructure", + "info": "OSINT - Attackers Deploy New ICS Attack Framework \"TRITON\" and Cause Operational Disruption to Critical Infrastructure", "org_id": "2", "orgc_id": "2", "published": false, @@ -4019,7 +4019,7 @@ "date": "2017-10-23", "distribution": "3", "id": "9208", - "info": "Talos: “Cyber Conflict” Decoy Document Used In Real Cyber Conflict", + "info": "Talos: \"Cyber Conflict\" Decoy Document Used In Real Cyber Conflict", "org_id": "291", "orgc_id": "291", "published": true, diff --git a/tests/mispevent_testfiles/existing_event_edited.json b/tests/mispevent_testfiles/existing_event_edited.json index 91d0e53..51da02b 100644 --- a/tests/mispevent_testfiles/existing_event_edited.json +++ b/tests/mispevent_testfiles/existing_event_edited.json @@ -3922,7 +3922,7 @@ "date": "2017-12-14", "distribution": "3", "id": "9616", - "info": "OSINT - Attackers Deploy New ICS Attack Framework “TRITON” and Cause Operational Disruption to Critical Infrastructure", + "info": "OSINT - Attackers Deploy New ICS Attack Framework \"TRITON\" and Cause Operational Disruption to Critical Infrastructure", "org_id": "2", "orgc_id": "2", "published": false, @@ -4022,7 +4022,7 @@ "date": "2017-10-23", "distribution": "3", "id": "9208", - "info": "Talos: “Cyber Conflict” Decoy Document Used In Real Cyber Conflict", + "info": "Talos: \"Cyber Conflict\" Decoy Document Used In Real Cyber Conflict", "org_id": "291", "orgc_id": "291", "published": true, diff --git a/tests/test.py b/tests/test.py index 40477ec..45f55e4 100755 --- a/tests/test.py +++ b/tests/test.py @@ -6,7 +6,7 @@ try: except ImportError as e: print(e) url = 'https://localhost:8443' - key = 'K5yV0CcxdnklzDfCKlnPniIxrMX41utQ2dG13zZ3' + key = 'd6OmdDFvU3Seau3UjwvHS1y3tFQbaRNhJhDX0tjh' import time diff --git a/tests/test_mispevent.py b/tests/test_mispevent.py index b190541..5f31ca4 100644 --- a/tests/test_mispevent.py +++ b/tests/test_mispevent.py @@ -26,20 +26,20 @@ class TestMISPEvent(unittest.TestCase): def test_simple(self): with open('tests/mispevent_testfiles/simple.json', 'r') as f: ref_json = json.load(f) - self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) + self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_event(self): self.init_event() self.mispevent.publish() with open('tests/mispevent_testfiles/event.json', 'r') as f: ref_json = json.load(f) - self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) + self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_loadfile(self): self.mispevent.load_file('tests/mispevent_testfiles/event.json') with open('tests/mispevent_testfiles/event.json', 'r') as f: ref_json = json.load(f) - self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) + self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_event_tag(self): self.init_event() @@ -50,7 +50,7 @@ class TestMISPEvent(unittest.TestCase): self.mispevent.add_tag(new_tag) with open('tests/mispevent_testfiles/event_tags.json', 'r') as f: ref_json = json.load(f) - self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) + self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_attribute(self): self.init_event() @@ -62,13 +62,13 @@ class TestMISPEvent(unittest.TestCase): self.assertEqual(attr_tags[0].name, 'osint') with open('tests/mispevent_testfiles/attribute.json', 'r') as f: ref_json = json.load(f) - self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) + self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) # Fake setting an attribute ID for testing self.mispevent.attributes[0].id = 42 self.mispevent.delete_attribute(42) with open('tests/mispevent_testfiles/attribute_del.json', 'r') as f: ref_json = json.load(f) - self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) + self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_object_tag(self): self.mispevent.add_object(name='file', strict=True) @@ -90,7 +90,7 @@ class TestMISPEvent(unittest.TestCase): self.assertEqual(self.mispevent.objects[0].references[0].relationship_type, 'baz') with open('tests/mispevent_testfiles/event_obj_attr_tag.json', 'r') as f: ref_json = json.load(f) - self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) + self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) @unittest.skip("Not supported on MISP: https://github.com/MISP/MISP/issues/2638 - https://github.com/MISP/PyMISP/issues/168") def test_object_level_tag(self): @@ -100,7 +100,7 @@ class TestMISPEvent(unittest.TestCase): self.mispevent.objects[0].uuid = 'a' with open('tests/mispevent_testfiles/event_obj_tag.json', 'r') as f: ref_json = json.load(f) - self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) + self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_malware(self): with open('tests/mispevent_testfiles/simple.json', 'rb') as f: @@ -112,7 +112,7 @@ class TestMISPEvent(unittest.TestCase): self.assertEqual(attribute.malware_binary, pseudofile) with open('tests/mispevent_testfiles/malware.json', 'r') as f: ref_json = json.load(f) - self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) + self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_existing_malware(self): self.mispevent.load_file('tests/mispevent_testfiles/malware_exist.json') @@ -127,19 +127,20 @@ class TestMISPEvent(unittest.TestCase): sighting.from_dict(value='1', type='bar', timestamp=11111111) with open('tests/mispevent_testfiles/sighting.json', 'r') as f: ref_json = json.load(f) - self.assertEqual(sighting.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) + self.assertEqual(sighting.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_existing_event(self): self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') with open('tests/mispevent_testfiles/existing_event.json', 'r') as f: ref_json = json.load(f) - self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) + + self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_shadow_attributes_existing(self): self.mispevent.load_file('tests/mispevent_testfiles/shadow.json') with open('tests/mispevent_testfiles/shadow.json', 'r') as f: ref_json = json.load(f) - self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) + self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) @unittest.skip("Not supported on MISP.") def test_shadow_attributes(self): @@ -152,7 +153,7 @@ class TestMISPEvent(unittest.TestCase): del p.uuid with open('tests/mispevent_testfiles/proposals.json', 'r') as f: ref_json = json.load(f) - self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) + self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_default_attributes(self): self.mispevent.add_object(name='file', strict=True) @@ -165,7 +166,7 @@ class TestMISPEvent(unittest.TestCase): self.mispevent.objects[1].uuid = 'b' with open('tests/mispevent_testfiles/event_obj_def_param.json', 'r') as f: ref_json = json.load(f) - self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) + self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_obj_default_values(self): self.init_event() @@ -181,7 +182,7 @@ class TestMISPEvent(unittest.TestCase): self.mispevent.objects[0].uuid = 'a' with open('tests/mispevent_testfiles/def_param.json', 'r') as f: ref_json = json.load(f) - self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) + self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_event_not_edited(self): self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') @@ -246,7 +247,7 @@ class TestMISPEvent(unittest.TestCase): self.assertTrue(self.mispevent.edited) with open('tests/mispevent_testfiles/existing_event_edited.json', 'r') as f: ref_json = json.load(f) - self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) + self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_obj_by_id(self): self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') @@ -258,7 +259,7 @@ class TestMISPEvent(unittest.TestCase): self.mispevent.add_object(name='test_object_template', strict=True, misp_objects_path_custom='tests/mispevent_testfiles') with self.assertRaises(InvalidMISPObject) as e: # Fail on required - self.mispevent.to_json() + self.mispevent.to_json(sort_keys=True, indent=2) if sys.version_info >= (3, ): self.assertEqual(e.exception.message, '{\'member3\'} are required.') else: @@ -269,7 +270,7 @@ class TestMISPEvent(unittest.TestCase): del a.uuid with self.assertRaises(InvalidMISPObject) as e: # Fail on requiredOneOf - self.mispevent.to_json() + self.mispevent.to_json(sort_keys=True, indent=2) self.assertEqual(e.exception.message, 'At least one of the following attributes is required: member1, member2') a = self.mispevent.objects[0].add_attribute('member1', value='bar') @@ -278,14 +279,14 @@ class TestMISPEvent(unittest.TestCase): del a.uuid with self.assertRaises(InvalidMISPObject) as e: # member1 is not a multiple - self.mispevent.to_json() + self.mispevent.to_json(sort_keys=True, indent=2) self.assertEqual(e.exception.message, 'Multiple occurrences of member1 is not allowed') self.mispevent.objects[0].attributes = self.mispevent.objects[0].attributes[:2] self.mispevent.objects[0].uuid = 'a' with open('tests/mispevent_testfiles/misp_custom_obj.json', 'r') as f: ref_json = json.load(f) - self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) + self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) if __name__ == '__main__': diff --git a/tests/testlive_comprehensive.py b/tests/testlive_comprehensive.py index 628de3f..170c6eb 100644 --- a/tests/testlive_comprehensive.py +++ b/tests/testlive_comprehensive.py @@ -1577,7 +1577,7 @@ class TestComprehensive(unittest.TestCase): remote_types = remote.pop('types') remote_categories = remote.pop('categories') remote_category_type_mappings = remote.pop('category_type_mappings') - local = self.admin_misp_connector.describe_types_local + local = dict(self.admin_misp_connector.describe_types_local) local_types = local.pop('types') local_categories = local.pop('categories') local_category_type_mappings = local.pop('category_type_mappings')