MISP
/
PyMISP
mirror of https://github.com/MISP/PyMISP
2
0
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'master' of https://github.com/MISP/PyMISP

pull/508/head
Koen Van Impe 2019-12-23 21:20:51 +01:00
parent bdffbd46a6 fac748dd4c
commit f883161fd2
21 changed files with 897 additions and 425 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

103
CHANGELOG.txt
View File

@ -2,9 +2,112 @@ Changelog
========= =========
v2.4.119.1 (2019-12-17)
-----------------------
New
~~~
- URLObject (requires pyfaup) [Raphaël Vinot]
Changes
~~~~~~~
- Version bump. [Raphaël Vinot]
- Bump test files. [Raphaël Vinot]
- Bump misp-objects. [Raphaël Vinot]
- Debug travis error message. [Raphaël Vinot]
- [types] eppn type added. [Alexandre Dulaunoy]
- Fix typo. [Raphaël Vinot]
- Move scrippsco2 feed generator to a sub directory. [Raphaël Vinot]
- Update documentation. [Raphaël Vinot]
Fix #396
- Bump objects. [Raphaël Vinot]
Fix
~~~
- Properly test custom objects. [Raphaël Vinot]
- Adding a sighting takes a little bit of time. [Raphaël Vinot]
- Test case on reference. [Raphaël Vinot]
- Add missing fields to event & attribute for the feed output. [Raphaël
Vinot]
- Make sure the publish timestamp is bumped on update. [Raphaël Vinot]
v2.4.119 (2019-12-02)
---------------------
Changes
~~~~~~~
- Bump changelog. [Raphaël Vinot]
- Bump version. [Raphaël Vinot]
- Bump dependencies. [Raphaël Vinot]
Fix
~~~
- Bump lief to 0.10.1. [Raphaël Vinot]
- Update tests. [Raphaël Vinot]
- Raise PyMISPError instead of Exception. [Raphaël Vinot]
- Rename feed_meta_generator so it clearly fails with python<3.6.
[Raphaël Vinot]
v2.4.117.3 (2019-11-25)
-----------------------
New
~~~
- Script to generate the metadata of a feed out of a directory. [Raphaël
Vinot]
- Add to_feed export to MISPEvent. [Raphaël Vinot]
- Validate object templates. [Raphaël Vinot]
fix https://github.com/MISP/misp-objects/issues/199
- Test cases for restricted tags. [Raphaël Vinot]
Fix #483
- Get Database Schema Diagnostic. [Raphaël Vinot]
Fix #492
Changes
~~~~~~~
- Bump changelog. [Raphaël Vinot]
- Bump version. [Raphaël Vinot]
- Bump dependencies. [Raphaël Vinot]
- Require stable version of lief again. [Raphaël Vinot]
- Few more improvements on the feed export. [Raphaël Vinot]
- Bump misp-objects. [Raphaël Vinot]
- Make the feed generator more generic. [Raphaël Vinot]
- Use New version of PyMISP in the feed generator. [Raphaël Vinot]
- Bump misp-object. [Raphaël Vinot]
- Allow to sort and indent the json output for objects. [Raphaël Vinot]
- Bump objects. [Raphaël Vinot]
- Bump dependencies. [Raphaël Vinot]
- [test] feed test updated as botvrij is now TLS by default. [Alexandre
Dulaunoy]
Fix
~~~
- Improve stability of feed output. [Raphaël Vinot]
- Do not unitialize the uuid in MISPEvent. [Raphaël Vinot]
- Bump url template version in test cases. [Raphaël Vinot]
- Python 2.7 tests. [Raphaël Vinot]
- Print the full json blob in debug mode. [Raphaël Vinot]
Related https://github.com/MISP/PyMISP/issues/462
Other
~~~~~
- Cch: Bump misp-objects. [Raphaël Vinot]
v2.4.117.2 (2019-10-30) v2.4.117.2 (2019-10-30)
----------------------- -----------------------
Changes
~~~~~~~
- Bump changelog. [Raphaël Vinot]
Fix Fix
~~~ ~~~
- Avoid exception on legacy MISP. [Raphaël Vinot] - Avoid exception on legacy MISP. [Raphaël Vinot]

482
Pipfile.lock generated
View File

@ -18,24 +18,25 @@
"default": { "default": {
"attrs": { "attrs": {
"hashes": [ "hashes": [
"sha256:ec20e7a4825331c1b5ebf261d111e16fa9612c1f7a5e1f884f12bd53a664dfd2", "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c",
"sha256:f913492e1663d3c36f502e5e9ba6cd13cf19d7fab50aa13239e420fef95e1396" "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72"
], ],
"version": "==19.2.0" "version": "==19.3.0"
}, },
"beautifulsoup4": { "beautifulsoup4": {
"hashes": [ "hashes": [
"sha256:5279c36b4b2ec2cb4298d723791467e3000e5384a43ea0cdf5d45207c7e97169", "sha256:5279c36b4b2ec2cb4298d723791467e3000e5384a43ea0cdf5d45207c7e97169",
"sha256:6135db2ba678168c07950f9a16c4031822c6f4aec75a65e0a97bc5ca09789931",
"sha256:dcdef580e18a76d54002088602eba453eec38ebbcafafeaabd8cab12b6155d57" "sha256:dcdef580e18a76d54002088602eba453eec38ebbcafafeaabd8cab12b6155d57"
], ],
"version": "==4.8.1" "version": "==4.8.1"
}, },
"certifi": { "certifi": {
"hashes": [ "hashes": [
"sha256:e4f3620cfea4f83eedc95b24abd9cd56f3c4b146dd0177e83a21b4eb49e21e50", "sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3",
"sha256:fd7c7c74727ddcf00e9acd26bba8da604ffec95bf1c2144e67aff7a8b50e6cef" "sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f"
], ],
"version": "==2019.9.11" "version": "==2019.11.28"
}, },
"chardet": { "chardet": {
"hashes": [ "hashes": [
@ -46,17 +47,17 @@
}, },
"decorator": { "decorator": {
"hashes": [ "hashes": [
"sha256:86156361c50488b84a3f148056ea716ca587df2f0de1d34750d35c21312725de", "sha256:54c38050039232e1db4ad7375cfce6748d7b41c29e95a081c8a6d2c30364a2ce",
"sha256:f069f3a01830ca754ba5258fde2278454a0b5b79e0d7f5c13b3b97e57d4acff6" "sha256:5d19b92a3c8f7f101c8dd86afd86b0f061a8ce4540ab8cd401fa2542756bce6d"
], ],
"version": "==4.4.0" "version": "==4.4.1"
}, },
"deprecated": { "deprecated": {
"hashes": [ "hashes": [
"sha256:a515c4cf75061552e0284d123c3066fbbe398952c87333a92b8fc3dd8e4f9cc1", "sha256:408038ab5fdeca67554e8f6742d1521cd3cd0ee0ff9d47f29318a4f4da31c308",
"sha256:b07b414c8aac88f60c1d837d21def7e83ba711052e03b3cbaff27972567a8f8d" "sha256:8b6a5aa50e482d8244a62e5582b96c372e87e3a28e8b49c316e46b95c76a611d"
], ],
"version": "==1.2.6" "version": "==1.2.7"
}, },
"idna": { "idna": {
"hashes": [ "hashes": [
@ -65,60 +66,81 @@
], ],
"version": "==2.8" "version": "==2.8"
}, },
"importlib-metadata": {
"hashes": [
"sha256:b044f07694ef14a6683b097ba56bd081dbc7cdc7c7fe46011e499dfecc082f21",
"sha256:e6ac600a142cf2db707b1998382cc7fc3b02befb7273876e01b8ad10b9652742"
],
"markers": "python_version < '3.8'",
"version": "==1.1.0"
},
"jsonschema": { "jsonschema": {
"hashes": [ "hashes": [
"sha256:5f9c0a719ca2ce14c5de2fd350a64fd2d13e8539db29836a86adc990bb1a068f", "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163",
"sha256:8d4a2b7b6c2237e0199c8ea1a6d3e05bf118e289ae2b9d7ba444182a2959560d" "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"
], ],
"version": "==3.0.2" "version": "==3.2.0"
}, },
"lief": { "lief": {
"hashes": [ "hashes": [
"sha256:0efba18d7b9776529ea5c18c771b35871896a8ceb95a19351e50d4813a11c632", "sha256:276cc63ec12a21bdf01b8d30962692c17499788234f0765247ca7a35872097ec",
"sha256:3d9c7bb1e353e875f295a72a58d3a37ae1ba3e1ff1beb57b8a65f1a726064093", "sha256:3e6baaeb52bdc339b5f19688b58fd8d5778b92e50221f920cedfa2bec1f4d5c2",
"sha256:3db5939e7d95f776f9866586128c2a5be614eaec43ab985ac27ff2c531f8ac5f", "sha256:45e5c592b57168c447698381d927eb2386ffdd52afe0c48245f848d4cc7ee05a",
"sha256:4c61598818b0091d80839875aa107cfd10ae1017a3e9c9de4bc002622b8e3179", "sha256:6547752b5db105cd41c9fa65d0d7452a4d7541b77ffee716b46246c6d81e172f",
"sha256:4f26d07bdada8ca5ef3dc5fa2f71f20f7e8ab4f78f7c5e00134477f51feb6a80", "sha256:83b51e01627b5982662f9550ac1230758aa56945ed86829e4291932d98417da3",
"sha256:55fe3c8a0990dce16ab5bf88df707f1eacac4eb34561667ac478497e0e0807c7", "sha256:895599194ea7495bf304e39317b04df20cccf799fc2751867cc1aa4997cfcdae",
"sha256:68bcf18e40c9412d2d08d6311e04eb6c19e20ec174764706da2d602c45aa4fd5", "sha256:8a91cee2568306fe1d2bf84341b459c85368317d01d7105fa49e4f4ede837076",
"sha256:7ff910d99361022451e9c25e34cb844768e2fa347cfb0f4ad70f531810d776d4", "sha256:913b36a67707dc2afa72f117bab9856ea3f434f332b04a002a0f9723c8779320",
"sha256:ac571152d0b864e8d376bc733c5728a224316be1cdefc290174f1bf8ab10ec70", "sha256:9f604a361a3b1b3ed5fdafed0321c5956cb3b265b5efe2250d1bf8911a80c65b",
"sha256:dd17a7cdcd29a2efca3d4cb4fb078a06daf1cafec8912560965a8d8dbf346739", "sha256:a487fe7234c04bccd58223dbb79214421176e2629814c7a4a887764cceb5be7c",
"sha256:efa5f3523c01f7f0f5f2c14e5ac808e2447d1435c6a2872e5ab1a97ef1b0db9b", "sha256:bc8488fb0661cb436fe4bb4fe947d0f9aa020e9acaed233ccf01ab04d888c68a",
"sha256:f1aadb344b5e14b308167bd2c9f31f1915e3c4e3f9a9ca92ff7b7bfbede5034c" "sha256:bddbf333af62310a10cb738a1df1dc2b140dd9c663b55ba3500c10c249d416d2",
"sha256:cce48d7c97cef85e01e6cfeff55f2068956b5c0257eb9c2d2c6d15e33dd1e4fc",
"sha256:f8b3f66956c56b582b3adc573bf2a938c25fb21c8894b373a113e24c494fc982"
], ],
"version": "==0.10.0.dev0" "version": "==0.10.1"
},
"more-itertools": {
"hashes": [
"sha256:53ff73f186307d9c8ef17a9600309154a6ae27f25579e80af4db8f047ba14bc2",
"sha256:a0ea684c39bc4315ba7aae406596ef191fd84f873d2d2751f84d64e81a7a2d45"
],
"version": "==8.0.0"
}, },
"pillow": { "pillow": {
"hashes": [ "hashes": [
"sha256:00fdeb23820f30e43bba78eb9abb00b7a937a655de7760b2e09101d63708b64e", "sha256:047d9473cf68af50ac85f8ee5d5f21a60f849bc17d348da7fc85711287a75031",
"sha256:01f948e8220c85eae1aa1a7f8edddcec193918f933fb07aaebe0bfbbcffefbf1", "sha256:0f66dc6c8a3cc319561a633b6aa82c44107f12594643efa37210d8c924fc1c71",
"sha256:08abf39948d4b5017a137be58f1a52b7101700431f0777bec3d897c3949f74e6", "sha256:12c9169c4e8fe0a7329e8658c7e488001f6b4c8e88740e76292c2b857af2e94c",
"sha256:099a61618b145ecb50c6f279666bbc398e189b8bc97544ae32b8fcb49ad6b830", "sha256:248cffc168896982f125f5c13e9317c059f74fffdb4152893339f3be62a01340",
"sha256:2c1c61546e73de62747e65807d2cc4980c395d4c5600ecb1f47a650c6fa78c79", "sha256:27faf0552bf8c260a5cee21a76e031acaea68babb64daf7e8f2e2540745082aa",
"sha256:2ed9c4f694861642401f27dc3cb99772be67cd190e84845c749dae0a06c3bfae", "sha256:285edafad9bc60d96978ed24d77cdc0b91dace88e5da8c548ba5937c425bca8b",
"sha256:338581b30b908e111be578f0297255f6b57a51358cd16fa0e6f664c9a1f88bff", "sha256:384b12c9aa8ef95558abdcb50aada56d74bc7cc131dd62d28c2d0e4d3aadd573",
"sha256:38c7d48a21cd06fdeee93987147b9b1c55b73b4cfcbf83240568bfbd5adee447", "sha256:38950b3a707f6cef09cd3cbb142474357ad1a985ceb44d921bdf7b4647b3e13e",
"sha256:43fd026f613c8e48a25eba1a92f4d2ad7f3903c95d8c33a11611a7717d2ab654", "sha256:4aad1b88933fd6dc2846552b89ad0c74ddbba2f0884e2c162aa368374bf5abab",
"sha256:4548236844327a718ce3bb182ab32a16fa2050c61e334e959f554cac052fb0df", "sha256:4ac6148008c169603070c092e81f88738f1a0c511e07bd2bb0f9ef542d375da9",
"sha256:5090857876c58885cfa388dc649e5db30aae98a068c26f3fd0ac9d7d9a4d9572", "sha256:4deb1d2a45861ae6f0b12ea0a786a03d19d29edcc7e05775b85ec2877cb54c5e",
"sha256:5bbba34f97a26a93f5e8dec469ca4ddd712451418add43da946dbaed7f7a98d2", "sha256:59aa2c124df72cc75ed72c8d6005c442d4685691a30c55321e00ed915ad1a291",
"sha256:65a28969a025a0eb4594637b6103201dc4ed2a9508bdab56ac33e43e3081c404", "sha256:5a47d2123a9ec86660fe0e8d0ebf0aa6bc6a17edc63f338b73ea20ba11713f12",
"sha256:892bb52b70bd5ea9dbbc3ac44f38e84f5a04e9d8b1bff48159d96cb795b81159", "sha256:5cc901c2ab9409b4b7ac7b5bcc3e86ac14548627062463da0af3b6b7c555a871",
"sha256:8a9becd5cbd5062f973bcd2e7bc79483af310222de112b6541f8af1f93a3cc42", "sha256:6c1db03e8dff7b9f955a0fb9907eb9ca5da75b5ce056c0c93d33100a35050281",
"sha256:972a7aaeb7c4a2795b52eef52ee991ef040b31009f36deca6207a986607b55f3", "sha256:7ce80c0a65a6ea90ef9c1f63c8593fcd2929448613fc8da0adf3e6bfad669d08",
"sha256:97b119c436bfa96a92ac2ca525f7025836d4d4e64b1c9f9eff8dbaf3ff1d86f3", "sha256:809c19241c14433c5d6135e1b6c72da4e3b56d5c865ad5736ab99af8896b8f41",
"sha256:9ba37698e242223f8053cc158f130aee046a96feacbeab65893dbe94f5530118", "sha256:83792cb4e0b5af480588601467c0764242b9a483caea71ef12d22a0d0d6bdce2",
"sha256:b1b0e1f626a0f079c0d3696db70132fb1f29aa87c66aecb6501a9b8be64ce9f7", "sha256:846fa202bd7ee0f6215c897a1d33238ef071b50766339186687bd9b7a6d26ac5",
"sha256:c14c1224fd1a5be2733530d648a316974dbbb3c946913562c6005a76f21ca042", "sha256:9f5529fc02009f96ba95bea48870173426879dc19eec49ca8e08cd63ecd82ddb",
"sha256:c79a8546c48ae6465189e54e3245a97ddf21161e33ff7eaa42787353417bb2b6", "sha256:a423c2ea001c6265ed28700df056f75e26215fd28c001e93ef4380b0f05f9547",
"sha256:ceb76935ac4ebdf6d7bc845482a4450b284c6ccfb281e34da51d510658ab34d8", "sha256:ac4428094b42907aba5879c7c000d01c8278d451a3b7cccd2103e21f6397ea75",
"sha256:e22bffaad04b4d16e1c091baed7f2733fc1ebb91e0c602abf1b6834d17158b1f", "sha256:b1ae48d87f10d1384e5beecd169c77502fcc04a2c00a4c02b85f0a94b419e5f9",
"sha256:ec883b8e44d877bda6f94a36313a1c6063f8b1997aa091628ae2f34c7f97c8d5", "sha256:bf4e972a88f8841d8fdc6db1a75e0f8d763e66e3754b03006cbc3854d89f1cb1",
"sha256:f1baa54d50ec031d1a9beb89974108f8f2c0706f49798f4777df879df0e1adb6", "sha256:c6414f6aad598364aaf81068cabb077894eb88fed99c6a65e6e8217bab62ae7a",
"sha256:f53a5385932cda1e2c862d89460992911a89768c65d176ff8c50cddca4d29bed" "sha256:c710fcb7ee32f67baf25aa9ffede4795fd5d93b163ce95fdc724383e38c9df96",
"sha256:c7be4b8a09852291c3c48d3c25d1b876d2494a0a674980089ac9d5e0d78bd132",
"sha256:c9e5ffb910b14f090ac9c38599063e354887a5f6d7e6d26795e916b4514f2c1a",
"sha256:e0697b826da6c2472bb6488db4c0a7fa8af0d52fa08833ceb3681358914b14e5",
"sha256:e9a3edd5f714229d41057d56ac0f39ad9bdba6767e8c888c951869f0bdd129b0"
], ],
"version": "==6.2.0" "version": "==6.2.1"
}, },
"pydeep": { "pydeep": {
"hashes": [ "hashes": [
@ -139,20 +161,20 @@
"pymispwarninglists": { "pymispwarninglists": {
"editable": true, "editable": true,
"git": "https://github.com/MISP/PyMISPWarningLists.git", "git": "https://github.com/MISP/PyMISPWarningLists.git",
"ref": "8a47f8b7f723a268e5a6b5420fe4b873e4fd6a0b" "ref": "1257a2e378ffb9f3dfcc4a0e83bde4ae1b040c83"
}, },
"pyrsistent": { "pyrsistent": {
"hashes": [ "hashes": [
"sha256:34b47fa169d6006b32e99d4b3c4031f155e6e68ebcc107d6454852e8e0ee6533" "sha256:f3b280d030afb652f79d67c5586157c5c1355c9a58dfc7940566e28d28f3df1b"
], ],
"version": "==0.15.4" "version": "==0.15.6"
}, },
"python-dateutil": { "python-dateutil": {
"hashes": [ "hashes": [
"sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb", "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c",
"sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e" "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a"
], ],
"version": "==2.8.0" "version": "==2.8.1"
}, },
"python-magic": { "python-magic": {
"hashes": [ "hashes": [
@ -163,31 +185,36 @@
}, },
"reportlab": { "reportlab": {
"hashes": [ "hashes": [
"sha256:06b7c7436fa6d4844c7637161f3297c7a96240f35622ab2d219e4fd8387c0ab2", "sha256:149f0eeb4ea716441638b05fd6d3667d32f1463f3eac50b63e100a73a5533cdd",
"sha256:0a5acf67bd9812e38ed84be8994c07a8136b0a8f4c14a1c66c9c73a9567a9a44", "sha256:1aa9a2e1a87749db265b592ad25e498b39f70fce9f53a012cdf69f74259b6e43",
"sha256:1c8ca145d03e3c620866b06febb241b179197b58fb07454fbc8e9d6184cdcc93", "sha256:1f5ce489adb2db2862249492e6367539cfa65b781cb06dcf13363dc52219be7e",
"sha256:2f8d785660ee316874c86abad345633ce8c652e88e03ae8a10f1fdadc72fd23d", "sha256:23b28ba1784a6c52a926c075abd9f396d03670e71934b24db5ff684f8b870e0f",
"sha256:4869d342352c92a812ce40555ef2a9cfbd722390d67fe61f1d6ec770e9ca41a3", "sha256:3d3de0f4facdd7e3c56ecbc55733a958b86c35a8e7ba6066c7b1ba383e282f58",
"sha256:493e0dcd9c085d46acf4fe3f00f941e562490a74b651409039a0dee2a0d76555", "sha256:484d346b8f463ba2ddaf6d365c6ac5971cd062528b6d5ba68cac02b9435366c5",
"sha256:4e606e3ee9345e68cd205022d526250ad2a1164eea8f1e29d77d6ad08631b0ba", "sha256:4da2467def21f2e20720b21f6c18e7f7866720a955c716b990e94e3979fe913f",
"sha256:5bf91bae8995db91650fda658129c268515358b756fd16c0261a9dd641df1856", "sha256:5ebdf22daee7d8e630134d94f477fe6abd65a65449d4eec682a7b458b5249604",
"sha256:6df0730f8f715aa12333bd6d2a72eea3a989381451861186d9b5e71889454ac7", "sha256:655a1b68be18a73fec5233fb5d81f726b4db32269e487aecf5b6853cca926d86",
"sha256:7195c6ea096d10c91cc470f9f0ced3ad74470d9c0fd97923b5e764597dd13671", "sha256:6c535a304888dafe50c2c24d4924aeefc11e0542488ee6965f6133d415e86bbc",
"sha256:7431c979e2b498e8e20abf458f360a451717d76c3c1bd49d1fc5697d3504f8e5", "sha256:7560ef655ac6448bb257fd34bfdfb8d546f9c7c0900ed8963fb8509f75e8ca80",
"sha256:7f7f70a8d4b573d1ff65a81415b4b6ed9545630f381dff4a69307640e09d381d", "sha256:7a1c2fa3e6310dbe47efee2020dc0f25be7a75ff09a8fedc4a87d4397f3810c1",
"sha256:9945433667a46f054d1125b4ca86fe9ee31feb254728b38242e9a6008c135efe", "sha256:817c344b9aa53b5bfc2f58ff82111a1e85ca4c8b68d1add088b547360a6ebcfa",
"sha256:b1cdbfc1fd54ac947b9f0114e00ab94e945db679f1e03357a3c00f3a85e73eea", "sha256:81d950e398d6758aeaeeb267aa1a62940735414c980f77dd0a270cef1782a43d",
"sha256:bf149847a2fd8f24b788a8abbf97a2b9a73edc5b1bd719384b786eb84bcad15e", "sha256:83ef44936ef4e9c432d62bc2b72ec8d772b87af319d123e827a72e9b6884c851",
"sha256:ce514bfce2bf3e302f52aba9929fe3ac7d918cfea2f5d3e30bf9dac9658bf094", "sha256:9f975adc2c7a236403f0bc91d7a3916e644e47b1f1e3990325f15e73b83581ec",
"sha256:d243d4c8cf1a7e78b734c03628b684ec5de25df1f02ccea2e10fbd217430cb72", "sha256:a5ca59e2b7e70a856de6db9dadd3e11a1b3b471c999585284d5c1d479c01cf5d",
"sha256:d4bee20f52b8c3c477dc780780654cafcfc0eb34d8d6960c13a34a444b431f09", "sha256:ad2cf5a673c05fae9e91e987994b95205c13c5fa55d7393cf8b06f9de6f92990",
"sha256:e730529bd1f62034c50f70a2b05fadbf7d1402d39ff69c9dc63db066d0ef8a46", "sha256:b8c3d76276372f87b7c8ff22065dbc072cca5ffb06ba0267edc298df7acf942d",
"sha256:eb54ecfbf1abe6134073b7b35fd40442c4cd81bb9a5bee1a3038b8867b721bfb", "sha256:b93f7f908e916d9413dd8c04da1ccb3977e446803f59078424decdc0de449133",
"sha256:f18ec70f5ee6a78b3bb4361e55f3a5ef34eb253f1e72fba76f29f0d680cd446f", "sha256:c0ecd0af92c759edec0d24ba92f4a18c28d4a19229ae7c8249f94e82f3d76288",
"sha256:f6be66f69198dcd04a79faa6052f756d35643496321858f06931c7b1ed9833ab", "sha256:c9e38eefc90a02c072a87a627ff66b2d67c23f6f82274d2aa7fb28e644e8f409",
"sha256:fc5c23a53fbd97b8aab4968c8548ce5cea4a54a26b4f8c1e6835df7adb8d0fe2" "sha256:ca2a1592d2e181a04372d0276ee847308ea206dfe7c86fe94769e7ac126e6e85",
"sha256:ce1dfc9beec83e66250ca3afaf5ddf6b9a3ce70a30a9526dec7c6bec3266baf1",
"sha256:d3550c90751132b26b72a78954905974f33b1237335fbe0d8be957f9636c376a",
"sha256:e35a574f4e5ec0fdd5dc354e74ec143d853abd7f76db435ffe2a57d0161a22eb",
"sha256:ee5cafca6ef1a38fef8cbf3140dd2198ad1ee82331530b546039216ef94f93cb",
"sha256:fa1c969176cb3594a785c6818bcb943ebd49453791f702380b13a35fa23b385a"
], ],
"version": "==3.5.28" "version": "==3.5.32"
}, },
"requests": { "requests": {
"hashes": [ "hashes": [
@ -198,24 +225,24 @@
}, },
"six": { "six": {
"hashes": [ "hashes": [
"sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", "sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd",
"sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" "sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66"
], ],
"version": "==1.12.0" "version": "==1.13.0"
}, },
"soupsieve": { "soupsieve": {
"hashes": [ "hashes": [
"sha256:605f89ad5fdbfefe30cdc293303665eff2d188865d4dbe4eb510bba1edfbfce3", "sha256:bdb0d917b03a1369ce964056fc195cfdff8819c40de04695a80bc813c3cfa1f5",
"sha256:b91d676b330a0ebd5b21719cb6e9b57c57d433671f65b9c28dd3461d9a1ed0b6" "sha256:e2c1c5dee4a1c36bcb790e0fabd5492d874b8ebd4617622c4f6a731701060dda"
], ],
"version": "==1.9.4" "version": "==1.9.5"
}, },
"urllib3": { "urllib3": {
"hashes": [ "hashes": [
"sha256:3de946ffbed6e6746608990594d08faac602528ac7015ac28d33cee6a45b7398", "sha256:a8a318824cc77d1fd4b2bec2ded92646630d7fe8619497b142c84a9e6f5a7293",
"sha256:9a107b99a5393caf59c7aa3c1249c16e6879447533d0887f4336dde834c7be86" "sha256:f3c5fd51747d450d4dcf6f923c81f78f811aab8205fda64b0aba34a4e48b0745"
], ],
"version": "==1.25.6" "version": "==1.25.7"
}, },
"validators": { "validators": {
"hashes": [ "hashes": [
@ -228,6 +255,13 @@
"sha256:565a021fd19419476b9362b05eeaa094178de64f8361e44468f9e9d7843901e1" "sha256:565a021fd19419476b9362b05eeaa094178de64f8361e44468f9e9d7843901e1"
], ],
"version": "==1.11.2" "version": "==1.11.2"
},
"zipp": {
"hashes": [
"sha256:3718b1cbcd963c7d4c5511a8240812904164b7f381b647143a89d3b98f9bcd8e",
"sha256:f06903e9f1f43b12d371004b4ac7b06ab39a44adc747266928ae6debfa7b3335"
],
"version": "==0.6.0"
} }
}, },
"develop": { "develop": {
@ -240,10 +274,10 @@
}, },
"attrs": { "attrs": {
"hashes": [ "hashes": [
"sha256:ec20e7a4825331c1b5ebf261d111e16fa9612c1f7a5e1f884f12bd53a664dfd2", "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c",
"sha256:f913492e1663d3c36f502e5e9ba6cd13cf19d7fab50aa13239e420fef95e1396" "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72"
], ],
"version": "==19.2.0" "version": "==19.3.0"
}, },
"babel": { "babel": {
"hashes": [ "hashes": [
@ -255,16 +289,17 @@
"beautifulsoup4": { "beautifulsoup4": {
"hashes": [ "hashes": [
"sha256:5279c36b4b2ec2cb4298d723791467e3000e5384a43ea0cdf5d45207c7e97169", "sha256:5279c36b4b2ec2cb4298d723791467e3000e5384a43ea0cdf5d45207c7e97169",
"sha256:6135db2ba678168c07950f9a16c4031822c6f4aec75a65e0a97bc5ca09789931",
"sha256:dcdef580e18a76d54002088602eba453eec38ebbcafafeaabd8cab12b6155d57" "sha256:dcdef580e18a76d54002088602eba453eec38ebbcafafeaabd8cab12b6155d57"
], ],
"version": "==4.8.1" "version": "==4.8.1"
}, },
"certifi": { "certifi": {
"hashes": [ "hashes": [
"sha256:e4f3620cfea4f83eedc95b24abd9cd56f3c4b146dd0177e83a21b4eb49e21e50", "sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3",
"sha256:fd7c7c74727ddcf00e9acd26bba8da604ffec95bf1c2144e67aff7a8b50e6cef" "sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f"
], ],
"version": "==2019.9.11" "version": "==2019.11.28"
}, },
"chardet": { "chardet": {
"hashes": [ "hashes": [
@ -349,17 +384,17 @@
}, },
"decorator": { "decorator": {
"hashes": [ "hashes": [
"sha256:86156361c50488b84a3f148056ea716ca587df2f0de1d34750d35c21312725de", "sha256:54c38050039232e1db4ad7375cfce6748d7b41c29e95a081c8a6d2c30364a2ce",
"sha256:f069f3a01830ca754ba5258fde2278454a0b5b79e0d7f5c13b3b97e57d4acff6" "sha256:5d19b92a3c8f7f101c8dd86afd86b0f061a8ce4540ab8cd401fa2542756bce6d"
], ],
"version": "==4.4.0" "version": "==4.4.1"
}, },
"deprecated": { "deprecated": {
"hashes": [ "hashes": [
"sha256:a515c4cf75061552e0284d123c3066fbbe398952c87333a92b8fc3dd8e4f9cc1", "sha256:408038ab5fdeca67554e8f6742d1521cd3cd0ee0ff9d47f29318a4f4da31c308",
"sha256:b07b414c8aac88f60c1d837d21def7e83ba711052e03b3cbaff27972567a8f8d" "sha256:8b6a5aa50e482d8244a62e5582b96c372e87e3a28e8b49c316e46b95c76a611d"
], ],
"version": "==1.2.6" "version": "==1.2.7"
}, },
"docopt": { "docopt": {
"hashes": [ "hashes": [
@ -389,6 +424,14 @@
], ],
"version": "==1.1.0" "version": "==1.1.0"
}, },
"importlib-metadata": {
"hashes": [
"sha256:b044f07694ef14a6683b097ba56bd081dbc7cdc7c7fe46011e499dfecc082f21",
"sha256:e6ac600a142cf2db707b1998382cc7fc3b02befb7273876e01b8ad10b9652742"
],
"markers": "python_version < '3.8'",
"version": "==1.1.0"
},
"jinja2": { "jinja2": {
"hashes": [ "hashes": [
"sha256:74320bb91f31270f9551d46522e33af46a80c3d619f4a4bf42b3164d30b5911f", "sha256:74320bb91f31270f9551d46522e33af46a80c3d619f4a4bf42b3164d30b5911f",
@ -398,27 +441,29 @@
}, },
"jsonschema": { "jsonschema": {
"hashes": [ "hashes": [
"sha256:5f9c0a719ca2ce14c5de2fd350a64fd2d13e8539db29836a86adc990bb1a068f", "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163",
"sha256:8d4a2b7b6c2237e0199c8ea1a6d3e05bf118e289ae2b9d7ba444182a2959560d" "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"
], ],
"version": "==3.0.2" "version": "==3.2.0"
}, },
"lief": { "lief": {
"hashes": [ "hashes": [
"sha256:0efba18d7b9776529ea5c18c771b35871896a8ceb95a19351e50d4813a11c632", "sha256:276cc63ec12a21bdf01b8d30962692c17499788234f0765247ca7a35872097ec",
"sha256:3d9c7bb1e353e875f295a72a58d3a37ae1ba3e1ff1beb57b8a65f1a726064093", "sha256:3e6baaeb52bdc339b5f19688b58fd8d5778b92e50221f920cedfa2bec1f4d5c2",
"sha256:3db5939e7d95f776f9866586128c2a5be614eaec43ab985ac27ff2c531f8ac5f", "sha256:45e5c592b57168c447698381d927eb2386ffdd52afe0c48245f848d4cc7ee05a",
"sha256:4c61598818b0091d80839875aa107cfd10ae1017a3e9c9de4bc002622b8e3179", "sha256:6547752b5db105cd41c9fa65d0d7452a4d7541b77ffee716b46246c6d81e172f",
"sha256:4f26d07bdada8ca5ef3dc5fa2f71f20f7e8ab4f78f7c5e00134477f51feb6a80", "sha256:83b51e01627b5982662f9550ac1230758aa56945ed86829e4291932d98417da3",
"sha256:55fe3c8a0990dce16ab5bf88df707f1eacac4eb34561667ac478497e0e0807c7", "sha256:895599194ea7495bf304e39317b04df20cccf799fc2751867cc1aa4997cfcdae",
"sha256:68bcf18e40c9412d2d08d6311e04eb6c19e20ec174764706da2d602c45aa4fd5", "sha256:8a91cee2568306fe1d2bf84341b459c85368317d01d7105fa49e4f4ede837076",
"sha256:7ff910d99361022451e9c25e34cb844768e2fa347cfb0f4ad70f531810d776d4", "sha256:913b36a67707dc2afa72f117bab9856ea3f434f332b04a002a0f9723c8779320",
"sha256:ac571152d0b864e8d376bc733c5728a224316be1cdefc290174f1bf8ab10ec70", "sha256:9f604a361a3b1b3ed5fdafed0321c5956cb3b265b5efe2250d1bf8911a80c65b",
"sha256:dd17a7cdcd29a2efca3d4cb4fb078a06daf1cafec8912560965a8d8dbf346739", "sha256:a487fe7234c04bccd58223dbb79214421176e2629814c7a4a887764cceb5be7c",
"sha256:efa5f3523c01f7f0f5f2c14e5ac808e2447d1435c6a2872e5ab1a97ef1b0db9b", "sha256:bc8488fb0661cb436fe4bb4fe947d0f9aa020e9acaed233ccf01ab04d888c68a",
"sha256:f1aadb344b5e14b308167bd2c9f31f1915e3c4e3f9a9ca92ff7b7bfbede5034c" "sha256:bddbf333af62310a10cb738a1df1dc2b140dd9c663b55ba3500c10c249d416d2",
"sha256:cce48d7c97cef85e01e6cfeff55f2068956b5c0257eb9c2d2c6d15e33dd1e4fc",
"sha256:f8b3f66956c56b582b3adc573bf2a938c25fb21c8894b373a113e24c494fc982"
], ],
"version": "==0.10.0.dev0" "version": "==0.10.1"
}, },
"markupsafe": { "markupsafe": {
"hashes": [ "hashes": [
@ -460,6 +505,13 @@
"index": "pypi", "index": "pypi",
"version": "==0.55.0" "version": "==0.55.0"
}, },
"more-itertools": {
"hashes": [
"sha256:53ff73f186307d9c8ef17a9600309154a6ae27f25579e80af4db8f047ba14bc2",
"sha256:a0ea684c39bc4315ba7aae406596ef191fd84f873d2d2751f84d64e81a7a2d45"
],
"version": "==8.0.0"
},
"neobolt": { "neobolt": {
"hashes": [ "hashes": [
"sha256:56b86b8b2c3facdd54589e60ecd22e0234d6f40645ab2e2cf87ef0cd79df20af" "sha256:56b86b8b2c3facdd54589e60ecd22e0234d6f40645ab2e2cf87ef0cd79df20af"
@ -490,34 +542,38 @@
}, },
"pillow": { "pillow": {
"hashes": [ "hashes": [
"sha256:00fdeb23820f30e43bba78eb9abb00b7a937a655de7760b2e09101d63708b64e", "sha256:047d9473cf68af50ac85f8ee5d5f21a60f849bc17d348da7fc85711287a75031",
"sha256:01f948e8220c85eae1aa1a7f8edddcec193918f933fb07aaebe0bfbbcffefbf1", "sha256:0f66dc6c8a3cc319561a633b6aa82c44107f12594643efa37210d8c924fc1c71",
"sha256:08abf39948d4b5017a137be58f1a52b7101700431f0777bec3d897c3949f74e6", "sha256:12c9169c4e8fe0a7329e8658c7e488001f6b4c8e88740e76292c2b857af2e94c",
"sha256:099a61618b145ecb50c6f279666bbc398e189b8bc97544ae32b8fcb49ad6b830", "sha256:248cffc168896982f125f5c13e9317c059f74fffdb4152893339f3be62a01340",
"sha256:2c1c61546e73de62747e65807d2cc4980c395d4c5600ecb1f47a650c6fa78c79", "sha256:27faf0552bf8c260a5cee21a76e031acaea68babb64daf7e8f2e2540745082aa",
"sha256:2ed9c4f694861642401f27dc3cb99772be67cd190e84845c749dae0a06c3bfae", "sha256:285edafad9bc60d96978ed24d77cdc0b91dace88e5da8c548ba5937c425bca8b",
"sha256:338581b30b908e111be578f0297255f6b57a51358cd16fa0e6f664c9a1f88bff", "sha256:384b12c9aa8ef95558abdcb50aada56d74bc7cc131dd62d28c2d0e4d3aadd573",
"sha256:38c7d48a21cd06fdeee93987147b9b1c55b73b4cfcbf83240568bfbd5adee447", "sha256:38950b3a707f6cef09cd3cbb142474357ad1a985ceb44d921bdf7b4647b3e13e",
"sha256:43fd026f613c8e48a25eba1a92f4d2ad7f3903c95d8c33a11611a7717d2ab654", "sha256:4aad1b88933fd6dc2846552b89ad0c74ddbba2f0884e2c162aa368374bf5abab",
"sha256:4548236844327a718ce3bb182ab32a16fa2050c61e334e959f554cac052fb0df", "sha256:4ac6148008c169603070c092e81f88738f1a0c511e07bd2bb0f9ef542d375da9",
"sha256:5090857876c58885cfa388dc649e5db30aae98a068c26f3fd0ac9d7d9a4d9572", "sha256:4deb1d2a45861ae6f0b12ea0a786a03d19d29edcc7e05775b85ec2877cb54c5e",
"sha256:5bbba34f97a26a93f5e8dec469ca4ddd712451418add43da946dbaed7f7a98d2", "sha256:59aa2c124df72cc75ed72c8d6005c442d4685691a30c55321e00ed915ad1a291",
"sha256:65a28969a025a0eb4594637b6103201dc4ed2a9508bdab56ac33e43e3081c404", "sha256:5a47d2123a9ec86660fe0e8d0ebf0aa6bc6a17edc63f338b73ea20ba11713f12",
"sha256:892bb52b70bd5ea9dbbc3ac44f38e84f5a04e9d8b1bff48159d96cb795b81159", "sha256:5cc901c2ab9409b4b7ac7b5bcc3e86ac14548627062463da0af3b6b7c555a871",
"sha256:8a9becd5cbd5062f973bcd2e7bc79483af310222de112b6541f8af1f93a3cc42", "sha256:6c1db03e8dff7b9f955a0fb9907eb9ca5da75b5ce056c0c93d33100a35050281",
"sha256:972a7aaeb7c4a2795b52eef52ee991ef040b31009f36deca6207a986607b55f3", "sha256:7ce80c0a65a6ea90ef9c1f63c8593fcd2929448613fc8da0adf3e6bfad669d08",
"sha256:97b119c436bfa96a92ac2ca525f7025836d4d4e64b1c9f9eff8dbaf3ff1d86f3", "sha256:809c19241c14433c5d6135e1b6c72da4e3b56d5c865ad5736ab99af8896b8f41",
"sha256:9ba37698e242223f8053cc158f130aee046a96feacbeab65893dbe94f5530118", "sha256:83792cb4e0b5af480588601467c0764242b9a483caea71ef12d22a0d0d6bdce2",
"sha256:b1b0e1f626a0f079c0d3696db70132fb1f29aa87c66aecb6501a9b8be64ce9f7", "sha256:846fa202bd7ee0f6215c897a1d33238ef071b50766339186687bd9b7a6d26ac5",
"sha256:c14c1224fd1a5be2733530d648a316974dbbb3c946913562c6005a76f21ca042", "sha256:9f5529fc02009f96ba95bea48870173426879dc19eec49ca8e08cd63ecd82ddb",
"sha256:c79a8546c48ae6465189e54e3245a97ddf21161e33ff7eaa42787353417bb2b6", "sha256:a423c2ea001c6265ed28700df056f75e26215fd28c001e93ef4380b0f05f9547",
"sha256:ceb76935ac4ebdf6d7bc845482a4450b284c6ccfb281e34da51d510658ab34d8", "sha256:ac4428094b42907aba5879c7c000d01c8278d451a3b7cccd2103e21f6397ea75",
"sha256:e22bffaad04b4d16e1c091baed7f2733fc1ebb91e0c602abf1b6834d17158b1f", "sha256:b1ae48d87f10d1384e5beecd169c77502fcc04a2c00a4c02b85f0a94b419e5f9",
"sha256:ec883b8e44d877bda6f94a36313a1c6063f8b1997aa091628ae2f34c7f97c8d5", "sha256:bf4e972a88f8841d8fdc6db1a75e0f8d763e66e3754b03006cbc3854d89f1cb1",
"sha256:f1baa54d50ec031d1a9beb89974108f8f2c0706f49798f4777df879df0e1adb6", "sha256:c6414f6aad598364aaf81068cabb077894eb88fed99c6a65e6e8217bab62ae7a",
"sha256:f53a5385932cda1e2c862d89460992911a89768c65d176ff8c50cddca4d29bed" "sha256:c710fcb7ee32f67baf25aa9ffede4795fd5d93b163ce95fdc724383e38c9df96",
"sha256:c7be4b8a09852291c3c48d3c25d1b876d2494a0a674980089ac9d5e0d78bd132",
"sha256:c9e5ffb910b14f090ac9c38599063e354887a5f6d7e6d26795e916b4514f2c1a",
"sha256:e0697b826da6c2472bb6488db4c0a7fa8af0d52fa08833ceb3681358914b14e5",
"sha256:e9a3edd5f714229d41057d56ac0f39ad9bdba6767e8c888c951869f0bdd129b0"
], ],
"version": "==6.2.0" "version": "==6.2.1"
}, },
"prompt-toolkit": { "prompt-toolkit": {
"hashes": [ "hashes": [
@ -529,17 +585,19 @@
}, },
"psutil": { "psutil": {
"hashes": [ "hashes": [
"sha256:028a1ec3c6197eadd11e7b46e8cc2f0720dc18ac6d7aabdb8e8c0d6c9704f000", "sha256:094f899ac3ef72422b7e00411b4ed174e3c5a2e04c267db6643937ddba67a05b",
"sha256:503e4b20fa9d3342bcf58191bbc20a4a5ef79ca7df8972e6197cc14c5513e73d", "sha256:10b7f75cc8bd676cfc6fa40cd7d5c25b3f45a0e06d43becd7c2d2871cbb5e806",
"sha256:863a85c1c0a5103a12c05a35e59d336e1d665747e531256e061213e2e90f63f3", "sha256:1b1575240ca9a90b437e5a40db662acd87bbf181f6aa02f0204978737b913c6b",
"sha256:954f782608bfef9ae9f78e660e065bd8ffcfaea780f9f2c8a133bb7cb9e826d7", "sha256:21231ef1c1a89728e29b98a885b8e0a8e00d09018f6da5cdc1f43f988471a995",
"sha256:b6e08f965a305cd84c2d07409bc16fbef4417d67b70c53b299116c5b895e3f45", "sha256:28f771129bfee9fc6b63d83a15d857663bbdcae3828e1cb926e91320a9b5b5cd",
"sha256:bc96d437dfbb8865fc8828cf363450001cb04056bbdcdd6fc152c436c8a74c61", "sha256:70387772f84fa5c3bb6a106915a2445e20ac8f9821c5914d7cbde148f4d7ff73",
"sha256:cf49178021075d47c61c03c0229ac0c60d5e2830f8cab19e2d88e579b18cdb76", "sha256:b560f5cd86cf8df7bcd258a851ca1ad98f0d5b8b98748e877a0aec4e9032b465",
"sha256:d5350cb66690915d60f8b233180f1e49938756fb2d501c93c44f8fb5b970cc63", "sha256:b74b43fecce384a57094a83d2778cdfc2e2d9a6afaadd1ebecb2e75e0d34e10d",
"sha256:eba238cf1989dfff7d483c029acb0ac4fcbfc15de295d682901f0e2497e6781a" "sha256:e85f727ffb21539849e6012f47b12f6dd4c44965e56591d8dec6e8bc9ab96f4a",
"sha256:fd2e09bb593ad9bdd7429e779699d2d47c1268cbde4dda95fcd1bd17544a0217",
"sha256:ffad8eb2ac614518bbe3c0b8eb9dffdb3a8d2e3a7d5da51c5b974fb723a5c5aa"
], ],
"version": "==5.6.3" "version": "==5.6.7"
}, },
"py2neo": { "py2neo": {
"hashes": [ "hashes": [
@ -572,23 +630,23 @@
}, },
"pyparsing": { "pyparsing": {
"hashes": [ "hashes": [
"sha256:6f98a7b9397e206d78cc01df10131398f1c8b8510a2f4d97d9abd82e1aacdd80", "sha256:20f995ecd72f2a1f4bf6b072b63b22e2eb457836601e76d6e5dfcd75436acc1f",
"sha256:d9338df12903bbf5d65a0e4e87c2161968b10d2e489652bb47001d82a9b028b4" "sha256:4ca62001be367f01bd3e92ecbb79070272a9d4964dce6a48a82ff0b8bc7e683a"
], ],
"version": "==2.4.2" "version": "==2.4.5"
}, },
"pyrsistent": { "pyrsistent": {
"hashes": [ "hashes": [
"sha256:34b47fa169d6006b32e99d4b3c4031f155e6e68ebcc107d6454852e8e0ee6533" "sha256:f3b280d030afb652f79d67c5586157c5c1355c9a58dfc7940566e28d28f3df1b"
], ],
"version": "==0.15.4" "version": "==0.15.6"
}, },
"python-dateutil": { "python-dateutil": {
"hashes": [ "hashes": [
"sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb", "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c",
"sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e" "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a"
], ],
"version": "==2.8.0" "version": "==2.8.1"
}, },
"python-magic": { "python-magic": {
"hashes": [ "hashes": [
@ -613,31 +671,36 @@
}, },
"reportlab": { "reportlab": {
"hashes": [ "hashes": [
"sha256:06b7c7436fa6d4844c7637161f3297c7a96240f35622ab2d219e4fd8387c0ab2", "sha256:149f0eeb4ea716441638b05fd6d3667d32f1463f3eac50b63e100a73a5533cdd",
"sha256:0a5acf67bd9812e38ed84be8994c07a8136b0a8f4c14a1c66c9c73a9567a9a44", "sha256:1aa9a2e1a87749db265b592ad25e498b39f70fce9f53a012cdf69f74259b6e43",
"sha256:1c8ca145d03e3c620866b06febb241b179197b58fb07454fbc8e9d6184cdcc93", "sha256:1f5ce489adb2db2862249492e6367539cfa65b781cb06dcf13363dc52219be7e",
"sha256:2f8d785660ee316874c86abad345633ce8c652e88e03ae8a10f1fdadc72fd23d", "sha256:23b28ba1784a6c52a926c075abd9f396d03670e71934b24db5ff684f8b870e0f",
"sha256:4869d342352c92a812ce40555ef2a9cfbd722390d67fe61f1d6ec770e9ca41a3", "sha256:3d3de0f4facdd7e3c56ecbc55733a958b86c35a8e7ba6066c7b1ba383e282f58",
"sha256:493e0dcd9c085d46acf4fe3f00f941e562490a74b651409039a0dee2a0d76555", "sha256:484d346b8f463ba2ddaf6d365c6ac5971cd062528b6d5ba68cac02b9435366c5",
"sha256:4e606e3ee9345e68cd205022d526250ad2a1164eea8f1e29d77d6ad08631b0ba", "sha256:4da2467def21f2e20720b21f6c18e7f7866720a955c716b990e94e3979fe913f",
"sha256:5bf91bae8995db91650fda658129c268515358b756fd16c0261a9dd641df1856", "sha256:5ebdf22daee7d8e630134d94f477fe6abd65a65449d4eec682a7b458b5249604",
"sha256:6df0730f8f715aa12333bd6d2a72eea3a989381451861186d9b5e71889454ac7", "sha256:655a1b68be18a73fec5233fb5d81f726b4db32269e487aecf5b6853cca926d86",
"sha256:7195c6ea096d10c91cc470f9f0ced3ad74470d9c0fd97923b5e764597dd13671", "sha256:6c535a304888dafe50c2c24d4924aeefc11e0542488ee6965f6133d415e86bbc",
"sha256:7431c979e2b498e8e20abf458f360a451717d76c3c1bd49d1fc5697d3504f8e5", "sha256:7560ef655ac6448bb257fd34bfdfb8d546f9c7c0900ed8963fb8509f75e8ca80",
"sha256:7f7f70a8d4b573d1ff65a81415b4b6ed9545630f381dff4a69307640e09d381d", "sha256:7a1c2fa3e6310dbe47efee2020dc0f25be7a75ff09a8fedc4a87d4397f3810c1",
"sha256:9945433667a46f054d1125b4ca86fe9ee31feb254728b38242e9a6008c135efe", "sha256:817c344b9aa53b5bfc2f58ff82111a1e85ca4c8b68d1add088b547360a6ebcfa",
"sha256:b1cdbfc1fd54ac947b9f0114e00ab94e945db679f1e03357a3c00f3a85e73eea", "sha256:81d950e398d6758aeaeeb267aa1a62940735414c980f77dd0a270cef1782a43d",
"sha256:bf149847a2fd8f24b788a8abbf97a2b9a73edc5b1bd719384b786eb84bcad15e", "sha256:83ef44936ef4e9c432d62bc2b72ec8d772b87af319d123e827a72e9b6884c851",
"sha256:ce514bfce2bf3e302f52aba9929fe3ac7d918cfea2f5d3e30bf9dac9658bf094", "sha256:9f975adc2c7a236403f0bc91d7a3916e644e47b1f1e3990325f15e73b83581ec",
"sha256:d243d4c8cf1a7e78b734c03628b684ec5de25df1f02ccea2e10fbd217430cb72", "sha256:a5ca59e2b7e70a856de6db9dadd3e11a1b3b471c999585284d5c1d479c01cf5d",
"sha256:d4bee20f52b8c3c477dc780780654cafcfc0eb34d8d6960c13a34a444b431f09", "sha256:ad2cf5a673c05fae9e91e987994b95205c13c5fa55d7393cf8b06f9de6f92990",
"sha256:e730529bd1f62034c50f70a2b05fadbf7d1402d39ff69c9dc63db066d0ef8a46", "sha256:b8c3d76276372f87b7c8ff22065dbc072cca5ffb06ba0267edc298df7acf942d",
"sha256:eb54ecfbf1abe6134073b7b35fd40442c4cd81bb9a5bee1a3038b8867b721bfb", "sha256:b93f7f908e916d9413dd8c04da1ccb3977e446803f59078424decdc0de449133",
"sha256:f18ec70f5ee6a78b3bb4361e55f3a5ef34eb253f1e72fba76f29f0d680cd446f", "sha256:c0ecd0af92c759edec0d24ba92f4a18c28d4a19229ae7c8249f94e82f3d76288",
"sha256:f6be66f69198dcd04a79faa6052f756d35643496321858f06931c7b1ed9833ab", "sha256:c9e38eefc90a02c072a87a627ff66b2d67c23f6f82274d2aa7fb28e644e8f409",
"sha256:fc5c23a53fbd97b8aab4968c8548ce5cea4a54a26b4f8c1e6835df7adb8d0fe2" "sha256:ca2a1592d2e181a04372d0276ee847308ea206dfe7c86fe94769e7ac126e6e85",
"sha256:ce1dfc9beec83e66250ca3afaf5ddf6b9a3ce70a30a9526dec7c6bec3266baf1",
"sha256:d3550c90751132b26b72a78954905974f33b1237335fbe0d8be957f9636c376a",
"sha256:e35a574f4e5ec0fdd5dc354e74ec143d853abd7f76db435ffe2a57d0161a22eb",
"sha256:ee5cafca6ef1a38fef8cbf3140dd2198ad1ee82331530b546039216ef94f93cb",
"sha256:fa1c969176cb3594a785c6818bcb943ebd49453791f702380b13a35fa23b385a"
], ],
"version": "==3.5.28" "version": "==3.5.32"
}, },
"requests": { "requests": {
"hashes": [ "hashes": [
@ -656,10 +719,10 @@
}, },
"six": { "six": {
"hashes": [ "hashes": [
"sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", "sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd",
"sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" "sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66"
], ],
"version": "==1.12.0" "version": "==1.13.0"
}, },
"snowballstemmer": { "snowballstemmer": {
"hashes": [ "hashes": [
@ -670,24 +733,24 @@
}, },
"soupsieve": { "soupsieve": {
"hashes": [ "hashes": [
"sha256:605f89ad5fdbfefe30cdc293303665eff2d188865d4dbe4eb510bba1edfbfce3", "sha256:bdb0d917b03a1369ce964056fc195cfdff8819c40de04695a80bc813c3cfa1f5",
"sha256:b91d676b330a0ebd5b21719cb6e9b57c57d433671f65b9c28dd3461d9a1ed0b6" "sha256:e2c1c5dee4a1c36bcb790e0fabd5492d874b8ebd4617622c4f6a731701060dda"
], ],
"version": "==1.9.4" "version": "==1.9.5"
}, },
"sphinx": { "sphinx": {
"hashes": [ "hashes": [
"sha256:0d586b0f8c2fc3cc6559c5e8fd6124628110514fda0e5d7c82e682d749d2e845", "sha256:31088dfb95359384b1005619827eaee3056243798c62724fd3fa4b84ee4d71bd",
"sha256:839a3ed6f6b092bb60f492024489cc9e6991360fb9f52ed6361acd510d261069" "sha256:52286a0b9d7caa31efee301ec4300dbdab23c3b05da1c9024b4e84896fb73d79"
], ],
"version": "==2.2.0" "version": "==2.2.1"
}, },
"sphinx-autodoc-typehints": { "sphinx-autodoc-typehints": {
"hashes": [ "hashes": [
"sha256:0d968ec3ee4f7fe7695ab6facf5cd2d74d3cea67584277458ad9b2788ebbcc3b", "sha256:27c9e6ef4f4451766ab8d08b2d8520933b97beb21c913f3df9ab2e59b56e6c6c",
"sha256:8edca714fd3de8e43467d7e51dd3812fe999f8874408a639f7c38a9e1a5a4eb3" "sha256:a6b3180167479aca2c4d1ed3b5cb044a70a76cccd6b38662d39288ebd9f0dff0"
], ],
"version": "==1.8.0" "version": "==1.10.3"
}, },
"sphinxcontrib-applehelp": { "sphinxcontrib-applehelp": {
"hashes": [ "hashes": [
@ -733,10 +796,10 @@
}, },
"urllib3": { "urllib3": {
"hashes": [ "hashes": [
"sha256:3de946ffbed6e6746608990594d08faac602528ac7015ac28d33cee6a45b7398", "sha256:a8a318824cc77d1fd4b2bec2ded92646630d7fe8619497b142c84a9e6f5a7293",
"sha256:9a107b99a5393caf59c7aa3c1249c16e6879447533d0887f4336dde834c7be86" "sha256:f3c5fd51747d450d4dcf6f923c81f78f811aab8205fda64b0aba34a4e48b0745"
], ],
"version": "==1.25.6" "version": "==1.25.7"
}, },
"validators": { "validators": {
"hashes": [ "hashes": [
@ -756,6 +819,13 @@
"sha256:565a021fd19419476b9362b05eeaa094178de64f8361e44468f9e9d7843901e1" "sha256:565a021fd19419476b9362b05eeaa094178de64f8361e44468f9e9d7843901e1"
], ],
"version": "==1.11.2" "version": "==1.11.2"
},
"zipp": {
"hashes": [
"sha256:3718b1cbcd963c7d4c5511a8240812904164b7f381b647143a89d3b98f9bcd8e",
"sha256:f06903e9f1f43b12d371004b4ac7b06ab39a44adc747266928ae6debfa7b3335"
],
"version": "==0.6.0"
} }
} }
} }

82
examples/import_scrippsc02.py → examples/climate/scrippsco2.py
View File

@ -1,21 +1,42 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import datetime
from dateutil.parser import parse from dateutil.parser import parse
import csv import csv
from pathlib import Path from pathlib import Path
import json
from uuid import uuid4
import requests import requests
from pymisp import MISPEvent, MISPObject, MISPTag from pymisp import MISPEvent, MISPObject, MISPTag, MISPOrganisation
from pymisp.tools import feed_meta_generator
from keys import misp_url, misp_key, misp_verifycert
from pymisp import ExpandedPyMISP
class Scrippts: class Scrippts:
def __init__(self): def __init__(self, output_dir: str= 'output', org_name: str='CIRCL',
self.misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert) org_uuid: str='55f6ea5e-2c60-40e5-964f-47a8950d210f'):
self.misp_org = MISPOrganisation()
self.misp_org.name = org_name
self.misp_org.uuid = org_uuid
self.output_dir = Path(output_dir)
self.output_dir.mkdir(exist_ok=True)
self.data_dir = self.output_dir / 'data'
self.data_dir.mkdir(exist_ok=True)
self.scrippts_meta_file = self.output_dir / '.meta_scrippts'
self.scrippts_meta = {}
if self.scrippts_meta_file.exists():
# Format: <infofield>,<uuid>.json
with self.scrippts_meta_file.open() as f:
reader = csv.reader(f)
for row in reader:
self.scrippts_meta[row[0]] = row[1]
else:
self.scrippts_meta_file.touch()
def geolocation_alt(self) -> MISPObject: def geolocation_alt(self) -> MISPObject:
# Alert, NWT, Canada # Alert, NWT, Canada
@ -200,9 +221,7 @@ class Scrippts:
return tag return tag
def fetch(self, url): def fetch(self, url):
filepath = Path('scrippts') / Path(url).name filepath = self.data_dir / Path(url).name
if filepath.exists():
return filepath
r = requests.get(url) r = requests.get(url)
if r.status_code != 200 or r.text[0] != '"': if r.status_code != 200 or r.text[0] != '"':
print(url) print(url)
@ -211,42 +230,45 @@ class Scrippts:
f.write(r.text) f.write(r.text)
return filepath return filepath
def get_existing_event_to_update(self, infofield):
found = self.misp.search(eventinfo=infofield, pythonify=True)
if found:
event = found[0]
return event
return False
def import_all(self, stations_short_names, interval, data_type): def import_all(self, stations_short_names, interval, data_type):
object_creator = getattr(self, f'{interval}_flask_{data_type}') object_creator = getattr(self, f'{interval}_flask_{data_type}')
if data_type == 'co2': if data_type == 'co2':
base_url = 'http://scrippsco2.ucsd.edu/assets/data/atmospheric/stations/flask_co2/' base_url = 'https://scrippsco2.ucsd.edu/assets/data/atmospheric/stations/flask_co2/'
elif data_type in ['c13', 'o18']: elif data_type in ['c13', 'o18']:
base_url = 'http://scrippsco2.ucsd.edu/assets/data/atmospheric/stations/flask_isotopic/' base_url = 'https://scrippsco2.ucsd.edu/assets/data/atmospheric/stations/flask_isotopic/'
for station in stations_short_names: for station in stations_short_names:
url = f'{base_url}/{interval}/{interval}_flask_{data_type}_{station}.csv' url = f'{base_url}/{interval}/{interval}_flask_{data_type}_{station}.csv'
infofield = f'[{station.upper()}] {interval} average atmospheric {data_type} concentrations' infofield = f'[{station.upper()}] {interval} average atmospheric {data_type} concentrations'
filepath = self.fetch(url) filepath = self.fetch(url)
if not filepath: if not filepath:
continue continue
update = True if infofield in self.scrippts_meta:
event = self.get_existing_event_to_update(infofield)
if event:
location = event.get_objects_by_name('geolocation')[0]
if not event:
event = MISPEvent() event = MISPEvent()
event.load_file(str(self.output_dir / self.scrippts_meta[infofield]))
location = event.get_objects_by_name('geolocation')[0]
update = True
else:
event = MISPEvent()
event.uuid = str(uuid4())
event.info = infofield event.info = infofield
event.Orgc = self.misp_org
event.add_tag(getattr(self, f'tag_{station}')()) event.add_tag(getattr(self, f'tag_{station}')())
location = getattr(self, f'geolocation_{station}')() location = getattr(self, f'geolocation_{station}')()
event.add_object(location) event.add_object(location)
event.add_attribute('link', f'http://scrippsco2.ucsd.edu/data/atmospheric_co2/{station}') event.add_attribute('link', f'https://scrippsco2.ucsd.edu/data/atmospheric_co2/{station}')
update = False update = False
with self.scrippts_meta_file.open('a') as f:
writer = csv.writer(f)
writer.writerow([infofield, f'{event.uuid}.json'])
object_creator(event, location, filepath, update) object_creator(event, location, filepath, update)
if update: if update:
self.misp.update_event(event) # Bump the publish timestamp
else: event.publish_timestamp = datetime.datetime.timestamp(datetime.datetime.now())
self.misp.add_event(event) feed_output = event.to_feed(with_meta=False)
with (self.output_dir / f'{event.uuid}.json').open('w') as f:
# json.dump(feed_output, f, indent=2, sort_keys=True) # For testing
json.dump(feed_output, f)
def import_monthly_co2_all(self): def import_monthly_co2_all(self):
to_import = ['alt', 'ptb', 'stp', 'ljo', 'bcs', 'mlo', 'kum', 'chr', 'sam', 'ker', 'nzd'] to_import = ['alt', 'ptb', 'stp', 'ljo', 'bcs', 'mlo', 'kum', 'chr', 'sam', 'ker', 'nzd']
@ -458,10 +480,14 @@ class Scrippts:
if __name__ == '__main__': if __name__ == '__main__':
i = Scrippts() output_dir = 'scrippsco2_feed'
i = Scrippts(output_dir=output_dir)
i.import_daily_co2_all() i.import_daily_co2_all()
i.import_daily_c13_all() i.import_daily_c13_all()
i.import_daily_o18_all() i.import_daily_o18_all()
i.import_monthly_co2_all() i.import_monthly_co2_all()
i.import_monthly_c13_all() i.import_monthly_c13_all()
i.import_monthly_o18_all() i.import_monthly_o18_all()
feed_meta_generator(Path(output_dir))

179
examples/feed-generator/generate.py
View File

@ -4,151 +4,36 @@
import sys import sys
import json import json
import os import os
import hashlib from pymisp import ExpandedPyMISP
from pymisp import PyMISP
from settings import url, key, ssl, outputdir, filters, valid_attribute_distribution_levels from settings import url, key, ssl, outputdir, filters, valid_attribute_distribution_levels
objectsFields = {
'Attribute': {
'uuid',
'value',
'category',
'type',
'comment',
'data',
'timestamp',
'to_ids',
'object_relation',
'disable_correlation'
},
'Event': {
'uuid',
'info',
'threat_level_id',
'analysis',
'timestamp',
'publish_timestamp',
'published',
'date',
'extends_uuid'
},
'Object': {
'name',
'meta-category',
'description',
'template_uuid',
'template_version',
'uuid',
'timestamp',
'distribution',
'sharing_group_id',
'comment'
},
'ObjectReference': {
'uuid',
'timestamp',
'relationship_type',
'comment',
'object_uuid',
'referenced_uuid'
},
'Orgc': {
'name',
'uuid'
},
'Tag': {
'name',
'colour',
'exportable'
}
}
objectsToSave = {
'Orgc': {},
'Tag': {},
'Attribute': {
'Tag': {}
},
'Object': {
'Attribute': {
'Tag': {}
},
'ObjectReference': {}
}
}
valid_attribute_distributions = [] valid_attribute_distributions = []
attributeHashes = []
def init(): def init():
# If we have an old settings.py file then this variable won't exist # If we have an old settings.py file then this variable won't exist
global valid_attribute_distributions global valid_attribute_distributions
try: try:
valid_attribute_distributions = valid_attribute_distribution_levels valid_attribute_distributions = [int(v) for v in valid_attribute_distribution_levels]
except Exception: except Exception:
valid_attribute_distributions = ['0', '1', '2', '3', '4', '5'] valid_attribute_distributions = [0, 1, 2, 3, 4, 5]
return PyMISP(url, key, ssl) return ExpandedPyMISP(url, key, ssl)
def recursiveExtract(container, containerType, leaf, eventUuid): def saveEvent(event):
temp = {}
if containerType in ['Attribute', 'Object']:
if (__blockByDistribution(container)):
return False
for field in objectsFields[containerType]:
if field in container:
temp[field] = container[field]
if (containerType == 'Attribute'):
global attributeHashes
if ('|' in container['type'] or container['type'] == 'malware-sample'):
split = container['value'].split('|')
attributeHashes.append([hashlib.md5(split[0].encode("utf-8")).hexdigest(), eventUuid])
attributeHashes.append([hashlib.md5(split[1].encode("utf-8")).hexdigest(), eventUuid])
else:
attributeHashes.append([hashlib.md5(container['value'].encode("utf-8")).hexdigest(), eventUuid])
children = leaf.keys()
for childType in children:
childContainer = container.get(childType)
if (childContainer):
if (type(childContainer) is dict):
temp[childType] = recursiveExtract(childContainer, childType, leaf[childType], eventUuid)
else:
temp[childType] = []
for element in childContainer:
processed = recursiveExtract(element, childType, leaf[childType], eventUuid)
if (processed):
temp[childType].append(processed)
return temp
def saveEvent(misp, uuid):
event = misp.get_event(uuid)
if not event.get('Event'):
print('Error while fetching event: {}'.format(event['message']))
sys.exit('Could not create file for event ' + uuid + '.')
event['Event'] = recursiveExtract(event['Event'], 'Event', objectsToSave, event['Event']['uuid'])
event = json.dumps(event)
eventFile = open(os.path.join(outputdir, uuid + '.json'), 'w')
eventFile.write(event)
eventFile.close()
def __blockByDistribution(element):
if element['distribution'] not in valid_attribute_distributions:
return True
return False
def saveHashes():
if not attributeHashes:
return False
try: try:
hashFile = open(os.path.join(outputdir, 'hashes.csv'), 'w') with open(os.path.join(outputdir, f'{event["uuid"]}.json'), 'w') as f:
for element in attributeHashes: json.dump(event, f, indent=2)
except Exception as e:
print(e)
sys.exit('Could not create the event dump.')
def saveHashes(hashes):
try:
with open(os.path.join(outputdir, 'hashes.csv'), 'w') as hashFile:
for element in hashes:
hashFile.write('{},{}\n'.format(element[0], element[1])) hashFile.write('{},{}\n'.format(element[0], element[1]))
hashFile.close()
except Exception as e: except Exception as e:
print(e) print(e)
sys.exit('Could not create the quick hash lookup file.') sys.exit('Could not create the quick hash lookup file.')
@ -164,41 +49,31 @@ def saveManifest(manifest):
sys.exit('Could not create the manifest file.') sys.exit('Could not create the manifest file.')
def __addEventToManifest(event):
tags = []
for eventTag in event['EventTag']:
tags.append({'name': eventTag['Tag']['name'],
'colour': eventTag['Tag']['colour']})
return {'Orgc': event['Orgc'],
'Tag': tags,
'info': event['info'],
'date': event['date'],
'analysis': event['analysis'],
'threat_level_id': event['threat_level_id'],
'timestamp': event['timestamp']
}
if __name__ == '__main__': if __name__ == '__main__':
misp = init() misp = init()
try: try:
r = misp.get_index(filters) events = misp.search(metadata=True, limit=200, **filters, pythonify=True)
events = r['response']
print(events[0])
except Exception as e: except Exception as e:
print(e) print(e)
sys.exit("Invalid response received from MISP.") sys.exit("Invalid response received from MISP.")
if len(events) == 0: if len(events) == 0:
sys.exit("No events returned.") sys.exit("No events returned.")
manifest = {} manifest = {}
hashes = []
counter = 1 counter = 1
total = len(events) total = len(events)
for event in events: for event in events:
saveEvent(misp, event['uuid']) e = misp.get_event(event.uuid, pythonify=True)
manifest[event['uuid']] = __addEventToManifest(event) e_feed = e.to_feed(valid_distributions=valid_attribute_distributions, with_meta=True)
if not e_feed:
print(f'Invalid distribution {e.distribution}, skipping')
continue
hashes += [[h, e.uuid] for h in e_feed.pop('_hashes')]
manifest.update(e_feed.pop('_manifest'))
saveEvent(e_feed)
print("Event " + str(counter) + "/" + str(total) + " exported.") print("Event " + str(counter) + "/" + str(total) + " exported.")
counter += 1 counter += 1
saveManifest(manifest) saveManifest(manifest)
print('Manifest saved.') print('Manifest saved.')
saveHashes() saveHashes(hashes)
print('Hashes saved. Feed creation completed.') print('Hashes saved. Feed creation completed.')

15
examples/generate_meta_feed.py Normal file
View File

@ -0,0 +1,15 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from pymisp.tools import feed_meta_generator
import argparse
from pathlib import Path
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Build meta files for feed')
parser.add_argument("--feed", required=True, help="Path to directory containing the feed.")
args = parser.parse_args()
feed = Path(args.feed)
feed_meta_generator(feed)

2
pymisp/__init__.py
View File

@ -1,4 +1,4 @@
__version__ = '2.4.117.1' __version__ = '2.4.119.1'
import logging import logging
import warnings import warnings
import sys import sys

46
pymisp/abstract.py
View File

@ -24,7 +24,7 @@ except ImportError:
import logging import logging
from enum import Enum from enum import Enum
from .exceptions import PyMISPInvalidFormat from .exceptions import PyMISPInvalidFormat, PyMISPError
logger = logging.getLogger('pymisp') logger = logging.getLogger('pymisp')
@ -185,7 +185,12 @@ class AbstractMISP(MutableMapping, MISPFileCache):
__describe_types = describe_types __describe_types = describe_types
def __init__(self, **kwargs): def __init__(self, **kwargs):
"""Abstract class for all the MISP objects""" """Abstract class for all the MISP objects.
NOTE: Every method in every classes inheriting this one are doing
changes in memory and do not modify data on a remote MISP instance.
To do so, you need to call the respective add_* or update_*
methods in ExpandedPyMISP/PyMISP.
"""
super(AbstractMISP, self).__init__() super(AbstractMISP, self).__init__()
self.__edited = True # As we create a new object, we assume it is edited self.__edited = True # As we create a new object, we assume it is edited
self.__not_jsonable = [] self.__not_jsonable = []
@ -282,6 +287,27 @@ class AbstractMISP(MutableMapping, MISPFileCache):
"""This method is used by the JSON encoder""" """This method is used by the JSON encoder"""
return self.to_dict() return self.to_dict()
def _to_feed(self):
if not hasattr(self, '_fields_for_feed'):
raise PyMISPError('Unable to export in the feed format, _fields_for_feed is missing.')
if hasattr(self, '_set_default') and callable(self._set_default):
self._set_default()
to_return = {}
for field in self._fields_for_feed:
if getattr(self, field, None) is not None:
if field in ['timestamp', 'publish_timestamp']:
to_return[field] = self._datetime_to_timestamp(getattr(self, field))
elif isinstance(getattr(self, field), (datetime.datetime, datetime.date)):
to_return[field] = getattr(self, field).isoformat()
else:
to_return[field] = getattr(self, field)
else:
if field == 'data':
# data in attribute is special
continue
raise PyMISPError('The field {} is required in {} when generating a feed.'.format(field, self.__class__.__name__))
return to_return
def to_json(self, sort_keys=False, indent=None): def to_json(self, sort_keys=False, indent=None):
"""Dump recursively any class of type MISPAbstract to a json string""" """Dump recursively any class of type MISPAbstract to a json string"""
return dumps(self, default=pymisp_json_default, sort_keys=sort_keys, indent=indent) return dumps(self, default=pymisp_json_default, sort_keys=sort_keys, indent=indent)
@ -329,7 +355,7 @@ class AbstractMISP(MutableMapping, MISPFileCache):
if isinstance(val, bool): if isinstance(val, bool):
self.__edited = val self.__edited = val
else: else:
raise Exception('edited can only be True or False') raise PyMISPError('edited can only be True or False')
def __setattr__(self, name, value): def __setattr__(self, name, value):
if name[0] != '_' and not self.__edited and name in self.keys(): if name[0] != '_' and not self.__edited and name in self.keys():
@ -340,7 +366,7 @@ class AbstractMISP(MutableMapping, MISPFileCache):
def _datetime_to_timestamp(self, d): def _datetime_to_timestamp(self, d):
"""Convert a datetime.datetime object to a timestamp (int)""" """Convert a datetime.datetime object to a timestamp (int)"""
if isinstance(d, (int, str)) or (sys.version_info < (3, 0) and isinstance(d, unicode)): if isinstance(d, (int, float, str)) or (sys.version_info < (3, 0) and isinstance(d, unicode)):
# Assume we already have a timestamp # Assume we already have a timestamp
return int(d) return int(d)
if sys.version_info >= (3, 3): if sys.version_info >= (3, 3):
@ -393,6 +419,9 @@ class AbstractMISP(MutableMapping, MISPFileCache):
class MISPTag(AbstractMISP): class MISPTag(AbstractMISP):
_fields_for_feed = {'name', 'colour'}
def __init__(self): def __init__(self):
super(MISPTag, self).__init__() super(MISPTag, self).__init__()
@ -400,3 +429,12 @@ class MISPTag(AbstractMISP):
if kwargs.get('Tag'): if kwargs.get('Tag'):
kwargs = kwargs.get('Tag') kwargs = kwargs.get('Tag')
super(MISPTag, self).from_dict(**kwargs) super(MISPTag, self).from_dict(**kwargs)
def _set_default(self):
if not hasattr(self, 'colour'):
self.colour = '#ffffff'
def _to_feed(self):
if hasattr(self, 'exportable') and not self.exportable:
return False
return super(MISPTag, self)._to_feed()

32
pymisp/aping.py
View File

@ -179,6 +179,10 @@ class ExpandedPyMISP(PyMISP):
response = self._prepare_request('POST', f'/servers/restartWorkers') response = self._prepare_request('POST', f'/servers/restartWorkers')
return self._check_response(response, expect_json=True) return self._check_response(response, expect_json=True)
def db_schema_diagnostic(self):
response = self._prepare_request('GET', f'/servers/dbSchemaDiagnostic')
return self._check_response(response, expect_json=True)
def toggle_global_pythonify(self): def toggle_global_pythonify(self):
self.global_pythonify = not self.global_pythonify self.global_pythonify = not self.global_pythonify
@ -620,7 +624,11 @@ class ExpandedPyMISP(PyMISP):
return t return t
def add_tag(self, tag: MISPTag, pythonify: bool=False): def add_tag(self, tag: MISPTag, pythonify: bool=False):
'''Add a new tag on a MISP instance''' '''Add a new tag on a MISP instance
Notes:
* The user calling this method needs the Tag Editor permission
* It doesn't add a tag to an event, simply create it on a MISP instance.
'''
new_tag = self._prepare_request('POST', 'tags/add', data=tag) new_tag = self._prepare_request('POST', 'tags/add', data=tag)
new_tag = self._check_response(new_tag, expect_json=True) new_tag = self._check_response(new_tag, expect_json=True)
if not (self.global_pythonify or pythonify) or 'errors' in new_tag: if not (self.global_pythonify or pythonify) or 'errors' in new_tag:
@ -1891,7 +1899,7 @@ class ExpandedPyMISP(PyMISP):
if adhereToWarninglists in wl_params: if adhereToWarninglists in wl_params:
query['adhereToWarninglists'] = adhereToWarninglists query['adhereToWarninglists'] = adhereToWarninglists
else: else:
raise Exception('Invalid parameter, adhereToWarninglists Can only be {}'.format(', '.join(wl_params))) raise PyMISPError('Invalid parameter, adhereToWarninglists Can only be {}'.format(', '.join(wl_params)))
if distribution is not None: if distribution is not None:
query['distribution'] = distribution query['distribution'] = distribution
if returnMetaAttributes: if returnMetaAttributes:
@ -2023,7 +2031,6 @@ class ExpandedPyMISP(PyMISP):
response = self._prepare_request('POST', f'user_settings/delete', data=query) response = self._prepare_request('POST', f'user_settings/delete', data=query)
return self._check_response(response, expect_json=True) return self._check_response(response, expect_json=True)
# ## END User Settings ### # ## END User Settings ###
# ## BEGIN Global helpers ### # ## BEGIN Global helpers ###
@ -2045,22 +2052,26 @@ class ExpandedPyMISP(PyMISP):
raise PyMISPError('The misp_entity must be MISPEvent, MISPObject or MISPAttribute') raise PyMISPError('The misp_entity must be MISPEvent, MISPObject or MISPAttribute')
def tag(self, misp_entity: Union[AbstractMISP, str], tag: str, local: bool=False): def tag(self, misp_entity: Union[AbstractMISP, str], tag: Union[MISPTag, str], local: bool=False):
"""Tag an event or an attribute. misp_entity can be a UUID""" """Tag an event or an attribute. misp_entity can be a MISPEvent, a MISP Attribute, or a UUID"""
if 'uuid' in misp_entity: if 'uuid' in misp_entity:
uuid = misp_entity.uuid uuid = misp_entity.uuid
else: else:
uuid = misp_entity uuid = misp_entity
if isinstance(tag, MISPTag):
tag = tag.name
to_post = {'uuid': uuid, 'tag': tag, 'local': local} to_post = {'uuid': uuid, 'tag': tag, 'local': local}
response = self._prepare_request('POST', 'tags/attachTagToObject', data=to_post) response = self._prepare_request('POST', 'tags/attachTagToObject', data=to_post)
return self._check_response(response, expect_json=True) return self._check_response(response, expect_json=True)
def untag(self, misp_entity: Union[AbstractMISP, str], tag: str): def untag(self, misp_entity: Union[AbstractMISP, str], tag: Union[MISPTag, str]):
"""Untag an event or an attribute. misp_entity can be a UUID""" """Untag an event or an attribute. misp_entity can be a UUID"""
if 'uuid' in misp_entity: if 'uuid' in misp_entity:
uuid = misp_entity.uuid uuid = misp_entity.uuid
else: else:
uuid = misp_entity uuid = misp_entity
if isinstance(tag, MISPTag):
tag = tag.name
to_post = {'uuid': uuid, 'tag': tag} to_post = {'uuid': uuid, 'tag': tag}
response = self._prepare_request('POST', 'tags/removeTagFromObject', data=to_post) response = self._prepare_request('POST', 'tags/removeTagFromObject', data=to_post)
return self._check_response(response, expect_json=True) return self._check_response(response, expect_json=True)
@ -2188,10 +2199,6 @@ class ExpandedPyMISP(PyMISP):
kw_params: dict={}, output_type: str='json'): kw_params: dict={}, output_type: str='json'):
'''Prepare a request for python-requests''' '''Prepare a request for python-requests'''
url = urljoin(self.root_url, url) url = urljoin(self.root_url, url)
if logger.isEnabledFor(logging.DEBUG):
logger.debug(f'{request_type} - {url}')
if data is not None:
logger.debug(data)
if data: if data:
if not isinstance(data, str): # Else, we already have a text blob to send if not isinstance(data, str): # Else, we already have a text blob to send
if isinstance(data, dict): # Else, we can directly json encode. if isinstance(data, dict): # Else, we can directly json encode.
@ -2199,6 +2206,11 @@ class ExpandedPyMISP(PyMISP):
data = {k: v for k, v in data.items() if v is not None} data = {k: v for k, v in data.items() if v is not None}
data = json.dumps(data, default=pymisp_json_default) data = json.dumps(data, default=pymisp_json_default)
if logger.isEnabledFor(logging.DEBUG):
logger.debug(f'{request_type} - {url}')
if data is not None:
logger.debug(data)
if kw_params: if kw_params:
# CakePHP params in URL # CakePHP params in URL
to_append_url = '/'.join([f'{k}:{v}' for k, v in kw_params.items()]) to_append_url = '/'.join([f'{k}:{v}' for k, v in kw_params.items()])

8
pymisp/data/describeTypes.json
View File

@ -187,7 +187,9 @@
"domain", "domain",
"domain|ip", "domain|ip",
"email-dst", "email-dst",
"email-src",
"email-subject", "email-subject",
"eppn",
"hassh-md5", "hassh-md5",
"hasshserver-md5", "hasshserver-md5",
"hex", "hex",
@ -418,6 +420,7 @@
"comment", "comment",
"email-dst", "email-dst",
"email-src", "email-src",
"eppn",
"github-organisation", "github-organisation",
"github-repository", "github-repository",
"github-username", "github-username",
@ -612,6 +615,10 @@
"default_category": "Payload delivery", "default_category": "Payload delivery",
"to_ids": 0 "to_ids": 0
}, },
"eppn": {
"default_category": "Network activity",
"to_ids": 1
},
"filename": { "filename": {
"default_category": "Payload delivery", "default_category": "Payload delivery",
"to_ids": 1 "to_ids": 1
@ -1135,6 +1142,7 @@
"email-subject", "email-subject",
"email-thread-index", "email-thread-index",
"email-x-mailer", "email-x-mailer",
"eppn",
"filename", "filename",
"filename|authentihash", "filename|authentihash",
"filename|impfuzzy", "filename|impfuzzy",

2
pymisp/data/misp-objects

@ -1 +1 @@
Subproject commit 58d6722f5e276a0ec5889e6e67316b7401960542 Subproject commit 33a7d6b574b7354ba8243ba461bfb30db0528023

201
pymisp/mispevent.py
View File

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import datetime import datetime
@ -11,6 +10,7 @@ import sys
import uuid import uuid
from collections import defaultdict from collections import defaultdict
import logging import logging
import hashlib
from deprecated import deprecated from deprecated import deprecated
@ -99,10 +99,12 @@ def make_bool(value):
return False return False
return True return True
else: else:
raise Exception('Unable to convert {} to a boolean.'.format(value)) raise PyMISPError('Unable to convert {} to a boolean.'.format(value))
class MISPAttribute(AbstractMISP): class MISPAttribute(AbstractMISP):
_fields_for_feed = {'uuid', 'value', 'category', 'type', 'comment', 'data',
'timestamp', 'to_ids', 'disable_correlation'}
def __init__(self, describe_types=None, strict=False): def __init__(self, describe_types=None, strict=False):
"""Represents an Attribute """Represents an Attribute
@ -121,6 +123,39 @@ class MISPAttribute(AbstractMISP):
self.ShadowAttribute = [] self.ShadowAttribute = []
self.Sighting = [] self.Sighting = []
def hash_values(self, algorithm='sha512'):
"""Compute the hash of every values for fast lookups"""
if algorithm not in hashlib.algorithms_available:
raise PyMISPError('The algorithm {} is not available for hashing.'.format(algorithm))
if '|' in self.type or self.type == 'malware-sample':
hashes = []
for v in self.value.split('|'):
h = hashlib.new(algorithm)
h.update(v.encode("utf-8"))
hashes.append(h.hexdigest())
return hashes
else:
h = hashlib.new(algorithm)
to_encode = self.value
if not isinstance(to_encode, str):
to_encode = str(to_encode)
h.update(to_encode.encode("utf-8"))
return [h.hexdigest()]
def _set_default(self):
if not hasattr(self, 'comment'):
self.comment = ''
if not hasattr(self, 'timestamp'):
self.timestamp = datetime.datetime.timestamp(datetime.datetime.now())
def _to_feed(self):
to_return = super(MISPAttribute, self)._to_feed()
if self.data:
to_return['data'] = base64.b64encode(self.data.getvalue()).decode()
if self.tags:
to_return['Tag'] = list(filter(None, [tag._to_feed() for tag in self.tags]))
return to_return
@property @property
def known_types(self): def known_types(self):
"""Returns a list of all the known MISP attributes types""" """Returns a list of all the known MISP attributes types"""
@ -349,7 +384,7 @@ class MISPAttribute(AbstractMISP):
try: try:
with ZipFile(self.data) as f: with ZipFile(self.data) as f:
if not self.__is_misp_encrypted_file(f): if not self.__is_misp_encrypted_file(f):
raise Exception('Not an existing malware sample') raise PyMISPError('Not an existing malware sample')
for name in f.namelist(): for name in f.namelist():
if name.endswith('.filename.txt'): if name.endswith('.filename.txt'):
with f.open(name, pwd=b'infected') as unpacked: with f.open(name, pwd=b'infected') as unpacked:
@ -421,6 +456,9 @@ class MISPAttribute(AbstractMISP):
class MISPEvent(AbstractMISP): class MISPEvent(AbstractMISP):
_fields_for_feed = {'uuid', 'info', 'threat_level_id', 'analysis', 'timestamp',
'publish_timestamp', 'published', 'date', 'extends_uuid'}
def __init__(self, describe_types=None, strict_validation=False, **kwargs): def __init__(self, describe_types=None, strict_validation=False, **kwargs):
super(MISPEvent, self).__init__(**kwargs) super(MISPEvent, self).__init__(**kwargs)
if strict_validation: if strict_validation:
@ -440,6 +478,105 @@ class MISPEvent(AbstractMISP):
self.RelatedEvent = [] self.RelatedEvent = []
self.ShadowAttribute = [] self.ShadowAttribute = []
def _set_default(self):
"""There are a few keys that could, or need to be set by default for the feed generator"""
if not hasattr(self, 'published'):
self.published = True
if not hasattr(self, 'uuid'):
self.uuid = str(uuid.uuid4())
if not hasattr(self, 'extends_uuid'):
self.extends_uuid = ''
if not hasattr(self, 'date'):
self.set_date(datetime.date.today())
if not hasattr(self, 'timestamp'):
self.timestamp = datetime.datetime.timestamp(datetime.datetime.now())
if not hasattr(self, 'publish_timestamp'):
self.publish_timestamp = datetime.datetime.timestamp(datetime.datetime.now())
if not hasattr(self, 'analysis'):
# analysis: 0 means initial, 1 ongoing, 2 completed
self.analysis = 2
if not hasattr(self, 'threat_level_id'):
# threat_level_id 4 means undefined. Tags are recommended.
self.threat_level_id = 4
@property
def manifest(self):
required = ['info', 'Orgc']
for r in required:
if not hasattr(self, r):
raise PyMISPError('The field {} is required to generate the event manifest.')
self._set_default()
return {
self.uuid: {
'Orgc': self.Orgc._to_feed(),
'Tag': list(filter(None, [tag._to_feed() for tag in self.tags])),
'info': self.info,
'date': self.date.isoformat(),
'analysis': self.analysis,
'threat_level_id': self.threat_level_id,
'timestamp': self._datetime_to_timestamp(self.timestamp)
}
}
def attributes_hashes(self, algorithm='sha512'):
to_return = []
for attribute in self.attributes:
to_return += attribute.hash_values(algorithm)
for obj in self.objects:
for attribute in obj.attributes:
to_return += attribute.hash_values(algorithm)
return to_return
def to_feed(self, valid_distributions=[0, 1, 2, 3, 4, 5], with_meta=False):
""" Generate a json output for MISP Feed.
Notes:
* valid_distributions only makes sense if the distribution key is set (i.e. the event is exported from a MISP instance)
"""
required = ['info', 'Orgc']
for r in required:
if not hasattr(self, r):
raise PyMISPError('The field {} is required to generate the event feed output.')
if (hasattr(self, 'distribution')
and self.distribution is not None
and int(self.distribution) not in valid_distributions):
return
to_return = super(MISPEvent, self)._to_feed()
if with_meta:
to_return['_hashes'] = []
to_return['_manifest'] = self.manifest
to_return['Orgc'] = self.Orgc._to_feed()
to_return['Tag'] = list(filter(None, [tag._to_feed() for tag in self.tags]))
if self.attributes:
to_return['Attribute'] = []
for attribute in self.attributes:
if (valid_distributions and attribute.get('distribution') is not None and attribute.distribution not in valid_distributions):
continue
to_return['Attribute'].append(attribute._to_feed())
if with_meta:
to_return['_hashes'] += attribute.hash_values('md5')
if self.objects:
to_return['Object'] = []
for obj in self.objects:
if (valid_distributions and obj.get('distribution') is not None and obj.distribution not in valid_distributions):
continue
obj_to_attach = obj._to_feed()
obj_to_attach['Attribute'] = []
for attribute in obj.attributes:
if (valid_distributions and attribute.get('distribution') is not None and attribute.distribution not in valid_distributions):
continue
obj_to_attach['Attribute'].append(attribute._to_feed())
if with_meta:
to_return['_hashes'] += attribute.hash_values('md5')
to_return['Object'].append(obj_to_attach)
return {'Event': to_return}
@property @property
def known_types(self): def known_types(self):
return self.describe_types['types'] return self.describe_types['types']
@ -496,14 +633,14 @@ class MISPEvent(AbstractMISP):
else: else:
raise PyMISPError('All the attributes have to be of type MISPObject.') raise PyMISPError('All the attributes have to be of type MISPObject.')
def load_file(self, event_path): def load_file(self, event_path, validate=False, metadata_only=False):
"""Load a JSON dump from a file on the disk""" """Load a JSON dump from a file on the disk"""
if not os.path.exists(event_path): if not os.path.exists(event_path):
raise PyMISPError('Invalid path, unable to load the event.') raise PyMISPError('Invalid path, unable to load the event.')
with open(event_path, 'rb') as f: with open(event_path, 'rb') as f:
self.load(f) self.load(f, validate, metadata_only)
def load(self, json_event, validate=False): def load(self, json_event, validate=False, metadata_only=False):
"""Load a JSON dump from a pseudo file or a JSON string""" """Load a JSON dump from a pseudo file or a JSON string"""
if hasattr(json_event, 'read'): if hasattr(json_event, 'read'):
# python2 and python3 compatible to find if we have a file # python2 and python3 compatible to find if we have a file
@ -518,6 +655,9 @@ class MISPEvent(AbstractMISP):
event = json_event event = json_event
if not event: if not event:
raise PyMISPError('Invalid event') raise PyMISPError('Invalid event')
if metadata_only:
event.pop('Attribute', None)
event.pop('Object', None)
self.from_dict(**event) self.from_dict(**event)
if validate: if validate:
jsonschema.validate(json.loads(self.to_json()), self.__json_schema) jsonschema.validate(json.loads(self.to_json()), self.__json_schema)
@ -591,6 +731,11 @@ class MISPEvent(AbstractMISP):
self.publish_timestamp = datetime.datetime.fromtimestamp(int(kwargs.pop('publish_timestamp')), datetime.timezone.utc) self.publish_timestamp = datetime.datetime.fromtimestamp(int(kwargs.pop('publish_timestamp')), datetime.timezone.utc)
else: else:
self.publish_timestamp = datetime.datetime.fromtimestamp(int(kwargs.pop('publish_timestamp')), UTC()) self.publish_timestamp = datetime.datetime.fromtimestamp(int(kwargs.pop('publish_timestamp')), UTC())
if kwargs.get('sighting_timestamp'):
if sys.version_info >= (3, 3):
self.sighting_timestamp = datetime.datetime.fromtimestamp(int(kwargs.pop('sighting_timestamp')), datetime.timezone.utc)
else:
self.sighting_timestamp = datetime.datetime.fromtimestamp(int(kwargs.pop('sighting_timestamp')), UTC())
if kwargs.get('sharing_group_id'): if kwargs.get('sharing_group_id'):
self.sharing_group_id = int(kwargs.pop('sharing_group_id')) self.sharing_group_id = int(kwargs.pop('sharing_group_id'))
if kwargs.get('RelatedEvent'): if kwargs.get('RelatedEvent'):
@ -620,6 +765,8 @@ class MISPEvent(AbstractMISP):
to_return['date'] = self.date.isoformat() to_return['date'] = self.date.isoformat()
if to_return.get('publish_timestamp'): if to_return.get('publish_timestamp'):
to_return['publish_timestamp'] = self._datetime_to_timestamp(self.publish_timestamp) to_return['publish_timestamp'] = self._datetime_to_timestamp(self.publish_timestamp)
if to_return.get('sighting_timestamp'):
to_return['sighting_timestamp'] = self._datetime_to_timestamp(self.sighting_timestamp)
return to_return return to_return
@ -671,7 +818,7 @@ class MISPEvent(AbstractMISP):
attributes.append(a) attributes.append(a)
if not attributes: if not attributes:
raise Exception('No attribute with identifier {} found.'.format(attribute_identifier)) raise PyMISPError('No attribute with identifier {} found.'.format(attribute_identifier))
self.edited = True self.edited = True
return attributes return attributes
@ -693,7 +840,7 @@ class MISPEvent(AbstractMISP):
found = True found = True
break break
if not found: if not found:
raise Exception('No attribute with UUID/ID {} found.'.format(attribute_id)) raise PyMISPError('No attribute with UUID/ID {} found.'.format(attribute_id))
def add_attribute(self, type, value, **kwargs): def add_attribute(self, type, value, **kwargs):
"""Add an attribute. type and value are required but you can pass all """Add an attribute. type and value are required but you can pass all
@ -857,8 +1004,18 @@ class MISPEvent(AbstractMISP):
class MISPObjectReference(AbstractMISP): class MISPObjectReference(AbstractMISP):
_fields_for_feed = {'uuid', 'timestamp', 'relationship_type', 'comment',
'object_uuid', 'referenced_uuid'}
def __init__(self): def __init__(self):
super(MISPObjectReference, self).__init__() super(MISPObjectReference, self).__init__()
self.uuid = str(uuid.uuid4())
def _set_default(self):
if not hasattr(self, 'comment'):
self.comment = ''
if not hasattr(self, 'timestamp'):
self.timestamp = datetime.datetime.timestamp(datetime.datetime.now())
def from_dict(self, **kwargs): def from_dict(self, **kwargs):
if 'ObjectReference' in kwargs: if 'ObjectReference' in kwargs:
@ -891,6 +1048,8 @@ class MISPUser(AbstractMISP):
if 'User' in kwargs: if 'User' in kwargs:
kwargs = kwargs['User'] kwargs = kwargs['User']
super(MISPUser, self).from_dict(**kwargs) super(MISPUser, self).from_dict(**kwargs)
if hasattr(self, 'password') and set(self.password) == set(['*']):
self.password = None
def __repr__(self): def __repr__(self):
if hasattr(self, 'email'): if hasattr(self, 'email'):
@ -900,6 +1059,8 @@ class MISPUser(AbstractMISP):
class MISPOrganisation(AbstractMISP): class MISPOrganisation(AbstractMISP):
_fields_for_feed = {'name', 'uuid'}
def __init__(self): def __init__(self):
super(MISPOrganisation, self).__init__() super(MISPOrganisation, self).__init__()
@ -1056,6 +1217,9 @@ class MISPSighting(AbstractMISP):
class MISPObjectAttribute(MISPAttribute): class MISPObjectAttribute(MISPAttribute):
_fields_for_feed = {'uuid', 'object_relation', 'value', 'category', 'type',
'comment', 'data', 'timestamp', 'to_ids', 'disable_correlation'}
def __init__(self, definition): def __init__(self, definition):
super(MISPObjectAttribute, self).__init__() super(MISPObjectAttribute, self).__init__()
self._definition = definition self._definition = definition
@ -1140,6 +1304,10 @@ class MISPUserSetting(AbstractMISP):
class MISPObject(AbstractMISP): class MISPObject(AbstractMISP):
_fields_for_feed = {'name', 'meta-category', 'description', 'template_uuid',
'template_version', 'uuid', 'timestamp', 'distribution',
'sharing_group_id', 'comment'}
def __init__(self, name, strict=False, standalone=False, default_attributes_parameters={}, **kwargs): def __init__(self, name, strict=False, standalone=False, default_attributes_parameters={}, **kwargs):
''' Master class representing a generic MISP object ''' Master class representing a generic MISP object
:name: Name of the object :name: Name of the object
@ -1203,6 +1371,18 @@ class MISPObject(AbstractMISP):
self.template_version = self._definition['version'] self.template_version = self._definition['version']
return True return True
def _set_default(self):
if not hasattr(self, 'comment'):
self.comment = ''
if not hasattr(self, 'timestamp'):
self.timestamp = datetime.datetime.timestamp(datetime.datetime.now())
def _to_feed(self):
to_return = super(MISPObject, self)._to_feed()
if self.references:
to_return['ObjectReference'] = [reference._to_feed() for reference in self.references]
return to_return
def force_misp_objects_path_custom(self, misp_objects_path_custom, object_name=None): def force_misp_objects_path_custom(self, misp_objects_path_custom, object_name=None):
if object_name: if object_name:
self.name = object_name self.name = object_name
@ -1309,6 +1489,7 @@ class MISPObject(AbstractMISP):
relationship_type=relationship_type, comment=comment, **kwargs) relationship_type=relationship_type, comment=comment, **kwargs)
self.ObjectReference.append(reference) self.ObjectReference.append(reference)
self.edited = True self.edited = True
return reference
def get_attributes_by_relation(self, object_relation): def get_attributes_by_relation(self, object_relation):
'''Returns the list of attributes with the given object relation in the object''' '''Returns the list of attributes with the given object relation in the object'''
@ -1370,10 +1551,10 @@ class MISPObject(AbstractMISP):
self._validate() self._validate()
return super(MISPObject, self).to_dict() return super(MISPObject, self).to_dict()
def to_json(self, strict=False): def to_json(self, strict=False, sort_keys=False, indent=None):
if strict or self._strict and self._known_template: if strict or self._strict and self._known_template:
self._validate() self._validate()
return super(MISPObject, self).to_json() return super(MISPObject, self).to_json(sort_keys=sort_keys, indent=indent)
def _validate(self): def _validate(self):
"""Make sure the object we're creating has the required fields""" """Make sure the object we're creating has the required fields"""

6
pymisp/tools/__init__.py
View File

@ -21,3 +21,9 @@ if sys.version_info >= (3, 6):
from .vehicleobject import VehicleObject # noqa from .vehicleobject import VehicleObject # noqa
from .csvloader import CSVLoader # noqa from .csvloader import CSVLoader # noqa
from .sshauthkeyobject import SSHAuthorizedKeysObject # noqa from .sshauthkeyobject import SSHAuthorizedKeysObject # noqa
from .feed import feed_meta_generator # noqa
try:
from .urlobject import URLObject # noqa
except ImportError:
# Requires faup, which is a bit difficult to install
pass

13
pymisp/tools/emailobject.py
View File

@ -50,17 +50,30 @@ class EMailObject(AbstractMISPObjectGenerator):
if 'Message-ID' in self.__email: if 'Message-ID' in self.__email:
self.add_attribute('message-id', value=self.__email['Message-ID']) self.add_attribute('message-id', value=self.__email['Message-ID'])
if 'To' in self.__email: if 'To' in self.__email:
# TODO: split name and email address
to_add = [to.strip() for to in self.__email['To'].split(',')] to_add = [to.strip() for to in self.__email['To'].split(',')]
self.add_attributes('to', *to_add) self.add_attributes('to', *to_add)
if 'Cc' in self.__email: if 'Cc' in self.__email:
# TODO: split name and email address
to_add = [to.strip() for to in self.__email['Cc'].split(',')] to_add = [to.strip() for to in self.__email['Cc'].split(',')]
self.add_attributes('cc', *to_add) self.add_attributes('cc', *to_add)
if 'Subject' in self.__email: if 'Subject' in self.__email:
self.add_attribute('subject', value=self.__email['Subject']) self.add_attribute('subject', value=self.__email['Subject'])
if 'From' in self.__email: if 'From' in self.__email:
# TODO: split name and email address
to_add = [to.strip() for to in self.__email['From'].split(',')] to_add = [to.strip() for to in self.__email['From'].split(',')]
self.add_attributes('from', *to_add) self.add_attributes('from', *to_add)
if 'Return-Path' in self.__email: if 'Return-Path' in self.__email:
# TODO: split name and email address
self.add_attribute('return-path', value=self.__email['Return-Path']) self.add_attribute('return-path', value=self.__email['Return-Path'])
if 'User-Agent' in self.__email: if 'User-Agent' in self.__email:
self.add_attribute('user-agent', value=self.__email['User-Agent']) self.add_attribute('user-agent', value=self.__email['User-Agent'])
if self.__email.get_boundary():
self.add_attribute('mime-boundary', value=self.__email.get_boundary())
if 'X-Mailer' in self.__email:
self.add_attribute('x-mailer', value=self.__email['X-Mailer'])
if 'Thread-Index' in self.__email:
self.add_attribute('thread-index', value=self.__email['Thread-Index'])
# TODO: email-header: all headers in one bloc
# TODO: BCC?
# TODO: received headers sometimes have TO email addresses

26
pymisp/tools/feed.py Normal file
View File

@ -0,0 +1,26 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from pathlib import Path
from pymisp import MISPEvent
import json
def feed_meta_generator(path: Path):
manifests = {}
hashes = []
for f_name in path.glob('*.json'):
if str(f_name.name) == 'manifest.json':
continue
event = MISPEvent()
event.load_file(str(f_name))
manifests.update(event.manifest)
hashes += [f'{h},{event.uuid}' for h in event.attributes_hashes('md5')]
with (path / 'manifest.json').open('w') as f:
json.dump(manifests, f)
with (path / 'hashes.csv').open('w') as f:
for h in hashes:
f.write(f'{h}\n')

28
pymisp/tools/urlobject.py Normal file
View File

@ -0,0 +1,28 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from .abstractgenerator import AbstractMISPObjectGenerator
import logging
from pyfaup.faup import Faup
from urllib.parse import unquote_plus
logger = logging.getLogger('pymisp')
faup = Faup()
class URLObject(AbstractMISPObjectGenerator):
def __init__(self, url, standalone=True, **kwargs):
# PY3 way:
# super().__init__('file')
super(URLObject, self).__init__('url', standalone=standalone, **kwargs)
faup.decode(unquote_plus(url))
self.generate_attributes()
def generate_attributes(self):
self.add_attribute('url', value=faup.url.decode())
if faup.get_host():
self.add_attribute('host', value=faup.get_host())
if faup.get_domain():
self.add_attribute('domain', value=faup.get_domain())

2
setup.py
View File

@ -42,7 +42,7 @@ setup(
install_requires=['six', 'requests', 'python-dateutil', 'jsonschema', install_requires=['six', 'requests', 'python-dateutil', 'jsonschema',
'python-dateutil', 'enum34;python_version<"3.4"', 'python-dateutil', 'enum34;python_version<"3.4"',
'functools32;python_version<"3.0"', 'deprecated', 'cachetools;python_version<"3.0"'], 'functools32;python_version<"3.0"', 'deprecated', 'cachetools;python_version<"3.0"'],
extras_require={'fileobjects': ['lief>=0.8,<0.10;python_version<"3.5"', 'lief>=0.10.0.dev0;python_version>"3.5"', 'python-magic', 'pydeep'], extras_require={'fileobjects': ['lief>=0.8,<0.10;python_version<"3.5"', 'lief>=0.10.1;python_version>"3.5"', 'python-magic', 'pydeep'],
'neo': ['py2neo'], 'neo': ['py2neo'],
'openioc': ['beautifulsoup4'], 'openioc': ['beautifulsoup4'],
'virustotal': ['validators'], 'virustotal': ['validators'],

4
tests/mispevent_testfiles/event_obj_attr_tag.json
View File

@ -30,7 +30,7 @@
"name": "file", "name": "file",
"sharing_group_id": "0", "sharing_group_id": "0",
"template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215",
"template_version": "17", "template_version": "18",
"uuid": "a" "uuid": "a"
}, },
{ {
@ -50,7 +50,7 @@
"name": "url", "name": "url",
"sharing_group_id": "0", "sharing_group_id": "0",
"template_uuid": "60efb77b-40b5-4c46-871b-ed1ed999fce5", "template_uuid": "60efb77b-40b5-4c46-871b-ed1ed999fce5",
"template_version": "7", "template_version": "8",
"uuid": "b" "uuid": "b"
} }
] ]

4
tests/mispevent_testfiles/event_obj_def_param.json
View File

@ -30,7 +30,7 @@
"name": "file", "name": "file",
"sharing_group_id": "0", "sharing_group_id": "0",
"template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215",
"template_version": "17", "template_version": "18",
"uuid": "a" "uuid": "a"
}, },
{ {
@ -55,7 +55,7 @@
"name": "file", "name": "file",
"sharing_group_id": "0", "sharing_group_id": "0",
"template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215",
"template_version": "17", "template_version": "18",
"uuid": "b" "uuid": "b"
} }
] ]

6
tests/mispevent_testfiles/overwrite_file/definition.json
View File

@ -449,9 +449,9 @@
] ]
} }
}, },
"version": 17, "version": 1,
"description": "File object describing a file with meta-information", "description": "File object describing a file with meta-information",
"meta-category": "file", "meta-category": "file",
"uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", "uuid": "688c46fb-5edb-40a3-8273-1af7923e0000",
"name": "file" "name": "overwrite_file"
} }

26
tests/test_mispevent.py
View File

@ -5,6 +5,7 @@ import unittest
import json import json
import sys import sys
from io import BytesIO from io import BytesIO
import glob
from pymisp import MISPEvent, MISPSighting, MISPTag from pymisp import MISPEvent, MISPSighting, MISPTag
from pymisp.exceptions import InvalidMISPObject from pymisp.exceptions import InvalidMISPObject
@ -86,7 +87,8 @@ class TestMISPEvent(unittest.TestCase):
del a.uuid del a.uuid
self.mispevent.objects[0].uuid = 'a' self.mispevent.objects[0].uuid = 'a'
self.mispevent.objects[1].uuid = 'b' self.mispevent.objects[1].uuid = 'b'
self.mispevent.objects[0].add_reference(self.mispevent.objects[1], 'baz', comment='foo') reference = self.mispevent.objects[0].add_reference(self.mispevent.objects[1], 'baz', comment='foo')
del reference.uuid
self.assertEqual(self.mispevent.objects[0].references[0].relationship_type, 'baz') self.assertEqual(self.mispevent.objects[0].references[0].relationship_type, 'baz')
with open('tests/mispevent_testfiles/event_obj_attr_tag.json', 'r') as f: with open('tests/mispevent_testfiles/event_obj_attr_tag.json', 'r') as f:
ref_json = json.load(f) ref_json = json.load(f)
@ -291,6 +293,28 @@ class TestMISPEvent(unittest.TestCase):
ref_json = json.load(f) ref_json = json.load(f)
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2))
'''
# Reenable that the 1st of jan 2020.
@unittest.skipIf(sys.version_info < (3, 6), 'Not supported on python < 3.6')
def test_object_templates(self):
me = MISPEvent()
for template in glob.glob(str(me.misp_objects_path / '*' / 'definition.json')):
with open(template) as f:
t_json = json.load(f)
if 'requiredOneOf' in t_json:
obj_relations = set(t_json['attributes'].keys())
subset = set(t_json['requiredOneOf']).issubset(obj_relations)
self.assertTrue(subset, f'{t_json["name"]}')
if 'required' in t_json:
obj_relations = set(t_json['attributes'].keys())
subset = set(t_json['required']).issubset(obj_relations)
self.assertTrue(subset, f'{t_json["name"]}')
for obj_relation, entry in t_json['attributes'].items():
self.assertTrue(entry['misp-attribute'] in me.describe_types['types'])
if 'categories' in entry:
subset = set(entry['categories']).issubset(me.describe_types['categories'])
self.assertTrue(subset, f'{t_json["name"]} - {obj_relation}')
'''
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()

53
tests/testlive_comprehensive.py
View File

@ -23,6 +23,8 @@ from collections import defaultdict
import logging import logging
logging.disable(logging.CRITICAL) logging.disable(logging.CRITICAL)
logger = logging.getLogger('pymisp')
try: try:
from pymisp import ExpandedPyMISP, MISPEvent, MISPOrganisation, MISPUser, Distribution, ThreatLevel, Analysis, MISPObject, MISPAttribute, MISPSighting, MISPShadowAttribute, MISPTag, MISPSharingGroup, MISPFeed, MISPServer, MISPUserSetting from pymisp import ExpandedPyMISP, MISPEvent, MISPOrganisation, MISPUser, Distribution, ThreatLevel, Analysis, MISPObject, MISPAttribute, MISPSighting, MISPShadowAttribute, MISPTag, MISPSharingGroup, MISPFeed, MISPServer, MISPUserSetting
@ -75,7 +77,7 @@ class TestComprehensive(unittest.TestCase):
user.email = 'testusr@user.local' user.email = 'testusr@user.local'
user.org_id = cls.test_org.id user.org_id = cls.test_org.id
cls.test_usr = cls.admin_misp_connector.add_user(user, pythonify=True) cls.test_usr = cls.admin_misp_connector.add_user(user, pythonify=True)
cls.user_misp_connector = ExpandedPyMISP(url, cls.test_usr.authkey, verifycert, debug=False) cls.user_misp_connector = ExpandedPyMISP(url, cls.test_usr.authkey, verifycert, debug=True)
cls.user_misp_connector.toggle_global_pythonify() cls.user_misp_connector.toggle_global_pythonify()
# Creates a publisher # Creates a publisher
user = MISPUser() user = MISPUser()
@ -839,6 +841,7 @@ class TestComprehensive(unittest.TestCase):
second = self.user_misp_connector.add_event(second) second = self.user_misp_connector.add_event(second)
current_ts = int(time.time()) current_ts = int(time.time())
time.sleep(5)
r = self.user_misp_connector.add_sighting({'value': first.attributes[0].value}) r = self.user_misp_connector.add_sighting({'value': first.attributes[0].value})
self.assertEqual(int(r.attribute_id), first.attributes[0].id) self.assertEqual(int(r.attribute_id), first.attributes[0].id)
@ -1072,8 +1075,24 @@ class TestComprehensive(unittest.TestCase):
file_object = first.get_objects_by_name('file')[0] file_object = first.get_objects_by_name('file')[0]
file_object.force_misp_objects_path_custom('tests/mispevent_testfiles', 'overwrite_file') file_object.force_misp_objects_path_custom('tests/mispevent_testfiles', 'overwrite_file')
file_object.add_attribute('test_overwrite', 'blah') file_object.add_attribute('test_overwrite', 'blah')
obj = self.admin_misp_connector.update_object(file_object, pythonify=True) obj_json = self.admin_misp_connector.update_object(file_object)
self.assertTrue('Object' in obj_json, obj_json)
self.assertTrue('name' in obj_json['Object'], obj_json)
obj = MISPObject(obj_json['Object']['name'])
obj.from_dict(**obj_json)
self.assertEqual(obj.get_attributes_by_relation('test_overwrite')[0].value, 'blah') self.assertEqual(obj.get_attributes_by_relation('test_overwrite')[0].value, 'blah')
# FULL object add & update with custom template
new_object = MISPObject('overwrite_file', misp_objects_path_custom='tests/mispevent_testfiles')
new_object.add_attribute('test_overwrite', 'barbaz')
new_object.add_attribute('filename', 'barbaz.exe')
new_object = self.admin_misp_connector.add_object(first, new_object, pythonify=True)
self.assertEqual(new_object.get_attributes_by_relation('test_overwrite')[0].value, 'barbaz', new_object)
new_object.force_misp_objects_path_custom('tests/mispevent_testfiles', 'overwrite_file')
new_object.add_attribute('filename', 'foobar.exe')
new_object = self.admin_misp_connector.update_object(new_object, pythonify=True)
self.assertEqual(new_object.get_attributes_by_relation('filename')[1].value, 'foobar.exe', new_object)
finally: finally:
# Delete event # Delete event
self.admin_misp_connector.delete_event(first) self.admin_misp_connector.delete_event(first)
@ -1166,12 +1185,34 @@ class TestComprehensive(unittest.TestCase):
self.assertFalse(non_exportable_tag.exportable) self.assertFalse(non_exportable_tag.exportable)
first = self.create_simple_event() first = self.create_simple_event()
first.attributes[0].add_tag('non-exportable tag') first.attributes[0].add_tag('non-exportable tag')
# Add tag restricted to an org
tag = MISPTag()
tag.name = f'restricted to org {self.test_org.id}'
tag.org_id = self.test_org.id
tag_org_restricted = self.admin_misp_connector.add_tag(tag, pythonify=True)
self.assertEqual(tag_org_restricted.org_id, tag.org_id)
# Add tag restricted to a user
tag.name = f'restricted to user {self.test_usr.id}'
tag.user_id = self.test_usr.id
tag_user_restricted = self.admin_misp_connector.add_tag(tag, pythonify=True)
self.assertEqual(tag_user_restricted.user_id, tag.user_id)
try: try:
first = self.user_misp_connector.add_event(first) first = self.user_misp_connector.add_event(first)
self.assertFalse(first.attributes[0].tags) self.assertFalse(first.attributes[0].tags)
first = self.admin_misp_connector.get_event(first, pythonify=True) first = self.admin_misp_connector.get_event(first, pythonify=True)
# Reference: https://github.com/MISP/MISP/issues/1394 # Reference: https://github.com/MISP/MISP/issues/1394
self.assertFalse(first.attributes[0].tags) self.assertFalse(first.attributes[0].tags)
# Reference: https://github.com/MISP/PyMISP/issues/483
r = self.delegate_user_misp_connector.tag(first, tag_org_restricted)
# FIXME: The error message changed and is unhelpful.
# self.assertEqual(r['errors'][1]['message'], 'Invalid Tag. This tag can only be set by a fixed organisation.')
self.assertEqual(r['errors'][1]['message'], 'Invalid Target.')
r = self.user_misp_connector.tag(first, tag_org_restricted)
self.assertEqual(r['name'], f'Global tag {tag_org_restricted.name}({tag_org_restricted.id}) successfully attached to Event({first.id}).')
r = self.pub_misp_connector.tag(first.attributes[0], tag_user_restricted)
self.assertEqual(r['errors'][1]['message'], 'Invalid Tag. This tag can only be set by a fixed user.')
r = self.user_misp_connector.tag(first.attributes[0], tag_user_restricted)
self.assertEqual(r['name'], f'Global tag {tag_user_restricted.name}({tag_user_restricted.id}) successfully attached to Attribute({first.attributes[0].id}).')
finally: finally:
# Delete event # Delete event
self.admin_misp_connector.delete_event(first) self.admin_misp_connector.delete_event(first)
@ -1181,6 +1222,8 @@ class TestComprehensive(unittest.TestCase):
self.assertEqual(response['message'], 'Tag deleted.') self.assertEqual(response['message'], 'Tag deleted.')
response = self.admin_misp_connector.delete_tag(non_exportable_tag) response = self.admin_misp_connector.delete_tag(non_exportable_tag)
self.assertEqual(response['message'], 'Tag deleted.') self.assertEqual(response['message'], 'Tag deleted.')
response = self.admin_misp_connector.delete_tag(tag_org_restricted)
response = self.admin_misp_connector.delete_tag(tag_user_restricted)
def test_add_event_with_attachment_object_controller(self): def test_add_event_with_attachment_object_controller(self):
first = self.create_simple_event() first = self.create_simple_event()
@ -1561,6 +1604,10 @@ class TestComprehensive(unittest.TestCase):
self.assertEqual(entry.action, 'edit') self.assertEqual(entry.action, 'edit')
r = self.admin_misp_connector.update_user({'email': 'testusr@user.local'}, self.test_usr) r = self.admin_misp_connector.update_user({'email': 'testusr@user.local'}, self.test_usr)
def test_db_schema(self):
diag = self.admin_misp_connector.db_schema_diagnostic()
self.assertEqual(diag['actual_db_version'], diag['expected_db_version'], diag)
def test_live_acl(self): def test_live_acl(self):
missing_acls = self.admin_misp_connector.remote_acl() missing_acls = self.admin_misp_connector.remote_acl()
self.assertEqual(missing_acls, [], msg=missing_acls) self.assertEqual(missing_acls, [], msg=missing_acls)
@ -1757,7 +1804,7 @@ class TestComprehensive(unittest.TestCase):
break break
# Get # Get
botvrij = self.admin_misp_connector.get_feed(feed, pythonify=True) botvrij = self.admin_misp_connector.get_feed(feed, pythonify=True)
self.assertEqual(botvrij.url, "http://www.botvrij.eu/data/feed-osint") self.assertEqual(botvrij.url, "https://www.botvrij.eu/data/feed-osint")
# Enable # Enable
# MISP OSINT # MISP OSINT
feed = self.admin_misp_connector.enable_feed(feeds[0].id, pythonify=True) feed = self.admin_misp_connector.enable_feed(feeds[0].id, pythonify=True)