mirror of https://github.com/MISP/PyMISP
Merge branch 'master' of https://github.com/MISP/PyMISP
Koen Van Impe
commit
f883161fd2
103
CHANGELOG.txt
103
CHANGELOG.txt
|
|
@ -2,9 +2,112 @@ Changelog
|
|||
=========
|
||||
|
||||
|
||||
v2.4.119.1 (2019-12-17)
|
||||
-----------------------
|
||||
|
||||
New
|
||||
~~~
|
||||
- URLObject (requires pyfaup) [Raphaël Vinot]
|
||||
|
||||
Changes
|
||||
~~~~~~~
|
||||
- Version bump. [Raphaël Vinot]
|
||||
- Bump test files. [Raphaël Vinot]
|
||||
- Bump misp-objects. [Raphaël Vinot]
|
||||
- Debug travis error message. [Raphaël Vinot]
|
||||
- [types] eppn type added. [Alexandre Dulaunoy]
|
||||
- Fix typo. [Raphaël Vinot]
|
||||
- Move scrippsco2 feed generator to a sub directory. [Raphaël Vinot]
|
||||
- Update documentation. [Raphaël Vinot]
|
||||
|
||||
Fix #396
|
||||
- Bump objects. [Raphaël Vinot]
|
||||
|
||||
Fix
|
||||
~~~
|
||||
- Properly test custom objects. [Raphaël Vinot]
|
||||
- Adding a sighting takes a little bit of time. [Raphaël Vinot]
|
||||
- Test case on reference. [Raphaël Vinot]
|
||||
- Add missing fields to event & attribute for the feed output. [Raphaël
|
||||
Vinot]
|
||||
- Make sure the publish timestamp is bumped on update. [Raphaël Vinot]
|
||||
|
||||
|
||||
v2.4.119 (2019-12-02)
|
||||
---------------------
|
||||
|
||||
Changes
|
||||
~~~~~~~
|
||||
- Bump changelog. [Raphaël Vinot]
|
||||
- Bump version. [Raphaël Vinot]
|
||||
- Bump dependencies. [Raphaël Vinot]
|
||||
|
||||
Fix
|
||||
~~~
|
||||
- Bump lief to 0.10.1. [Raphaël Vinot]
|
||||
- Update tests. [Raphaël Vinot]
|
||||
- Raise PyMISPError instead of Exception. [Raphaël Vinot]
|
||||
- Rename feed_meta_generator so it clearly fails with python<3.6.
|
||||
[Raphaël Vinot]
|
||||
|
||||
|
||||
v2.4.117.3 (2019-11-25)
|
||||
-----------------------
|
||||
|
||||
New
|
||||
~~~
|
||||
- Script to generate the metadata of a feed out of a directory. [Raphaël
|
||||
Vinot]
|
||||
- Add to_feed export to MISPEvent. [Raphaël Vinot]
|
||||
- Validate object templates. [Raphaël Vinot]
|
||||
|
||||
fix https://github.com/MISP/misp-objects/issues/199
|
||||
- Test cases for restricted tags. [Raphaël Vinot]
|
||||
|
||||
Fix #483
|
||||
- Get Database Schema Diagnostic. [Raphaël Vinot]
|
||||
|
||||
Fix #492
|
||||
|
||||
Changes
|
||||
~~~~~~~
|
||||
- Bump changelog. [Raphaël Vinot]
|
||||
- Bump version. [Raphaël Vinot]
|
||||
- Bump dependencies. [Raphaël Vinot]
|
||||
- Require stable version of lief again. [Raphaël Vinot]
|
||||
- Few more improvements on the feed export. [Raphaël Vinot]
|
||||
- Bump misp-objects. [Raphaël Vinot]
|
||||
- Make the feed generator more generic. [Raphaël Vinot]
|
||||
- Use New version of PyMISP in the feed generator. [Raphaël Vinot]
|
||||
- Bump misp-object. [Raphaël Vinot]
|
||||
- Allow to sort and indent the json output for objects. [Raphaël Vinot]
|
||||
- Bump objects. [Raphaël Vinot]
|
||||
- Bump dependencies. [Raphaël Vinot]
|
||||
- [test] feed test updated as botvrij is now TLS by default. [Alexandre
|
||||
Dulaunoy]
|
||||
|
||||
Fix
|
||||
~~~
|
||||
- Improve stability of feed output. [Raphaël Vinot]
|
||||
- Do not unitialize the uuid in MISPEvent. [Raphaël Vinot]
|
||||
- Bump url template version in test cases. [Raphaël Vinot]
|
||||
- Python 2.7 tests. [Raphaël Vinot]
|
||||
- Print the full json blob in debug mode. [Raphaël Vinot]
|
||||
|
||||
Related https://github.com/MISP/PyMISP/issues/462
|
||||
|
||||
Other
|
||||
~~~~~
|
||||
- Cch: Bump misp-objects. [Raphaël Vinot]
|
||||
|
||||
|
||||
v2.4.117.2 (2019-10-30)
|
||||
-----------------------
|
||||
|
||||
Changes
|
||||
~~~~~~~
|
||||
- Bump changelog. [Raphaël Vinot]
|
||||
|
||||
Fix
|
||||
~~~
|
||||
- Avoid exception on legacy MISP. [Raphaël Vinot]
|
||||
|
|
|
|||
|
|
@ -18,24 +18,25 @@
|
|||
"default": {
|
||||
"attrs": {
|
||||
"hashes": [
|
||||
"sha256:ec20e7a4825331c1b5ebf261d111e16fa9612c1f7a5e1f884f12bd53a664dfd2",
|
||||
"sha256:f913492e1663d3c36f502e5e9ba6cd13cf19d7fab50aa13239e420fef95e1396"
|
||||
"sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c",
|
||||
"sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72"
|
||||
],
|
||||
"version": "==19.2.0"
|
||||
"version": "==19.3.0"
|
||||
},
|
||||
"beautifulsoup4": {
|
||||
"hashes": [
|
||||
"sha256:5279c36b4b2ec2cb4298d723791467e3000e5384a43ea0cdf5d45207c7e97169",
|
||||
"sha256:6135db2ba678168c07950f9a16c4031822c6f4aec75a65e0a97bc5ca09789931",
|
||||
"sha256:dcdef580e18a76d54002088602eba453eec38ebbcafafeaabd8cab12b6155d57"
|
||||
],
|
||||
"version": "==4.8.1"
|
||||
},
|
||||
"certifi": {
|
||||
"hashes": [
|
||||
"sha256:e4f3620cfea4f83eedc95b24abd9cd56f3c4b146dd0177e83a21b4eb49e21e50",
|
||||
"sha256:fd7c7c74727ddcf00e9acd26bba8da604ffec95bf1c2144e67aff7a8b50e6cef"
|
||||
"sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3",
|
||||
"sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f"
|
||||
],
|
||||
"version": "==2019.9.11"
|
||||
"version": "==2019.11.28"
|
||||
},
|
||||
"chardet": {
|
||||
"hashes": [
|
||||
|
|
@ -46,17 +47,17 @@
|
|||
},
|
||||
"decorator": {
|
||||
"hashes": [
|
||||
"sha256:86156361c50488b84a3f148056ea716ca587df2f0de1d34750d35c21312725de",
|
||||
"sha256:f069f3a01830ca754ba5258fde2278454a0b5b79e0d7f5c13b3b97e57d4acff6"
|
||||
"sha256:54c38050039232e1db4ad7375cfce6748d7b41c29e95a081c8a6d2c30364a2ce",
|
||||
"sha256:5d19b92a3c8f7f101c8dd86afd86b0f061a8ce4540ab8cd401fa2542756bce6d"
|
||||
],
|
||||
"version": "==4.4.0"
|
||||
"version": "==4.4.1"
|
||||
},
|
||||
"deprecated": {
|
||||
"hashes": [
|
||||
"sha256:a515c4cf75061552e0284d123c3066fbbe398952c87333a92b8fc3dd8e4f9cc1",
|
||||
"sha256:b07b414c8aac88f60c1d837d21def7e83ba711052e03b3cbaff27972567a8f8d"
|
||||
"sha256:408038ab5fdeca67554e8f6742d1521cd3cd0ee0ff9d47f29318a4f4da31c308",
|
||||
"sha256:8b6a5aa50e482d8244a62e5582b96c372e87e3a28e8b49c316e46b95c76a611d"
|
||||
],
|
||||
"version": "==1.2.6"
|
||||
"version": "==1.2.7"
|
||||
},
|
||||
"idna": {
|
||||
"hashes": [
|
||||
|
|
@ -65,60 +66,81 @@
|
|||
],
|
||||
"version": "==2.8"
|
||||
},
|
||||
"importlib-metadata": {
|
||||
"hashes": [
|
||||
"sha256:b044f07694ef14a6683b097ba56bd081dbc7cdc7c7fe46011e499dfecc082f21",
|
||||
"sha256:e6ac600a142cf2db707b1998382cc7fc3b02befb7273876e01b8ad10b9652742"
|
||||
],
|
||||
"markers": "python_version < '3.8'",
|
||||
"version": "==1.1.0"
|
||||
},
|
||||
"jsonschema": {
|
||||
"hashes": [
|
||||
"sha256:5f9c0a719ca2ce14c5de2fd350a64fd2d13e8539db29836a86adc990bb1a068f",
|
||||
"sha256:8d4a2b7b6c2237e0199c8ea1a6d3e05bf118e289ae2b9d7ba444182a2959560d"
|
||||
"sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163",
|
||||
"sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"
|
||||
],
|
||||
"version": "==3.0.2"
|
||||
"version": "==3.2.0"
|
||||
},
|
||||
"lief": {
|
||||
"hashes": [
|
||||
"sha256:0efba18d7b9776529ea5c18c771b35871896a8ceb95a19351e50d4813a11c632",
|
||||
"sha256:3d9c7bb1e353e875f295a72a58d3a37ae1ba3e1ff1beb57b8a65f1a726064093",
|
||||
"sha256:3db5939e7d95f776f9866586128c2a5be614eaec43ab985ac27ff2c531f8ac5f",
|
||||
"sha256:4c61598818b0091d80839875aa107cfd10ae1017a3e9c9de4bc002622b8e3179",
|
||||
"sha256:4f26d07bdada8ca5ef3dc5fa2f71f20f7e8ab4f78f7c5e00134477f51feb6a80",
|
||||
"sha256:55fe3c8a0990dce16ab5bf88df707f1eacac4eb34561667ac478497e0e0807c7",
|
||||
"sha256:68bcf18e40c9412d2d08d6311e04eb6c19e20ec174764706da2d602c45aa4fd5",
|
||||
"sha256:7ff910d99361022451e9c25e34cb844768e2fa347cfb0f4ad70f531810d776d4",
|
||||
"sha256:ac571152d0b864e8d376bc733c5728a224316be1cdefc290174f1bf8ab10ec70",
|
||||
"sha256:dd17a7cdcd29a2efca3d4cb4fb078a06daf1cafec8912560965a8d8dbf346739",
|
||||
"sha256:efa5f3523c01f7f0f5f2c14e5ac808e2447d1435c6a2872e5ab1a97ef1b0db9b",
|
||||
"sha256:f1aadb344b5e14b308167bd2c9f31f1915e3c4e3f9a9ca92ff7b7bfbede5034c"
|
||||
"sha256:276cc63ec12a21bdf01b8d30962692c17499788234f0765247ca7a35872097ec",
|
||||
"sha256:3e6baaeb52bdc339b5f19688b58fd8d5778b92e50221f920cedfa2bec1f4d5c2",
|
||||
"sha256:45e5c592b57168c447698381d927eb2386ffdd52afe0c48245f848d4cc7ee05a",
|
||||
"sha256:6547752b5db105cd41c9fa65d0d7452a4d7541b77ffee716b46246c6d81e172f",
|
||||
"sha256:83b51e01627b5982662f9550ac1230758aa56945ed86829e4291932d98417da3",
|
||||
"sha256:895599194ea7495bf304e39317b04df20cccf799fc2751867cc1aa4997cfcdae",
|
||||
"sha256:8a91cee2568306fe1d2bf84341b459c85368317d01d7105fa49e4f4ede837076",
|
||||
"sha256:913b36a67707dc2afa72f117bab9856ea3f434f332b04a002a0f9723c8779320",
|
||||
"sha256:9f604a361a3b1b3ed5fdafed0321c5956cb3b265b5efe2250d1bf8911a80c65b",
|
||||
"sha256:a487fe7234c04bccd58223dbb79214421176e2629814c7a4a887764cceb5be7c",
|
||||
"sha256:bc8488fb0661cb436fe4bb4fe947d0f9aa020e9acaed233ccf01ab04d888c68a",
|
||||
"sha256:bddbf333af62310a10cb738a1df1dc2b140dd9c663b55ba3500c10c249d416d2",
|
||||
"sha256:cce48d7c97cef85e01e6cfeff55f2068956b5c0257eb9c2d2c6d15e33dd1e4fc",
|
||||
"sha256:f8b3f66956c56b582b3adc573bf2a938c25fb21c8894b373a113e24c494fc982"
|
||||
],
|
||||
"version": "==0.10.0.dev0"
|
||||
"version": "==0.10.1"
|
||||
},
|
||||
"more-itertools": {
|
||||
"hashes": [
|
||||
"sha256:53ff73f186307d9c8ef17a9600309154a6ae27f25579e80af4db8f047ba14bc2",
|
||||
"sha256:a0ea684c39bc4315ba7aae406596ef191fd84f873d2d2751f84d64e81a7a2d45"
|
||||
],
|
||||
"version": "==8.0.0"
|
||||
},
|
||||
"pillow": {
|
||||
"hashes": [
|
||||
"sha256:00fdeb23820f30e43bba78eb9abb00b7a937a655de7760b2e09101d63708b64e",
|
||||
"sha256:01f948e8220c85eae1aa1a7f8edddcec193918f933fb07aaebe0bfbbcffefbf1",
|
||||
"sha256:08abf39948d4b5017a137be58f1a52b7101700431f0777bec3d897c3949f74e6",
|
||||
"sha256:099a61618b145ecb50c6f279666bbc398e189b8bc97544ae32b8fcb49ad6b830",
|
||||
"sha256:2c1c61546e73de62747e65807d2cc4980c395d4c5600ecb1f47a650c6fa78c79",
|
||||
"sha256:2ed9c4f694861642401f27dc3cb99772be67cd190e84845c749dae0a06c3bfae",
|
||||
"sha256:338581b30b908e111be578f0297255f6b57a51358cd16fa0e6f664c9a1f88bff",
|
||||
"sha256:38c7d48a21cd06fdeee93987147b9b1c55b73b4cfcbf83240568bfbd5adee447",
|
||||
"sha256:43fd026f613c8e48a25eba1a92f4d2ad7f3903c95d8c33a11611a7717d2ab654",
|
||||
"sha256:4548236844327a718ce3bb182ab32a16fa2050c61e334e959f554cac052fb0df",
|
||||
"sha256:5090857876c58885cfa388dc649e5db30aae98a068c26f3fd0ac9d7d9a4d9572",
|
||||
"sha256:5bbba34f97a26a93f5e8dec469ca4ddd712451418add43da946dbaed7f7a98d2",
|
||||
"sha256:65a28969a025a0eb4594637b6103201dc4ed2a9508bdab56ac33e43e3081c404",
|
||||
"sha256:892bb52b70bd5ea9dbbc3ac44f38e84f5a04e9d8b1bff48159d96cb795b81159",
|
||||
"sha256:8a9becd5cbd5062f973bcd2e7bc79483af310222de112b6541f8af1f93a3cc42",
|
||||
"sha256:972a7aaeb7c4a2795b52eef52ee991ef040b31009f36deca6207a986607b55f3",
|
||||
"sha256:97b119c436bfa96a92ac2ca525f7025836d4d4e64b1c9f9eff8dbaf3ff1d86f3",
|
||||
"sha256:9ba37698e242223f8053cc158f130aee046a96feacbeab65893dbe94f5530118",
|
||||
"sha256:b1b0e1f626a0f079c0d3696db70132fb1f29aa87c66aecb6501a9b8be64ce9f7",
|
||||
"sha256:c14c1224fd1a5be2733530d648a316974dbbb3c946913562c6005a76f21ca042",
|
||||
"sha256:c79a8546c48ae6465189e54e3245a97ddf21161e33ff7eaa42787353417bb2b6",
|
||||
"sha256:ceb76935ac4ebdf6d7bc845482a4450b284c6ccfb281e34da51d510658ab34d8",
|
||||
"sha256:e22bffaad04b4d16e1c091baed7f2733fc1ebb91e0c602abf1b6834d17158b1f",
|
||||
"sha256:ec883b8e44d877bda6f94a36313a1c6063f8b1997aa091628ae2f34c7f97c8d5",
|
||||
"sha256:f1baa54d50ec031d1a9beb89974108f8f2c0706f49798f4777df879df0e1adb6",
|
||||
"sha256:f53a5385932cda1e2c862d89460992911a89768c65d176ff8c50cddca4d29bed"
|
||||
"sha256:047d9473cf68af50ac85f8ee5d5f21a60f849bc17d348da7fc85711287a75031",
|
||||
"sha256:0f66dc6c8a3cc319561a633b6aa82c44107f12594643efa37210d8c924fc1c71",
|
||||
"sha256:12c9169c4e8fe0a7329e8658c7e488001f6b4c8e88740e76292c2b857af2e94c",
|
||||
"sha256:248cffc168896982f125f5c13e9317c059f74fffdb4152893339f3be62a01340",
|
||||
"sha256:27faf0552bf8c260a5cee21a76e031acaea68babb64daf7e8f2e2540745082aa",
|
||||
"sha256:285edafad9bc60d96978ed24d77cdc0b91dace88e5da8c548ba5937c425bca8b",
|
||||
"sha256:384b12c9aa8ef95558abdcb50aada56d74bc7cc131dd62d28c2d0e4d3aadd573",
|
||||
"sha256:38950b3a707f6cef09cd3cbb142474357ad1a985ceb44d921bdf7b4647b3e13e",
|
||||
"sha256:4aad1b88933fd6dc2846552b89ad0c74ddbba2f0884e2c162aa368374bf5abab",
|
||||
"sha256:4ac6148008c169603070c092e81f88738f1a0c511e07bd2bb0f9ef542d375da9",
|
||||
"sha256:4deb1d2a45861ae6f0b12ea0a786a03d19d29edcc7e05775b85ec2877cb54c5e",
|
||||
"sha256:59aa2c124df72cc75ed72c8d6005c442d4685691a30c55321e00ed915ad1a291",
|
||||
"sha256:5a47d2123a9ec86660fe0e8d0ebf0aa6bc6a17edc63f338b73ea20ba11713f12",
|
||||
"sha256:5cc901c2ab9409b4b7ac7b5bcc3e86ac14548627062463da0af3b6b7c555a871",
|
||||
"sha256:6c1db03e8dff7b9f955a0fb9907eb9ca5da75b5ce056c0c93d33100a35050281",
|
||||
"sha256:7ce80c0a65a6ea90ef9c1f63c8593fcd2929448613fc8da0adf3e6bfad669d08",
|
||||
"sha256:809c19241c14433c5d6135e1b6c72da4e3b56d5c865ad5736ab99af8896b8f41",
|
||||
"sha256:83792cb4e0b5af480588601467c0764242b9a483caea71ef12d22a0d0d6bdce2",
|
||||
"sha256:846fa202bd7ee0f6215c897a1d33238ef071b50766339186687bd9b7a6d26ac5",
|
||||
"sha256:9f5529fc02009f96ba95bea48870173426879dc19eec49ca8e08cd63ecd82ddb",
|
||||
"sha256:a423c2ea001c6265ed28700df056f75e26215fd28c001e93ef4380b0f05f9547",
|
||||
"sha256:ac4428094b42907aba5879c7c000d01c8278d451a3b7cccd2103e21f6397ea75",
|
||||
"sha256:b1ae48d87f10d1384e5beecd169c77502fcc04a2c00a4c02b85f0a94b419e5f9",
|
||||
"sha256:bf4e972a88f8841d8fdc6db1a75e0f8d763e66e3754b03006cbc3854d89f1cb1",
|
||||
"sha256:c6414f6aad598364aaf81068cabb077894eb88fed99c6a65e6e8217bab62ae7a",
|
||||
"sha256:c710fcb7ee32f67baf25aa9ffede4795fd5d93b163ce95fdc724383e38c9df96",
|
||||
"sha256:c7be4b8a09852291c3c48d3c25d1b876d2494a0a674980089ac9d5e0d78bd132",
|
||||
"sha256:c9e5ffb910b14f090ac9c38599063e354887a5f6d7e6d26795e916b4514f2c1a",
|
||||
"sha256:e0697b826da6c2472bb6488db4c0a7fa8af0d52fa08833ceb3681358914b14e5",
|
||||
"sha256:e9a3edd5f714229d41057d56ac0f39ad9bdba6767e8c888c951869f0bdd129b0"
|
||||
],
|
||||
"version": "==6.2.0"
|
||||
"version": "==6.2.1"
|
||||
},
|
||||
"pydeep": {
|
||||
"hashes": [
|
||||
|
|
@ -139,20 +161,20 @@
|
|||
"pymispwarninglists": {
|
||||
"editable": true,
|
||||
"git": "https://github.com/MISP/PyMISPWarningLists.git",
|
||||
"ref": "8a47f8b7f723a268e5a6b5420fe4b873e4fd6a0b"
|
||||
"ref": "1257a2e378ffb9f3dfcc4a0e83bde4ae1b040c83"
|
||||
},
|
||||
"pyrsistent": {
|
||||
"hashes": [
|
||||
"sha256:34b47fa169d6006b32e99d4b3c4031f155e6e68ebcc107d6454852e8e0ee6533"
|
||||
"sha256:f3b280d030afb652f79d67c5586157c5c1355c9a58dfc7940566e28d28f3df1b"
|
||||
],
|
||||
"version": "==0.15.4"
|
||||
"version": "==0.15.6"
|
||||
},
|
||||
"python-dateutil": {
|
||||
"hashes": [
|
||||
"sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb",
|
||||
"sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e"
|
||||
"sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c",
|
||||
"sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a"
|
||||
],
|
||||
"version": "==2.8.0"
|
||||
"version": "==2.8.1"
|
||||
},
|
||||
"python-magic": {
|
||||
"hashes": [
|
||||
|
|
@ -163,31 +185,36 @@
|
|||
},
|
||||
"reportlab": {
|
||||
"hashes": [
|
||||
"sha256:06b7c7436fa6d4844c7637161f3297c7a96240f35622ab2d219e4fd8387c0ab2",
|
||||
"sha256:0a5acf67bd9812e38ed84be8994c07a8136b0a8f4c14a1c66c9c73a9567a9a44",
|
||||
"sha256:1c8ca145d03e3c620866b06febb241b179197b58fb07454fbc8e9d6184cdcc93",
|
||||
"sha256:2f8d785660ee316874c86abad345633ce8c652e88e03ae8a10f1fdadc72fd23d",
|
||||
"sha256:4869d342352c92a812ce40555ef2a9cfbd722390d67fe61f1d6ec770e9ca41a3",
|
||||
"sha256:493e0dcd9c085d46acf4fe3f00f941e562490a74b651409039a0dee2a0d76555",
|
||||
"sha256:4e606e3ee9345e68cd205022d526250ad2a1164eea8f1e29d77d6ad08631b0ba",
|
||||
"sha256:5bf91bae8995db91650fda658129c268515358b756fd16c0261a9dd641df1856",
|
||||
"sha256:6df0730f8f715aa12333bd6d2a72eea3a989381451861186d9b5e71889454ac7",
|
||||
"sha256:7195c6ea096d10c91cc470f9f0ced3ad74470d9c0fd97923b5e764597dd13671",
|
||||
"sha256:7431c979e2b498e8e20abf458f360a451717d76c3c1bd49d1fc5697d3504f8e5",
|
||||
"sha256:7f7f70a8d4b573d1ff65a81415b4b6ed9545630f381dff4a69307640e09d381d",
|
||||
"sha256:9945433667a46f054d1125b4ca86fe9ee31feb254728b38242e9a6008c135efe",
|
||||
"sha256:b1cdbfc1fd54ac947b9f0114e00ab94e945db679f1e03357a3c00f3a85e73eea",
|
||||
"sha256:bf149847a2fd8f24b788a8abbf97a2b9a73edc5b1bd719384b786eb84bcad15e",
|
||||
"sha256:ce514bfce2bf3e302f52aba9929fe3ac7d918cfea2f5d3e30bf9dac9658bf094",
|
||||
"sha256:d243d4c8cf1a7e78b734c03628b684ec5de25df1f02ccea2e10fbd217430cb72",
|
||||
"sha256:d4bee20f52b8c3c477dc780780654cafcfc0eb34d8d6960c13a34a444b431f09",
|
||||
"sha256:e730529bd1f62034c50f70a2b05fadbf7d1402d39ff69c9dc63db066d0ef8a46",
|
||||
"sha256:eb54ecfbf1abe6134073b7b35fd40442c4cd81bb9a5bee1a3038b8867b721bfb",
|
||||
"sha256:f18ec70f5ee6a78b3bb4361e55f3a5ef34eb253f1e72fba76f29f0d680cd446f",
|
||||
"sha256:f6be66f69198dcd04a79faa6052f756d35643496321858f06931c7b1ed9833ab",
|
||||
"sha256:fc5c23a53fbd97b8aab4968c8548ce5cea4a54a26b4f8c1e6835df7adb8d0fe2"
|
||||
"sha256:149f0eeb4ea716441638b05fd6d3667d32f1463f3eac50b63e100a73a5533cdd",
|
||||
"sha256:1aa9a2e1a87749db265b592ad25e498b39f70fce9f53a012cdf69f74259b6e43",
|
||||
"sha256:1f5ce489adb2db2862249492e6367539cfa65b781cb06dcf13363dc52219be7e",
|
||||
"sha256:23b28ba1784a6c52a926c075abd9f396d03670e71934b24db5ff684f8b870e0f",
|
||||
"sha256:3d3de0f4facdd7e3c56ecbc55733a958b86c35a8e7ba6066c7b1ba383e282f58",
|
||||
"sha256:484d346b8f463ba2ddaf6d365c6ac5971cd062528b6d5ba68cac02b9435366c5",
|
||||
"sha256:4da2467def21f2e20720b21f6c18e7f7866720a955c716b990e94e3979fe913f",
|
||||
"sha256:5ebdf22daee7d8e630134d94f477fe6abd65a65449d4eec682a7b458b5249604",
|
||||
"sha256:655a1b68be18a73fec5233fb5d81f726b4db32269e487aecf5b6853cca926d86",
|
||||
"sha256:6c535a304888dafe50c2c24d4924aeefc11e0542488ee6965f6133d415e86bbc",
|
||||
"sha256:7560ef655ac6448bb257fd34bfdfb8d546f9c7c0900ed8963fb8509f75e8ca80",
|
||||
"sha256:7a1c2fa3e6310dbe47efee2020dc0f25be7a75ff09a8fedc4a87d4397f3810c1",
|
||||
"sha256:817c344b9aa53b5bfc2f58ff82111a1e85ca4c8b68d1add088b547360a6ebcfa",
|
||||
"sha256:81d950e398d6758aeaeeb267aa1a62940735414c980f77dd0a270cef1782a43d",
|
||||
"sha256:83ef44936ef4e9c432d62bc2b72ec8d772b87af319d123e827a72e9b6884c851",
|
||||
"sha256:9f975adc2c7a236403f0bc91d7a3916e644e47b1f1e3990325f15e73b83581ec",
|
||||
"sha256:a5ca59e2b7e70a856de6db9dadd3e11a1b3b471c999585284d5c1d479c01cf5d",
|
||||
"sha256:ad2cf5a673c05fae9e91e987994b95205c13c5fa55d7393cf8b06f9de6f92990",
|
||||
"sha256:b8c3d76276372f87b7c8ff22065dbc072cca5ffb06ba0267edc298df7acf942d",
|
||||
"sha256:b93f7f908e916d9413dd8c04da1ccb3977e446803f59078424decdc0de449133",
|
||||
"sha256:c0ecd0af92c759edec0d24ba92f4a18c28d4a19229ae7c8249f94e82f3d76288",
|
||||
"sha256:c9e38eefc90a02c072a87a627ff66b2d67c23f6f82274d2aa7fb28e644e8f409",
|
||||
"sha256:ca2a1592d2e181a04372d0276ee847308ea206dfe7c86fe94769e7ac126e6e85",
|
||||
"sha256:ce1dfc9beec83e66250ca3afaf5ddf6b9a3ce70a30a9526dec7c6bec3266baf1",
|
||||
"sha256:d3550c90751132b26b72a78954905974f33b1237335fbe0d8be957f9636c376a",
|
||||
"sha256:e35a574f4e5ec0fdd5dc354e74ec143d853abd7f76db435ffe2a57d0161a22eb",
|
||||
"sha256:ee5cafca6ef1a38fef8cbf3140dd2198ad1ee82331530b546039216ef94f93cb",
|
||||
"sha256:fa1c969176cb3594a785c6818bcb943ebd49453791f702380b13a35fa23b385a"
|
||||
],
|
||||
"version": "==3.5.28"
|
||||
"version": "==3.5.32"
|
||||
},
|
||||
"requests": {
|
||||
"hashes": [
|
||||
|
|
@ -198,24 +225,24 @@
|
|||
},
|
||||
"six": {
|
||||
"hashes": [
|
||||
"sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c",
|
||||
"sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73"
|
||||
"sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd",
|
||||
"sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66"
|
||||
],
|
||||
"version": "==1.12.0"
|
||||
"version": "==1.13.0"
|
||||
},
|
||||
"soupsieve": {
|
||||
"hashes": [
|
||||
"sha256:605f89ad5fdbfefe30cdc293303665eff2d188865d4dbe4eb510bba1edfbfce3",
|
||||
"sha256:b91d676b330a0ebd5b21719cb6e9b57c57d433671f65b9c28dd3461d9a1ed0b6"
|
||||
"sha256:bdb0d917b03a1369ce964056fc195cfdff8819c40de04695a80bc813c3cfa1f5",
|
||||
"sha256:e2c1c5dee4a1c36bcb790e0fabd5492d874b8ebd4617622c4f6a731701060dda"
|
||||
],
|
||||
"version": "==1.9.4"
|
||||
"version": "==1.9.5"
|
||||
},
|
||||
"urllib3": {
|
||||
"hashes": [
|
||||
"sha256:3de946ffbed6e6746608990594d08faac602528ac7015ac28d33cee6a45b7398",
|
||||
"sha256:9a107b99a5393caf59c7aa3c1249c16e6879447533d0887f4336dde834c7be86"
|
||||
"sha256:a8a318824cc77d1fd4b2bec2ded92646630d7fe8619497b142c84a9e6f5a7293",
|
||||
"sha256:f3c5fd51747d450d4dcf6f923c81f78f811aab8205fda64b0aba34a4e48b0745"
|
||||
],
|
||||
"version": "==1.25.6"
|
||||
"version": "==1.25.7"
|
||||
},
|
||||
"validators": {
|
||||
"hashes": [
|
||||
|
|
@ -228,6 +255,13 @@
|
|||
"sha256:565a021fd19419476b9362b05eeaa094178de64f8361e44468f9e9d7843901e1"
|
||||
],
|
||||
"version": "==1.11.2"
|
||||
},
|
||||
"zipp": {
|
||||
"hashes": [
|
||||
"sha256:3718b1cbcd963c7d4c5511a8240812904164b7f381b647143a89d3b98f9bcd8e",
|
||||
"sha256:f06903e9f1f43b12d371004b4ac7b06ab39a44adc747266928ae6debfa7b3335"
|
||||
],
|
||||
"version": "==0.6.0"
|
||||
}
|
||||
},
|
||||
"develop": {
|
||||
|
|
@ -240,10 +274,10 @@
|
|||
},
|
||||
"attrs": {
|
||||
"hashes": [
|
||||
"sha256:ec20e7a4825331c1b5ebf261d111e16fa9612c1f7a5e1f884f12bd53a664dfd2",
|
||||
"sha256:f913492e1663d3c36f502e5e9ba6cd13cf19d7fab50aa13239e420fef95e1396"
|
||||
"sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c",
|
||||
"sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72"
|
||||
],
|
||||
"version": "==19.2.0"
|
||||
"version": "==19.3.0"
|
||||
},
|
||||
"babel": {
|
||||
"hashes": [
|
||||
|
|
@ -255,16 +289,17 @@
|
|||
"beautifulsoup4": {
|
||||
"hashes": [
|
||||
"sha256:5279c36b4b2ec2cb4298d723791467e3000e5384a43ea0cdf5d45207c7e97169",
|
||||
"sha256:6135db2ba678168c07950f9a16c4031822c6f4aec75a65e0a97bc5ca09789931",
|
||||
"sha256:dcdef580e18a76d54002088602eba453eec38ebbcafafeaabd8cab12b6155d57"
|
||||
],
|
||||
"version": "==4.8.1"
|
||||
},
|
||||
"certifi": {
|
||||
"hashes": [
|
||||
"sha256:e4f3620cfea4f83eedc95b24abd9cd56f3c4b146dd0177e83a21b4eb49e21e50",
|
||||
"sha256:fd7c7c74727ddcf00e9acd26bba8da604ffec95bf1c2144e67aff7a8b50e6cef"
|
||||
"sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3",
|
||||
"sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f"
|
||||
],
|
||||
"version": "==2019.9.11"
|
||||
"version": "==2019.11.28"
|
||||
},
|
||||
"chardet": {
|
||||
"hashes": [
|
||||
|
|
@ -349,17 +384,17 @@
|
|||
},
|
||||
"decorator": {
|
||||
"hashes": [
|
||||
"sha256:86156361c50488b84a3f148056ea716ca587df2f0de1d34750d35c21312725de",
|
||||
"sha256:f069f3a01830ca754ba5258fde2278454a0b5b79e0d7f5c13b3b97e57d4acff6"
|
||||
"sha256:54c38050039232e1db4ad7375cfce6748d7b41c29e95a081c8a6d2c30364a2ce",
|
||||
"sha256:5d19b92a3c8f7f101c8dd86afd86b0f061a8ce4540ab8cd401fa2542756bce6d"
|
||||
],
|
||||
"version": "==4.4.0"
|
||||
"version": "==4.4.1"
|
||||
},
|
||||
"deprecated": {
|
||||
"hashes": [
|
||||
"sha256:a515c4cf75061552e0284d123c3066fbbe398952c87333a92b8fc3dd8e4f9cc1",
|
||||
"sha256:b07b414c8aac88f60c1d837d21def7e83ba711052e03b3cbaff27972567a8f8d"
|
||||
"sha256:408038ab5fdeca67554e8f6742d1521cd3cd0ee0ff9d47f29318a4f4da31c308",
|
||||
"sha256:8b6a5aa50e482d8244a62e5582b96c372e87e3a28e8b49c316e46b95c76a611d"
|
||||
],
|
||||
"version": "==1.2.6"
|
||||
"version": "==1.2.7"
|
||||
},
|
||||
"docopt": {
|
||||
"hashes": [
|
||||
|
|
@ -389,6 +424,14 @@
|
|||
],
|
||||
"version": "==1.1.0"
|
||||
},
|
||||
"importlib-metadata": {
|
||||
"hashes": [
|
||||
"sha256:b044f07694ef14a6683b097ba56bd081dbc7cdc7c7fe46011e499dfecc082f21",
|
||||
"sha256:e6ac600a142cf2db707b1998382cc7fc3b02befb7273876e01b8ad10b9652742"
|
||||
],
|
||||
"markers": "python_version < '3.8'",
|
||||
"version": "==1.1.0"
|
||||
},
|
||||
"jinja2": {
|
||||
"hashes": [
|
||||
"sha256:74320bb91f31270f9551d46522e33af46a80c3d619f4a4bf42b3164d30b5911f",
|
||||
|
|
@ -398,27 +441,29 @@
|
|||
},
|
||||
"jsonschema": {
|
||||
"hashes": [
|
||||
"sha256:5f9c0a719ca2ce14c5de2fd350a64fd2d13e8539db29836a86adc990bb1a068f",
|
||||
"sha256:8d4a2b7b6c2237e0199c8ea1a6d3e05bf118e289ae2b9d7ba444182a2959560d"
|
||||
"sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163",
|
||||
"sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"
|
||||
],
|
||||
"version": "==3.0.2"
|
||||
"version": "==3.2.0"
|
||||
},
|
||||
"lief": {
|
||||
"hashes": [
|
||||
"sha256:0efba18d7b9776529ea5c18c771b35871896a8ceb95a19351e50d4813a11c632",
|
||||
"sha256:3d9c7bb1e353e875f295a72a58d3a37ae1ba3e1ff1beb57b8a65f1a726064093",
|
||||
"sha256:3db5939e7d95f776f9866586128c2a5be614eaec43ab985ac27ff2c531f8ac5f",
|
||||
"sha256:4c61598818b0091d80839875aa107cfd10ae1017a3e9c9de4bc002622b8e3179",
|
||||
"sha256:4f26d07bdada8ca5ef3dc5fa2f71f20f7e8ab4f78f7c5e00134477f51feb6a80",
|
||||
"sha256:55fe3c8a0990dce16ab5bf88df707f1eacac4eb34561667ac478497e0e0807c7",
|
||||
"sha256:68bcf18e40c9412d2d08d6311e04eb6c19e20ec174764706da2d602c45aa4fd5",
|
||||
"sha256:7ff910d99361022451e9c25e34cb844768e2fa347cfb0f4ad70f531810d776d4",
|
||||
"sha256:ac571152d0b864e8d376bc733c5728a224316be1cdefc290174f1bf8ab10ec70",
|
||||
"sha256:dd17a7cdcd29a2efca3d4cb4fb078a06daf1cafec8912560965a8d8dbf346739",
|
||||
"sha256:efa5f3523c01f7f0f5f2c14e5ac808e2447d1435c6a2872e5ab1a97ef1b0db9b",
|
||||
"sha256:f1aadb344b5e14b308167bd2c9f31f1915e3c4e3f9a9ca92ff7b7bfbede5034c"
|
||||
"sha256:276cc63ec12a21bdf01b8d30962692c17499788234f0765247ca7a35872097ec",
|
||||
"sha256:3e6baaeb52bdc339b5f19688b58fd8d5778b92e50221f920cedfa2bec1f4d5c2",
|
||||
"sha256:45e5c592b57168c447698381d927eb2386ffdd52afe0c48245f848d4cc7ee05a",
|
||||
"sha256:6547752b5db105cd41c9fa65d0d7452a4d7541b77ffee716b46246c6d81e172f",
|
||||
"sha256:83b51e01627b5982662f9550ac1230758aa56945ed86829e4291932d98417da3",
|
||||
"sha256:895599194ea7495bf304e39317b04df20cccf799fc2751867cc1aa4997cfcdae",
|
||||
"sha256:8a91cee2568306fe1d2bf84341b459c85368317d01d7105fa49e4f4ede837076",
|
||||
"sha256:913b36a67707dc2afa72f117bab9856ea3f434f332b04a002a0f9723c8779320",
|
||||
"sha256:9f604a361a3b1b3ed5fdafed0321c5956cb3b265b5efe2250d1bf8911a80c65b",
|
||||
"sha256:a487fe7234c04bccd58223dbb79214421176e2629814c7a4a887764cceb5be7c",
|
||||
"sha256:bc8488fb0661cb436fe4bb4fe947d0f9aa020e9acaed233ccf01ab04d888c68a",
|
||||
"sha256:bddbf333af62310a10cb738a1df1dc2b140dd9c663b55ba3500c10c249d416d2",
|
||||
"sha256:cce48d7c97cef85e01e6cfeff55f2068956b5c0257eb9c2d2c6d15e33dd1e4fc",
|
||||
"sha256:f8b3f66956c56b582b3adc573bf2a938c25fb21c8894b373a113e24c494fc982"
|
||||
],
|
||||
"version": "==0.10.0.dev0"
|
||||
"version": "==0.10.1"
|
||||
},
|
||||
"markupsafe": {
|
||||
"hashes": [
|
||||
|
|
@ -460,6 +505,13 @@
|
|||
"index": "pypi",
|
||||
"version": "==0.55.0"
|
||||
},
|
||||
"more-itertools": {
|
||||
"hashes": [
|
||||
"sha256:53ff73f186307d9c8ef17a9600309154a6ae27f25579e80af4db8f047ba14bc2",
|
||||
"sha256:a0ea684c39bc4315ba7aae406596ef191fd84f873d2d2751f84d64e81a7a2d45"
|
||||
],
|
||||
"version": "==8.0.0"
|
||||
},
|
||||
"neobolt": {
|
||||
"hashes": [
|
||||
"sha256:56b86b8b2c3facdd54589e60ecd22e0234d6f40645ab2e2cf87ef0cd79df20af"
|
||||
|
|
@ -490,34 +542,38 @@
|
|||
},
|
||||
"pillow": {
|
||||
"hashes": [
|
||||
"sha256:00fdeb23820f30e43bba78eb9abb00b7a937a655de7760b2e09101d63708b64e",
|
||||
"sha256:01f948e8220c85eae1aa1a7f8edddcec193918f933fb07aaebe0bfbbcffefbf1",
|
||||
"sha256:08abf39948d4b5017a137be58f1a52b7101700431f0777bec3d897c3949f74e6",
|
||||
"sha256:099a61618b145ecb50c6f279666bbc398e189b8bc97544ae32b8fcb49ad6b830",
|
||||
"sha256:2c1c61546e73de62747e65807d2cc4980c395d4c5600ecb1f47a650c6fa78c79",
|
||||
"sha256:2ed9c4f694861642401f27dc3cb99772be67cd190e84845c749dae0a06c3bfae",
|
||||
"sha256:338581b30b908e111be578f0297255f6b57a51358cd16fa0e6f664c9a1f88bff",
|
||||
"sha256:38c7d48a21cd06fdeee93987147b9b1c55b73b4cfcbf83240568bfbd5adee447",
|
||||
"sha256:43fd026f613c8e48a25eba1a92f4d2ad7f3903c95d8c33a11611a7717d2ab654",
|
||||
"sha256:4548236844327a718ce3bb182ab32a16fa2050c61e334e959f554cac052fb0df",
|
||||
"sha256:5090857876c58885cfa388dc649e5db30aae98a068c26f3fd0ac9d7d9a4d9572",
|
||||
"sha256:5bbba34f97a26a93f5e8dec469ca4ddd712451418add43da946dbaed7f7a98d2",
|
||||
"sha256:65a28969a025a0eb4594637b6103201dc4ed2a9508bdab56ac33e43e3081c404",
|
||||
"sha256:892bb52b70bd5ea9dbbc3ac44f38e84f5a04e9d8b1bff48159d96cb795b81159",
|
||||
"sha256:8a9becd5cbd5062f973bcd2e7bc79483af310222de112b6541f8af1f93a3cc42",
|
||||
"sha256:972a7aaeb7c4a2795b52eef52ee991ef040b31009f36deca6207a986607b55f3",
|
||||
"sha256:97b119c436bfa96a92ac2ca525f7025836d4d4e64b1c9f9eff8dbaf3ff1d86f3",
|
||||
"sha256:9ba37698e242223f8053cc158f130aee046a96feacbeab65893dbe94f5530118",
|
||||
"sha256:b1b0e1f626a0f079c0d3696db70132fb1f29aa87c66aecb6501a9b8be64ce9f7",
|
||||
"sha256:c14c1224fd1a5be2733530d648a316974dbbb3c946913562c6005a76f21ca042",
|
||||
"sha256:c79a8546c48ae6465189e54e3245a97ddf21161e33ff7eaa42787353417bb2b6",
|
||||
"sha256:ceb76935ac4ebdf6d7bc845482a4450b284c6ccfb281e34da51d510658ab34d8",
|
||||
"sha256:e22bffaad04b4d16e1c091baed7f2733fc1ebb91e0c602abf1b6834d17158b1f",
|
||||
"sha256:ec883b8e44d877bda6f94a36313a1c6063f8b1997aa091628ae2f34c7f97c8d5",
|
||||
"sha256:f1baa54d50ec031d1a9beb89974108f8f2c0706f49798f4777df879df0e1adb6",
|
||||
"sha256:f53a5385932cda1e2c862d89460992911a89768c65d176ff8c50cddca4d29bed"
|
||||
"sha256:047d9473cf68af50ac85f8ee5d5f21a60f849bc17d348da7fc85711287a75031",
|
||||
"sha256:0f66dc6c8a3cc319561a633b6aa82c44107f12594643efa37210d8c924fc1c71",
|
||||
"sha256:12c9169c4e8fe0a7329e8658c7e488001f6b4c8e88740e76292c2b857af2e94c",
|
||||
"sha256:248cffc168896982f125f5c13e9317c059f74fffdb4152893339f3be62a01340",
|
||||
"sha256:27faf0552bf8c260a5cee21a76e031acaea68babb64daf7e8f2e2540745082aa",
|
||||
"sha256:285edafad9bc60d96978ed24d77cdc0b91dace88e5da8c548ba5937c425bca8b",
|
||||
"sha256:384b12c9aa8ef95558abdcb50aada56d74bc7cc131dd62d28c2d0e4d3aadd573",
|
||||
"sha256:38950b3a707f6cef09cd3cbb142474357ad1a985ceb44d921bdf7b4647b3e13e",
|
||||
"sha256:4aad1b88933fd6dc2846552b89ad0c74ddbba2f0884e2c162aa368374bf5abab",
|
||||
"sha256:4ac6148008c169603070c092e81f88738f1a0c511e07bd2bb0f9ef542d375da9",
|
||||
"sha256:4deb1d2a45861ae6f0b12ea0a786a03d19d29edcc7e05775b85ec2877cb54c5e",
|
||||
"sha256:59aa2c124df72cc75ed72c8d6005c442d4685691a30c55321e00ed915ad1a291",
|
||||
"sha256:5a47d2123a9ec86660fe0e8d0ebf0aa6bc6a17edc63f338b73ea20ba11713f12",
|
||||
"sha256:5cc901c2ab9409b4b7ac7b5bcc3e86ac14548627062463da0af3b6b7c555a871",
|
||||
"sha256:6c1db03e8dff7b9f955a0fb9907eb9ca5da75b5ce056c0c93d33100a35050281",
|
||||
"sha256:7ce80c0a65a6ea90ef9c1f63c8593fcd2929448613fc8da0adf3e6bfad669d08",
|
||||
"sha256:809c19241c14433c5d6135e1b6c72da4e3b56d5c865ad5736ab99af8896b8f41",
|
||||
"sha256:83792cb4e0b5af480588601467c0764242b9a483caea71ef12d22a0d0d6bdce2",
|
||||
"sha256:846fa202bd7ee0f6215c897a1d33238ef071b50766339186687bd9b7a6d26ac5",
|
||||
"sha256:9f5529fc02009f96ba95bea48870173426879dc19eec49ca8e08cd63ecd82ddb",
|
||||
"sha256:a423c2ea001c6265ed28700df056f75e26215fd28c001e93ef4380b0f05f9547",
|
||||
"sha256:ac4428094b42907aba5879c7c000d01c8278d451a3b7cccd2103e21f6397ea75",
|
||||
"sha256:b1ae48d87f10d1384e5beecd169c77502fcc04a2c00a4c02b85f0a94b419e5f9",
|
||||
"sha256:bf4e972a88f8841d8fdc6db1a75e0f8d763e66e3754b03006cbc3854d89f1cb1",
|
||||
"sha256:c6414f6aad598364aaf81068cabb077894eb88fed99c6a65e6e8217bab62ae7a",
|
||||
"sha256:c710fcb7ee32f67baf25aa9ffede4795fd5d93b163ce95fdc724383e38c9df96",
|
||||
"sha256:c7be4b8a09852291c3c48d3c25d1b876d2494a0a674980089ac9d5e0d78bd132",
|
||||
"sha256:c9e5ffb910b14f090ac9c38599063e354887a5f6d7e6d26795e916b4514f2c1a",
|
||||
"sha256:e0697b826da6c2472bb6488db4c0a7fa8af0d52fa08833ceb3681358914b14e5",
|
||||
"sha256:e9a3edd5f714229d41057d56ac0f39ad9bdba6767e8c888c951869f0bdd129b0"
|
||||
],
|
||||
"version": "==6.2.0"
|
||||
"version": "==6.2.1"
|
||||
},
|
||||
"prompt-toolkit": {
|
||||
"hashes": [
|
||||
|
|
@ -529,17 +585,19 @@
|
|||
},
|
||||
"psutil": {
|
||||
"hashes": [
|
||||
"sha256:028a1ec3c6197eadd11e7b46e8cc2f0720dc18ac6d7aabdb8e8c0d6c9704f000",
|
||||
"sha256:503e4b20fa9d3342bcf58191bbc20a4a5ef79ca7df8972e6197cc14c5513e73d",
|
||||
"sha256:863a85c1c0a5103a12c05a35e59d336e1d665747e531256e061213e2e90f63f3",
|
||||
"sha256:954f782608bfef9ae9f78e660e065bd8ffcfaea780f9f2c8a133bb7cb9e826d7",
|
||||
"sha256:b6e08f965a305cd84c2d07409bc16fbef4417d67b70c53b299116c5b895e3f45",
|
||||
"sha256:bc96d437dfbb8865fc8828cf363450001cb04056bbdcdd6fc152c436c8a74c61",
|
||||
"sha256:cf49178021075d47c61c03c0229ac0c60d5e2830f8cab19e2d88e579b18cdb76",
|
||||
"sha256:d5350cb66690915d60f8b233180f1e49938756fb2d501c93c44f8fb5b970cc63",
|
||||
"sha256:eba238cf1989dfff7d483c029acb0ac4fcbfc15de295d682901f0e2497e6781a"
|
||||
"sha256:094f899ac3ef72422b7e00411b4ed174e3c5a2e04c267db6643937ddba67a05b",
|
||||
"sha256:10b7f75cc8bd676cfc6fa40cd7d5c25b3f45a0e06d43becd7c2d2871cbb5e806",
|
||||
"sha256:1b1575240ca9a90b437e5a40db662acd87bbf181f6aa02f0204978737b913c6b",
|
||||
"sha256:21231ef1c1a89728e29b98a885b8e0a8e00d09018f6da5cdc1f43f988471a995",
|
||||
"sha256:28f771129bfee9fc6b63d83a15d857663bbdcae3828e1cb926e91320a9b5b5cd",
|
||||
"sha256:70387772f84fa5c3bb6a106915a2445e20ac8f9821c5914d7cbde148f4d7ff73",
|
||||
"sha256:b560f5cd86cf8df7bcd258a851ca1ad98f0d5b8b98748e877a0aec4e9032b465",
|
||||
"sha256:b74b43fecce384a57094a83d2778cdfc2e2d9a6afaadd1ebecb2e75e0d34e10d",
|
||||
"sha256:e85f727ffb21539849e6012f47b12f6dd4c44965e56591d8dec6e8bc9ab96f4a",
|
||||
"sha256:fd2e09bb593ad9bdd7429e779699d2d47c1268cbde4dda95fcd1bd17544a0217",
|
||||
"sha256:ffad8eb2ac614518bbe3c0b8eb9dffdb3a8d2e3a7d5da51c5b974fb723a5c5aa"
|
||||
],
|
||||
"version": "==5.6.3"
|
||||
"version": "==5.6.7"
|
||||
},
|
||||
"py2neo": {
|
||||
"hashes": [
|
||||
|
|
@ -572,23 +630,23 @@
|
|||
},
|
||||
"pyparsing": {
|
||||
"hashes": [
|
||||
"sha256:6f98a7b9397e206d78cc01df10131398f1c8b8510a2f4d97d9abd82e1aacdd80",
|
||||
"sha256:d9338df12903bbf5d65a0e4e87c2161968b10d2e489652bb47001d82a9b028b4"
|
||||
"sha256:20f995ecd72f2a1f4bf6b072b63b22e2eb457836601e76d6e5dfcd75436acc1f",
|
||||
"sha256:4ca62001be367f01bd3e92ecbb79070272a9d4964dce6a48a82ff0b8bc7e683a"
|
||||
],
|
||||
"version": "==2.4.2"
|
||||
"version": "==2.4.5"
|
||||
},
|
||||
"pyrsistent": {
|
||||
"hashes": [
|
||||
"sha256:34b47fa169d6006b32e99d4b3c4031f155e6e68ebcc107d6454852e8e0ee6533"
|
||||
"sha256:f3b280d030afb652f79d67c5586157c5c1355c9a58dfc7940566e28d28f3df1b"
|
||||
],
|
||||
"version": "==0.15.4"
|
||||
"version": "==0.15.6"
|
||||
},
|
||||
"python-dateutil": {
|
||||
"hashes": [
|
||||
"sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb",
|
||||
"sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e"
|
||||
"sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c",
|
||||
"sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a"
|
||||
],
|
||||
"version": "==2.8.0"
|
||||
"version": "==2.8.1"
|
||||
},
|
||||
"python-magic": {
|
||||
"hashes": [
|
||||
|
|
@ -613,31 +671,36 @@
|
|||
},
|
||||
"reportlab": {
|
||||
"hashes": [
|
||||
"sha256:06b7c7436fa6d4844c7637161f3297c7a96240f35622ab2d219e4fd8387c0ab2",
|
||||
"sha256:0a5acf67bd9812e38ed84be8994c07a8136b0a8f4c14a1c66c9c73a9567a9a44",
|
||||
"sha256:1c8ca145d03e3c620866b06febb241b179197b58fb07454fbc8e9d6184cdcc93",
|
||||
"sha256:2f8d785660ee316874c86abad345633ce8c652e88e03ae8a10f1fdadc72fd23d",
|
||||
"sha256:4869d342352c92a812ce40555ef2a9cfbd722390d67fe61f1d6ec770e9ca41a3",
|
||||
"sha256:493e0dcd9c085d46acf4fe3f00f941e562490a74b651409039a0dee2a0d76555",
|
||||
"sha256:4e606e3ee9345e68cd205022d526250ad2a1164eea8f1e29d77d6ad08631b0ba",
|
||||
"sha256:5bf91bae8995db91650fda658129c268515358b756fd16c0261a9dd641df1856",
|
||||
"sha256:6df0730f8f715aa12333bd6d2a72eea3a989381451861186d9b5e71889454ac7",
|
||||
"sha256:7195c6ea096d10c91cc470f9f0ced3ad74470d9c0fd97923b5e764597dd13671",
|
||||
"sha256:7431c979e2b498e8e20abf458f360a451717d76c3c1bd49d1fc5697d3504f8e5",
|
||||
"sha256:7f7f70a8d4b573d1ff65a81415b4b6ed9545630f381dff4a69307640e09d381d",
|
||||
"sha256:9945433667a46f054d1125b4ca86fe9ee31feb254728b38242e9a6008c135efe",
|
||||
"sha256:b1cdbfc1fd54ac947b9f0114e00ab94e945db679f1e03357a3c00f3a85e73eea",
|
||||
"sha256:bf149847a2fd8f24b788a8abbf97a2b9a73edc5b1bd719384b786eb84bcad15e",
|
||||
"sha256:ce514bfce2bf3e302f52aba9929fe3ac7d918cfea2f5d3e30bf9dac9658bf094",
|
||||
"sha256:d243d4c8cf1a7e78b734c03628b684ec5de25df1f02ccea2e10fbd217430cb72",
|
||||
"sha256:d4bee20f52b8c3c477dc780780654cafcfc0eb34d8d6960c13a34a444b431f09",
|
||||
"sha256:e730529bd1f62034c50f70a2b05fadbf7d1402d39ff69c9dc63db066d0ef8a46",
|
||||
"sha256:eb54ecfbf1abe6134073b7b35fd40442c4cd81bb9a5bee1a3038b8867b721bfb",
|
||||
"sha256:f18ec70f5ee6a78b3bb4361e55f3a5ef34eb253f1e72fba76f29f0d680cd446f",
|
||||
"sha256:f6be66f69198dcd04a79faa6052f756d35643496321858f06931c7b1ed9833ab",
|
||||
"sha256:fc5c23a53fbd97b8aab4968c8548ce5cea4a54a26b4f8c1e6835df7adb8d0fe2"
|
||||
"sha256:149f0eeb4ea716441638b05fd6d3667d32f1463f3eac50b63e100a73a5533cdd",
|
||||
"sha256:1aa9a2e1a87749db265b592ad25e498b39f70fce9f53a012cdf69f74259b6e43",
|
||||
"sha256:1f5ce489adb2db2862249492e6367539cfa65b781cb06dcf13363dc52219be7e",
|
||||
"sha256:23b28ba1784a6c52a926c075abd9f396d03670e71934b24db5ff684f8b870e0f",
|
||||
"sha256:3d3de0f4facdd7e3c56ecbc55733a958b86c35a8e7ba6066c7b1ba383e282f58",
|
||||
"sha256:484d346b8f463ba2ddaf6d365c6ac5971cd062528b6d5ba68cac02b9435366c5",
|
||||
"sha256:4da2467def21f2e20720b21f6c18e7f7866720a955c716b990e94e3979fe913f",
|
||||
"sha256:5ebdf22daee7d8e630134d94f477fe6abd65a65449d4eec682a7b458b5249604",
|
||||
"sha256:655a1b68be18a73fec5233fb5d81f726b4db32269e487aecf5b6853cca926d86",
|
||||
"sha256:6c535a304888dafe50c2c24d4924aeefc11e0542488ee6965f6133d415e86bbc",
|
||||
"sha256:7560ef655ac6448bb257fd34bfdfb8d546f9c7c0900ed8963fb8509f75e8ca80",
|
||||
"sha256:7a1c2fa3e6310dbe47efee2020dc0f25be7a75ff09a8fedc4a87d4397f3810c1",
|
||||
"sha256:817c344b9aa53b5bfc2f58ff82111a1e85ca4c8b68d1add088b547360a6ebcfa",
|
||||
"sha256:81d950e398d6758aeaeeb267aa1a62940735414c980f77dd0a270cef1782a43d",
|
||||
"sha256:83ef44936ef4e9c432d62bc2b72ec8d772b87af319d123e827a72e9b6884c851",
|
||||
"sha256:9f975adc2c7a236403f0bc91d7a3916e644e47b1f1e3990325f15e73b83581ec",
|
||||
"sha256:a5ca59e2b7e70a856de6db9dadd3e11a1b3b471c999585284d5c1d479c01cf5d",
|
||||
"sha256:ad2cf5a673c05fae9e91e987994b95205c13c5fa55d7393cf8b06f9de6f92990",
|
||||
"sha256:b8c3d76276372f87b7c8ff22065dbc072cca5ffb06ba0267edc298df7acf942d",
|
||||
"sha256:b93f7f908e916d9413dd8c04da1ccb3977e446803f59078424decdc0de449133",
|
||||
"sha256:c0ecd0af92c759edec0d24ba92f4a18c28d4a19229ae7c8249f94e82f3d76288",
|
||||
"sha256:c9e38eefc90a02c072a87a627ff66b2d67c23f6f82274d2aa7fb28e644e8f409",
|
||||
"sha256:ca2a1592d2e181a04372d0276ee847308ea206dfe7c86fe94769e7ac126e6e85",
|
||||
"sha256:ce1dfc9beec83e66250ca3afaf5ddf6b9a3ce70a30a9526dec7c6bec3266baf1",
|
||||
"sha256:d3550c90751132b26b72a78954905974f33b1237335fbe0d8be957f9636c376a",
|
||||
"sha256:e35a574f4e5ec0fdd5dc354e74ec143d853abd7f76db435ffe2a57d0161a22eb",
|
||||
"sha256:ee5cafca6ef1a38fef8cbf3140dd2198ad1ee82331530b546039216ef94f93cb",
|
||||
"sha256:fa1c969176cb3594a785c6818bcb943ebd49453791f702380b13a35fa23b385a"
|
||||
],
|
||||
"version": "==3.5.28"
|
||||
"version": "==3.5.32"
|
||||
},
|
||||
"requests": {
|
||||
"hashes": [
|
||||
|
|
@ -656,10 +719,10 @@
|
|||
},
|
||||
"six": {
|
||||
"hashes": [
|
||||
"sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c",
|
||||
"sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73"
|
||||
"sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd",
|
||||
"sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66"
|
||||
],
|
||||
"version": "==1.12.0"
|
||||
"version": "==1.13.0"
|
||||
},
|
||||
"snowballstemmer": {
|
||||
"hashes": [
|
||||
|
|
@ -670,24 +733,24 @@
|
|||
},
|
||||
"soupsieve": {
|
||||
"hashes": [
|
||||
"sha256:605f89ad5fdbfefe30cdc293303665eff2d188865d4dbe4eb510bba1edfbfce3",
|
||||
"sha256:b91d676b330a0ebd5b21719cb6e9b57c57d433671f65b9c28dd3461d9a1ed0b6"
|
||||
"sha256:bdb0d917b03a1369ce964056fc195cfdff8819c40de04695a80bc813c3cfa1f5",
|
||||
"sha256:e2c1c5dee4a1c36bcb790e0fabd5492d874b8ebd4617622c4f6a731701060dda"
|
||||
],
|
||||
"version": "==1.9.4"
|
||||
"version": "==1.9.5"
|
||||
},
|
||||
"sphinx": {
|
||||
"hashes": [
|
||||
"sha256:0d586b0f8c2fc3cc6559c5e8fd6124628110514fda0e5d7c82e682d749d2e845",
|
||||
"sha256:839a3ed6f6b092bb60f492024489cc9e6991360fb9f52ed6361acd510d261069"
|
||||
"sha256:31088dfb95359384b1005619827eaee3056243798c62724fd3fa4b84ee4d71bd",
|
||||
"sha256:52286a0b9d7caa31efee301ec4300dbdab23c3b05da1c9024b4e84896fb73d79"
|
||||
],
|
||||
"version": "==2.2.0"
|
||||
"version": "==2.2.1"
|
||||
},
|
||||
"sphinx-autodoc-typehints": {
|
||||
"hashes": [
|
||||
"sha256:0d968ec3ee4f7fe7695ab6facf5cd2d74d3cea67584277458ad9b2788ebbcc3b",
|
||||
"sha256:8edca714fd3de8e43467d7e51dd3812fe999f8874408a639f7c38a9e1a5a4eb3"
|
||||
"sha256:27c9e6ef4f4451766ab8d08b2d8520933b97beb21c913f3df9ab2e59b56e6c6c",
|
||||
"sha256:a6b3180167479aca2c4d1ed3b5cb044a70a76cccd6b38662d39288ebd9f0dff0"
|
||||
],
|
||||
"version": "==1.8.0"
|
||||
"version": "==1.10.3"
|
||||
},
|
||||
"sphinxcontrib-applehelp": {
|
||||
"hashes": [
|
||||
|
|
@ -733,10 +796,10 @@
|
|||
},
|
||||
"urllib3": {
|
||||
"hashes": [
|
||||
"sha256:3de946ffbed6e6746608990594d08faac602528ac7015ac28d33cee6a45b7398",
|
||||
"sha256:9a107b99a5393caf59c7aa3c1249c16e6879447533d0887f4336dde834c7be86"
|
||||
"sha256:a8a318824cc77d1fd4b2bec2ded92646630d7fe8619497b142c84a9e6f5a7293",
|
||||
"sha256:f3c5fd51747d450d4dcf6f923c81f78f811aab8205fda64b0aba34a4e48b0745"
|
||||
],
|
||||
"version": "==1.25.6"
|
||||
"version": "==1.25.7"
|
||||
},
|
||||
"validators": {
|
||||
"hashes": [
|
||||
|
|
@ -756,6 +819,13 @@
|
|||
"sha256:565a021fd19419476b9362b05eeaa094178de64f8361e44468f9e9d7843901e1"
|
||||
],
|
||||
"version": "==1.11.2"
|
||||
},
|
||||
"zipp": {
|
||||
"hashes": [
|
||||
"sha256:3718b1cbcd963c7d4c5511a8240812904164b7f381b647143a89d3b98f9bcd8e",
|
||||
"sha256:f06903e9f1f43b12d371004b4ac7b06ab39a44adc747266928ae6debfa7b3335"
|
||||
],
|
||||
"version": "==0.6.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,21 +1,42 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import datetime
|
||||
from dateutil.parser import parse
|
||||
import csv
|
||||
from pathlib import Path
|
||||
import json
|
||||
from uuid import uuid4
|
||||
import requests
|
||||
|
||||
from pymisp import MISPEvent, MISPObject, MISPTag
|
||||
|
||||
from keys import misp_url, misp_key, misp_verifycert
|
||||
from pymisp import ExpandedPyMISP
|
||||
from pymisp import MISPEvent, MISPObject, MISPTag, MISPOrganisation
|
||||
from pymisp.tools import feed_meta_generator
|
||||
|
||||
|
||||
class Scrippts:
|
||||
|
||||
def __init__(self):
|
||||
self.misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert)
|
||||
def __init__(self, output_dir: str= 'output', org_name: str='CIRCL',
|
||||
org_uuid: str='55f6ea5e-2c60-40e5-964f-47a8950d210f'):
|
||||
self.misp_org = MISPOrganisation()
|
||||
self.misp_org.name = org_name
|
||||
self.misp_org.uuid = org_uuid
|
||||
|
||||
self.output_dir = Path(output_dir)
|
||||
self.output_dir.mkdir(exist_ok=True)
|
||||
|
||||
self.data_dir = self.output_dir / 'data'
|
||||
self.data_dir.mkdir(exist_ok=True)
|
||||
|
||||
self.scrippts_meta_file = self.output_dir / '.meta_scrippts'
|
||||
self.scrippts_meta = {}
|
||||
if self.scrippts_meta_file.exists():
|
||||
# Format: <infofield>,<uuid>.json
|
||||
with self.scrippts_meta_file.open() as f:
|
||||
reader = csv.reader(f)
|
||||
for row in reader:
|
||||
self.scrippts_meta[row[0]] = row[1]
|
||||
else:
|
||||
self.scrippts_meta_file.touch()
|
||||
|
||||
def geolocation_alt(self) -> MISPObject:
|
||||
# Alert, NWT, Canada
|
||||
|
|
@ -200,9 +221,7 @@ class Scrippts:
|
|||
return tag
|
||||
|
||||
def fetch(self, url):
|
||||
filepath = Path('scrippts') / Path(url).name
|
||||
if filepath.exists():
|
||||
return filepath
|
||||
filepath = self.data_dir / Path(url).name
|
||||
r = requests.get(url)
|
||||
if r.status_code != 200 or r.text[0] != '"':
|
||||
print(url)
|
||||
|
|
@ -211,42 +230,45 @@ class Scrippts:
|
|||
f.write(r.text)
|
||||
return filepath
|
||||
|
||||
def get_existing_event_to_update(self, infofield):
|
||||
found = self.misp.search(eventinfo=infofield, pythonify=True)
|
||||
if found:
|
||||
event = found[0]
|
||||
return event
|
||||
return False
|
||||
|
||||
def import_all(self, stations_short_names, interval, data_type):
|
||||
object_creator = getattr(self, f'{interval}_flask_{data_type}')
|
||||
if data_type == 'co2':
|
||||
base_url = 'http://scrippsco2.ucsd.edu/assets/data/atmospheric/stations/flask_co2/'
|
||||
base_url = 'https://scrippsco2.ucsd.edu/assets/data/atmospheric/stations/flask_co2/'
|
||||
elif data_type in ['c13', 'o18']:
|
||||
base_url = 'http://scrippsco2.ucsd.edu/assets/data/atmospheric/stations/flask_isotopic/'
|
||||
base_url = 'https://scrippsco2.ucsd.edu/assets/data/atmospheric/stations/flask_isotopic/'
|
||||
for station in stations_short_names:
|
||||
url = f'{base_url}/{interval}/{interval}_flask_{data_type}_{station}.csv'
|
||||
infofield = f'[{station.upper()}] {interval} average atmospheric {data_type} concentrations'
|
||||
filepath = self.fetch(url)
|
||||
if not filepath:
|
||||
continue
|
||||
update = True
|
||||
event = self.get_existing_event_to_update(infofield)
|
||||
if event:
|
||||
location = event.get_objects_by_name('geolocation')[0]
|
||||
if not event:
|
||||
if infofield in self.scrippts_meta:
|
||||
event = MISPEvent()
|
||||
event.load_file(str(self.output_dir / self.scrippts_meta[infofield]))
|
||||
location = event.get_objects_by_name('geolocation')[0]
|
||||
update = True
|
||||
else:
|
||||
event = MISPEvent()
|
||||
event.uuid = str(uuid4())
|
||||
event.info = infofield
|
||||
event.Orgc = self.misp_org
|
||||
event.add_tag(getattr(self, f'tag_{station}')())
|
||||
location = getattr(self, f'geolocation_{station}')()
|
||||
event.add_object(location)
|
||||
event.add_attribute('link', f'http://scrippsco2.ucsd.edu/data/atmospheric_co2/{station}')
|
||||
event.add_attribute('link', f'https://scrippsco2.ucsd.edu/data/atmospheric_co2/{station}')
|
||||
update = False
|
||||
with self.scrippts_meta_file.open('a') as f:
|
||||
writer = csv.writer(f)
|
||||
writer.writerow([infofield, f'{event.uuid}.json'])
|
||||
|
||||
object_creator(event, location, filepath, update)
|
||||
if update:
|
||||
self.misp.update_event(event)
|
||||
else:
|
||||
self.misp.add_event(event)
|
||||
# Bump the publish timestamp
|
||||
event.publish_timestamp = datetime.datetime.timestamp(datetime.datetime.now())
|
||||
feed_output = event.to_feed(with_meta=False)
|
||||
with (self.output_dir / f'{event.uuid}.json').open('w') as f:
|
||||
# json.dump(feed_output, f, indent=2, sort_keys=True) # For testing
|
||||
json.dump(feed_output, f)
|
||||
|
||||
def import_monthly_co2_all(self):
|
||||
to_import = ['alt', 'ptb', 'stp', 'ljo', 'bcs', 'mlo', 'kum', 'chr', 'sam', 'ker', 'nzd']
|
||||
|
|
@ -458,10 +480,14 @@ class Scrippts:
|
|||
|
||||
|
||||
if __name__ == '__main__':
|
||||
i = Scrippts()
|
||||
output_dir = 'scrippsco2_feed'
|
||||
|
||||
i = Scrippts(output_dir=output_dir)
|
||||
i.import_daily_co2_all()
|
||||
i.import_daily_c13_all()
|
||||
i.import_daily_o18_all()
|
||||
i.import_monthly_co2_all()
|
||||
i.import_monthly_c13_all()
|
||||
i.import_monthly_o18_all()
|
||||
|
||||
feed_meta_generator(Path(output_dir))
|
||||
|
|
@ -4,151 +4,36 @@
|
|||
import sys
|
||||
import json
|
||||
import os
|
||||
import hashlib
|
||||
from pymisp import PyMISP
|
||||
from pymisp import ExpandedPyMISP
|
||||
from settings import url, key, ssl, outputdir, filters, valid_attribute_distribution_levels
|
||||
|
||||
objectsFields = {
|
||||
'Attribute': {
|
||||
'uuid',
|
||||
'value',
|
||||
'category',
|
||||
'type',
|
||||
'comment',
|
||||
'data',
|
||||
'timestamp',
|
||||
'to_ids',
|
||||
'object_relation',
|
||||
'disable_correlation'
|
||||
},
|
||||
'Event': {
|
||||
'uuid',
|
||||
'info',
|
||||
'threat_level_id',
|
||||
'analysis',
|
||||
'timestamp',
|
||||
'publish_timestamp',
|
||||
'published',
|
||||
'date',
|
||||
'extends_uuid'
|
||||
},
|
||||
'Object': {
|
||||
'name',
|
||||
'meta-category',
|
||||
'description',
|
||||
'template_uuid',
|
||||
'template_version',
|
||||
'uuid',
|
||||
'timestamp',
|
||||
'distribution',
|
||||
'sharing_group_id',
|
||||
'comment'
|
||||
},
|
||||
'ObjectReference': {
|
||||
'uuid',
|
||||
'timestamp',
|
||||
'relationship_type',
|
||||
'comment',
|
||||
'object_uuid',
|
||||
'referenced_uuid'
|
||||
},
|
||||
'Orgc': {
|
||||
'name',
|
||||
'uuid'
|
||||
},
|
||||
'Tag': {
|
||||
'name',
|
||||
'colour',
|
||||
'exportable'
|
||||
}
|
||||
}
|
||||
|
||||
objectsToSave = {
|
||||
'Orgc': {},
|
||||
'Tag': {},
|
||||
'Attribute': {
|
||||
'Tag': {}
|
||||
},
|
||||
'Object': {
|
||||
'Attribute': {
|
||||
'Tag': {}
|
||||
},
|
||||
'ObjectReference': {}
|
||||
}
|
||||
}
|
||||
|
||||
valid_attribute_distributions = []
|
||||
|
||||
attributeHashes = []
|
||||
|
||||
|
||||
def init():
|
||||
# If we have an old settings.py file then this variable won't exist
|
||||
global valid_attribute_distributions
|
||||
try:
|
||||
valid_attribute_distributions = valid_attribute_distribution_levels
|
||||
valid_attribute_distributions = [int(v) for v in valid_attribute_distribution_levels]
|
||||
except Exception:
|
||||
valid_attribute_distributions = ['0', '1', '2', '3', '4', '5']
|
||||
return PyMISP(url, key, ssl)
|
||||
valid_attribute_distributions = [0, 1, 2, 3, 4, 5]
|
||||
return ExpandedPyMISP(url, key, ssl)
|
||||
|
||||
|
||||
def recursiveExtract(container, containerType, leaf, eventUuid):
|
||||
temp = {}
|
||||
if containerType in ['Attribute', 'Object']:
|
||||
if (__blockByDistribution(container)):
|
||||
return False
|
||||
for field in objectsFields[containerType]:
|
||||
if field in container:
|
||||
temp[field] = container[field]
|
||||
if (containerType == 'Attribute'):
|
||||
global attributeHashes
|
||||
if ('|' in container['type'] or container['type'] == 'malware-sample'):
|
||||
split = container['value'].split('|')
|
||||
attributeHashes.append([hashlib.md5(split[0].encode("utf-8")).hexdigest(), eventUuid])
|
||||
attributeHashes.append([hashlib.md5(split[1].encode("utf-8")).hexdigest(), eventUuid])
|
||||
else:
|
||||
attributeHashes.append([hashlib.md5(container['value'].encode("utf-8")).hexdigest(), eventUuid])
|
||||
children = leaf.keys()
|
||||
for childType in children:
|
||||
childContainer = container.get(childType)
|
||||
if (childContainer):
|
||||
if (type(childContainer) is dict):
|
||||
temp[childType] = recursiveExtract(childContainer, childType, leaf[childType], eventUuid)
|
||||
else:
|
||||
temp[childType] = []
|
||||
for element in childContainer:
|
||||
processed = recursiveExtract(element, childType, leaf[childType], eventUuid)
|
||||
if (processed):
|
||||
temp[childType].append(processed)
|
||||
return temp
|
||||
|
||||
|
||||
def saveEvent(misp, uuid):
|
||||
event = misp.get_event(uuid)
|
||||
if not event.get('Event'):
|
||||
print('Error while fetching event: {}'.format(event['message']))
|
||||
sys.exit('Could not create file for event ' + uuid + '.')
|
||||
event['Event'] = recursiveExtract(event['Event'], 'Event', objectsToSave, event['Event']['uuid'])
|
||||
event = json.dumps(event)
|
||||
eventFile = open(os.path.join(outputdir, uuid + '.json'), 'w')
|
||||
eventFile.write(event)
|
||||
eventFile.close()
|
||||
|
||||
|
||||
def __blockByDistribution(element):
|
||||
if element['distribution'] not in valid_attribute_distributions:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def saveHashes():
|
||||
if not attributeHashes:
|
||||
return False
|
||||
def saveEvent(event):
|
||||
try:
|
||||
hashFile = open(os.path.join(outputdir, 'hashes.csv'), 'w')
|
||||
for element in attributeHashes:
|
||||
hashFile.write('{},{}\n'.format(element[0], element[1]))
|
||||
hashFile.close()
|
||||
with open(os.path.join(outputdir, f'{event["uuid"]}.json'), 'w') as f:
|
||||
json.dump(event, f, indent=2)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
sys.exit('Could not create the event dump.')
|
||||
|
||||
|
||||
def saveHashes(hashes):
|
||||
try:
|
||||
with open(os.path.join(outputdir, 'hashes.csv'), 'w') as hashFile:
|
||||
for element in hashes:
|
||||
hashFile.write('{},{}\n'.format(element[0], element[1]))
|
||||
except Exception as e:
|
||||
print(e)
|
||||
sys.exit('Could not create the quick hash lookup file.')
|
||||
|
|
@ -164,41 +49,31 @@ def saveManifest(manifest):
|
|||
sys.exit('Could not create the manifest file.')
|
||||
|
||||
|
||||
def __addEventToManifest(event):
|
||||
tags = []
|
||||
for eventTag in event['EventTag']:
|
||||
tags.append({'name': eventTag['Tag']['name'],
|
||||
'colour': eventTag['Tag']['colour']})
|
||||
return {'Orgc': event['Orgc'],
|
||||
'Tag': tags,
|
||||
'info': event['info'],
|
||||
'date': event['date'],
|
||||
'analysis': event['analysis'],
|
||||
'threat_level_id': event['threat_level_id'],
|
||||
'timestamp': event['timestamp']
|
||||
}
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
misp = init()
|
||||
try:
|
||||
r = misp.get_index(filters)
|
||||
events = r['response']
|
||||
print(events[0])
|
||||
events = misp.search(metadata=True, limit=200, **filters, pythonify=True)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
sys.exit("Invalid response received from MISP.")
|
||||
if len(events) == 0:
|
||||
sys.exit("No events returned.")
|
||||
manifest = {}
|
||||
hashes = []
|
||||
counter = 1
|
||||
total = len(events)
|
||||
for event in events:
|
||||
saveEvent(misp, event['uuid'])
|
||||
manifest[event['uuid']] = __addEventToManifest(event)
|
||||
e = misp.get_event(event.uuid, pythonify=True)
|
||||
e_feed = e.to_feed(valid_distributions=valid_attribute_distributions, with_meta=True)
|
||||
if not e_feed:
|
||||
print(f'Invalid distribution {e.distribution}, skipping')
|
||||
continue
|
||||
hashes += [[h, e.uuid] for h in e_feed.pop('_hashes')]
|
||||
manifest.update(e_feed.pop('_manifest'))
|
||||
saveEvent(e_feed)
|
||||
print("Event " + str(counter) + "/" + str(total) + " exported.")
|
||||
counter += 1
|
||||
saveManifest(manifest)
|
||||
print('Manifest saved.')
|
||||
saveHashes()
|
||||
saveHashes(hashes)
|
||||
print('Hashes saved. Feed creation completed.')
|
||||
|
|
|
|||
|
|
@ -0,0 +1,15 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from pymisp.tools import feed_meta_generator
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(description='Build meta files for feed')
|
||||
parser.add_argument("--feed", required=True, help="Path to directory containing the feed.")
|
||||
args = parser.parse_args()
|
||||
|
||||
feed = Path(args.feed)
|
||||
|
||||
feed_meta_generator(feed)
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
__version__ = '2.4.117.1'
|
||||
__version__ = '2.4.119.1'
|
||||
import logging
|
||||
import warnings
|
||||
import sys
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ except ImportError:
|
|||
import logging
|
||||
from enum import Enum
|
||||
|
||||
from .exceptions import PyMISPInvalidFormat
|
||||
from .exceptions import PyMISPInvalidFormat, PyMISPError
|
||||
|
||||
|
||||
logger = logging.getLogger('pymisp')
|
||||
|
|
@ -185,7 +185,12 @@ class AbstractMISP(MutableMapping, MISPFileCache):
|
|||
__describe_types = describe_types
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
"""Abstract class for all the MISP objects"""
|
||||
"""Abstract class for all the MISP objects.
|
||||
NOTE: Every method in every classes inheriting this one are doing
|
||||
changes in memory and do not modify data on a remote MISP instance.
|
||||
To do so, you need to call the respective add_* or update_*
|
||||
methods in ExpandedPyMISP/PyMISP.
|
||||
"""
|
||||
super(AbstractMISP, self).__init__()
|
||||
self.__edited = True # As we create a new object, we assume it is edited
|
||||
self.__not_jsonable = []
|
||||
|
|
@ -282,6 +287,27 @@ class AbstractMISP(MutableMapping, MISPFileCache):
|
|||
"""This method is used by the JSON encoder"""
|
||||
return self.to_dict()
|
||||
|
||||
def _to_feed(self):
|
||||
if not hasattr(self, '_fields_for_feed'):
|
||||
raise PyMISPError('Unable to export in the feed format, _fields_for_feed is missing.')
|
||||
if hasattr(self, '_set_default') and callable(self._set_default):
|
||||
self._set_default()
|
||||
to_return = {}
|
||||
for field in self._fields_for_feed:
|
||||
if getattr(self, field, None) is not None:
|
||||
if field in ['timestamp', 'publish_timestamp']:
|
||||
to_return[field] = self._datetime_to_timestamp(getattr(self, field))
|
||||
elif isinstance(getattr(self, field), (datetime.datetime, datetime.date)):
|
||||
to_return[field] = getattr(self, field).isoformat()
|
||||
else:
|
||||
to_return[field] = getattr(self, field)
|
||||
else:
|
||||
if field == 'data':
|
||||
# data in attribute is special
|
||||
continue
|
||||
raise PyMISPError('The field {} is required in {} when generating a feed.'.format(field, self.__class__.__name__))
|
||||
return to_return
|
||||
|
||||
def to_json(self, sort_keys=False, indent=None):
|
||||
"""Dump recursively any class of type MISPAbstract to a json string"""
|
||||
return dumps(self, default=pymisp_json_default, sort_keys=sort_keys, indent=indent)
|
||||
|
|
@ -329,7 +355,7 @@ class AbstractMISP(MutableMapping, MISPFileCache):
|
|||
if isinstance(val, bool):
|
||||
self.__edited = val
|
||||
else:
|
||||
raise Exception('edited can only be True or False')
|
||||
raise PyMISPError('edited can only be True or False')
|
||||
|
||||
def __setattr__(self, name, value):
|
||||
if name[0] != '_' and not self.__edited and name in self.keys():
|
||||
|
|
@ -340,7 +366,7 @@ class AbstractMISP(MutableMapping, MISPFileCache):
|
|||
|
||||
def _datetime_to_timestamp(self, d):
|
||||
"""Convert a datetime.datetime object to a timestamp (int)"""
|
||||
if isinstance(d, (int, str)) or (sys.version_info < (3, 0) and isinstance(d, unicode)):
|
||||
if isinstance(d, (int, float, str)) or (sys.version_info < (3, 0) and isinstance(d, unicode)):
|
||||
# Assume we already have a timestamp
|
||||
return int(d)
|
||||
if sys.version_info >= (3, 3):
|
||||
|
|
@ -393,6 +419,9 @@ class AbstractMISP(MutableMapping, MISPFileCache):
|
|||
|
||||
|
||||
class MISPTag(AbstractMISP):
|
||||
|
||||
_fields_for_feed = {'name', 'colour'}
|
||||
|
||||
def __init__(self):
|
||||
super(MISPTag, self).__init__()
|
||||
|
||||
|
|
@ -400,3 +429,12 @@ class MISPTag(AbstractMISP):
|
|||
if kwargs.get('Tag'):
|
||||
kwargs = kwargs.get('Tag')
|
||||
super(MISPTag, self).from_dict(**kwargs)
|
||||
|
||||
def _set_default(self):
|
||||
if not hasattr(self, 'colour'):
|
||||
self.colour = '#ffffff'
|
||||
|
||||
def _to_feed(self):
|
||||
if hasattr(self, 'exportable') and not self.exportable:
|
||||
return False
|
||||
return super(MISPTag, self)._to_feed()
|
||||
|
|
|
|||
|
|
@ -179,6 +179,10 @@ class ExpandedPyMISP(PyMISP):
|
|||
response = self._prepare_request('POST', f'/servers/restartWorkers')
|
||||
return self._check_response(response, expect_json=True)
|
||||
|
||||
def db_schema_diagnostic(self):
|
||||
response = self._prepare_request('GET', f'/servers/dbSchemaDiagnostic')
|
||||
return self._check_response(response, expect_json=True)
|
||||
|
||||
def toggle_global_pythonify(self):
|
||||
self.global_pythonify = not self.global_pythonify
|
||||
|
||||
|
|
@ -620,7 +624,11 @@ class ExpandedPyMISP(PyMISP):
|
|||
return t
|
||||
|
||||
def add_tag(self, tag: MISPTag, pythonify: bool=False):
|
||||
'''Add a new tag on a MISP instance'''
|
||||
'''Add a new tag on a MISP instance
|
||||
Notes:
|
||||
* The user calling this method needs the Tag Editor permission
|
||||
* It doesn't add a tag to an event, simply create it on a MISP instance.
|
||||
'''
|
||||
new_tag = self._prepare_request('POST', 'tags/add', data=tag)
|
||||
new_tag = self._check_response(new_tag, expect_json=True)
|
||||
if not (self.global_pythonify or pythonify) or 'errors' in new_tag:
|
||||
|
|
@ -1891,7 +1899,7 @@ class ExpandedPyMISP(PyMISP):
|
|||
if adhereToWarninglists in wl_params:
|
||||
query['adhereToWarninglists'] = adhereToWarninglists
|
||||
else:
|
||||
raise Exception('Invalid parameter, adhereToWarninglists Can only be {}'.format(', '.join(wl_params)))
|
||||
raise PyMISPError('Invalid parameter, adhereToWarninglists Can only be {}'.format(', '.join(wl_params)))
|
||||
if distribution is not None:
|
||||
query['distribution'] = distribution
|
||||
if returnMetaAttributes:
|
||||
|
|
@ -2023,7 +2031,6 @@ class ExpandedPyMISP(PyMISP):
|
|||
response = self._prepare_request('POST', f'user_settings/delete', data=query)
|
||||
return self._check_response(response, expect_json=True)
|
||||
|
||||
|
||||
# ## END User Settings ###
|
||||
|
||||
# ## BEGIN Global helpers ###
|
||||
|
|
@ -2045,22 +2052,26 @@ class ExpandedPyMISP(PyMISP):
|
|||
|
||||
raise PyMISPError('The misp_entity must be MISPEvent, MISPObject or MISPAttribute')
|
||||
|
||||
def tag(self, misp_entity: Union[AbstractMISP, str], tag: str, local: bool=False):
|
||||
"""Tag an event or an attribute. misp_entity can be a UUID"""
|
||||
def tag(self, misp_entity: Union[AbstractMISP, str], tag: Union[MISPTag, str], local: bool=False):
|
||||
"""Tag an event or an attribute. misp_entity can be a MISPEvent, a MISP Attribute, or a UUID"""
|
||||
if 'uuid' in misp_entity:
|
||||
uuid = misp_entity.uuid
|
||||
else:
|
||||
uuid = misp_entity
|
||||
if isinstance(tag, MISPTag):
|
||||
tag = tag.name
|
||||
to_post = {'uuid': uuid, 'tag': tag, 'local': local}
|
||||
response = self._prepare_request('POST', 'tags/attachTagToObject', data=to_post)
|
||||
return self._check_response(response, expect_json=True)
|
||||
|
||||
def untag(self, misp_entity: Union[AbstractMISP, str], tag: str):
|
||||
def untag(self, misp_entity: Union[AbstractMISP, str], tag: Union[MISPTag, str]):
|
||||
"""Untag an event or an attribute. misp_entity can be a UUID"""
|
||||
if 'uuid' in misp_entity:
|
||||
uuid = misp_entity.uuid
|
||||
else:
|
||||
uuid = misp_entity
|
||||
if isinstance(tag, MISPTag):
|
||||
tag = tag.name
|
||||
to_post = {'uuid': uuid, 'tag': tag}
|
||||
response = self._prepare_request('POST', 'tags/removeTagFromObject', data=to_post)
|
||||
return self._check_response(response, expect_json=True)
|
||||
|
|
@ -2188,10 +2199,6 @@ class ExpandedPyMISP(PyMISP):
|
|||
kw_params: dict={}, output_type: str='json'):
|
||||
'''Prepare a request for python-requests'''
|
||||
url = urljoin(self.root_url, url)
|
||||
if logger.isEnabledFor(logging.DEBUG):
|
||||
logger.debug(f'{request_type} - {url}')
|
||||
if data is not None:
|
||||
logger.debug(data)
|
||||
if data:
|
||||
if not isinstance(data, str): # Else, we already have a text blob to send
|
||||
if isinstance(data, dict): # Else, we can directly json encode.
|
||||
|
|
@ -2199,6 +2206,11 @@ class ExpandedPyMISP(PyMISP):
|
|||
data = {k: v for k, v in data.items() if v is not None}
|
||||
data = json.dumps(data, default=pymisp_json_default)
|
||||
|
||||
if logger.isEnabledFor(logging.DEBUG):
|
||||
logger.debug(f'{request_type} - {url}')
|
||||
if data is not None:
|
||||
logger.debug(data)
|
||||
|
||||
if kw_params:
|
||||
# CakePHP params in URL
|
||||
to_append_url = '/'.join([f'{k}:{v}' for k, v in kw_params.items()])
|
||||
|
|
|
|||
|
|
@ -187,7 +187,9 @@
|
|||
"domain",
|
||||
"domain|ip",
|
||||
"email-dst",
|
||||
"email-src",
|
||||
"email-subject",
|
||||
"eppn",
|
||||
"hassh-md5",
|
||||
"hasshserver-md5",
|
||||
"hex",
|
||||
|
|
@ -418,6 +420,7 @@
|
|||
"comment",
|
||||
"email-dst",
|
||||
"email-src",
|
||||
"eppn",
|
||||
"github-organisation",
|
||||
"github-repository",
|
||||
"github-username",
|
||||
|
|
@ -612,6 +615,10 @@
|
|||
"default_category": "Payload delivery",
|
||||
"to_ids": 0
|
||||
},
|
||||
"eppn": {
|
||||
"default_category": "Network activity",
|
||||
"to_ids": 1
|
||||
},
|
||||
"filename": {
|
||||
"default_category": "Payload delivery",
|
||||
"to_ids": 1
|
||||
|
|
@ -1135,6 +1142,7 @@
|
|||
"email-subject",
|
||||
"email-thread-index",
|
||||
"email-x-mailer",
|
||||
"eppn",
|
||||
"filename",
|
||||
"filename|authentihash",
|
||||
"filename|impfuzzy",
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
Subproject commit 58d6722f5e276a0ec5889e6e67316b7401960542
|
||||
Subproject commit 33a7d6b574b7354ba8243ba461bfb30db0528023
|
||||
|
|
@ -1,4 +1,3 @@
|
|||
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import datetime
|
||||
|
|
@ -11,6 +10,7 @@ import sys
|
|||
import uuid
|
||||
from collections import defaultdict
|
||||
import logging
|
||||
import hashlib
|
||||
|
||||
from deprecated import deprecated
|
||||
|
||||
|
|
@ -99,10 +99,12 @@ def make_bool(value):
|
|||
return False
|
||||
return True
|
||||
else:
|
||||
raise Exception('Unable to convert {} to a boolean.'.format(value))
|
||||
raise PyMISPError('Unable to convert {} to a boolean.'.format(value))
|
||||
|
||||
|
||||
class MISPAttribute(AbstractMISP):
|
||||
_fields_for_feed = {'uuid', 'value', 'category', 'type', 'comment', 'data',
|
||||
'timestamp', 'to_ids', 'disable_correlation'}
|
||||
|
||||
def __init__(self, describe_types=None, strict=False):
|
||||
"""Represents an Attribute
|
||||
|
|
@ -121,6 +123,39 @@ class MISPAttribute(AbstractMISP):
|
|||
self.ShadowAttribute = []
|
||||
self.Sighting = []
|
||||
|
||||
def hash_values(self, algorithm='sha512'):
|
||||
"""Compute the hash of every values for fast lookups"""
|
||||
if algorithm not in hashlib.algorithms_available:
|
||||
raise PyMISPError('The algorithm {} is not available for hashing.'.format(algorithm))
|
||||
if '|' in self.type or self.type == 'malware-sample':
|
||||
hashes = []
|
||||
for v in self.value.split('|'):
|
||||
h = hashlib.new(algorithm)
|
||||
h.update(v.encode("utf-8"))
|
||||
hashes.append(h.hexdigest())
|
||||
return hashes
|
||||
else:
|
||||
h = hashlib.new(algorithm)
|
||||
to_encode = self.value
|
||||
if not isinstance(to_encode, str):
|
||||
to_encode = str(to_encode)
|
||||
h.update(to_encode.encode("utf-8"))
|
||||
return [h.hexdigest()]
|
||||
|
||||
def _set_default(self):
|
||||
if not hasattr(self, 'comment'):
|
||||
self.comment = ''
|
||||
if not hasattr(self, 'timestamp'):
|
||||
self.timestamp = datetime.datetime.timestamp(datetime.datetime.now())
|
||||
|
||||
def _to_feed(self):
|
||||
to_return = super(MISPAttribute, self)._to_feed()
|
||||
if self.data:
|
||||
to_return['data'] = base64.b64encode(self.data.getvalue()).decode()
|
||||
if self.tags:
|
||||
to_return['Tag'] = list(filter(None, [tag._to_feed() for tag in self.tags]))
|
||||
return to_return
|
||||
|
||||
@property
|
||||
def known_types(self):
|
||||
"""Returns a list of all the known MISP attributes types"""
|
||||
|
|
@ -349,7 +384,7 @@ class MISPAttribute(AbstractMISP):
|
|||
try:
|
||||
with ZipFile(self.data) as f:
|
||||
if not self.__is_misp_encrypted_file(f):
|
||||
raise Exception('Not an existing malware sample')
|
||||
raise PyMISPError('Not an existing malware sample')
|
||||
for name in f.namelist():
|
||||
if name.endswith('.filename.txt'):
|
||||
with f.open(name, pwd=b'infected') as unpacked:
|
||||
|
|
@ -421,6 +456,9 @@ class MISPAttribute(AbstractMISP):
|
|||
|
||||
class MISPEvent(AbstractMISP):
|
||||
|
||||
_fields_for_feed = {'uuid', 'info', 'threat_level_id', 'analysis', 'timestamp',
|
||||
'publish_timestamp', 'published', 'date', 'extends_uuid'}
|
||||
|
||||
def __init__(self, describe_types=None, strict_validation=False, **kwargs):
|
||||
super(MISPEvent, self).__init__(**kwargs)
|
||||
if strict_validation:
|
||||
|
|
@ -440,6 +478,105 @@ class MISPEvent(AbstractMISP):
|
|||
self.RelatedEvent = []
|
||||
self.ShadowAttribute = []
|
||||
|
||||
def _set_default(self):
|
||||
"""There are a few keys that could, or need to be set by default for the feed generator"""
|
||||
if not hasattr(self, 'published'):
|
||||
self.published = True
|
||||
if not hasattr(self, 'uuid'):
|
||||
self.uuid = str(uuid.uuid4())
|
||||
if not hasattr(self, 'extends_uuid'):
|
||||
self.extends_uuid = ''
|
||||
if not hasattr(self, 'date'):
|
||||
self.set_date(datetime.date.today())
|
||||
if not hasattr(self, 'timestamp'):
|
||||
self.timestamp = datetime.datetime.timestamp(datetime.datetime.now())
|
||||
if not hasattr(self, 'publish_timestamp'):
|
||||
self.publish_timestamp = datetime.datetime.timestamp(datetime.datetime.now())
|
||||
if not hasattr(self, 'analysis'):
|
||||
# analysis: 0 means initial, 1 ongoing, 2 completed
|
||||
self.analysis = 2
|
||||
if not hasattr(self, 'threat_level_id'):
|
||||
# threat_level_id 4 means undefined. Tags are recommended.
|
||||
self.threat_level_id = 4
|
||||
|
||||
@property
|
||||
def manifest(self):
|
||||
required = ['info', 'Orgc']
|
||||
for r in required:
|
||||
if not hasattr(self, r):
|
||||
raise PyMISPError('The field {} is required to generate the event manifest.')
|
||||
|
||||
self._set_default()
|
||||
|
||||
return {
|
||||
self.uuid: {
|
||||
'Orgc': self.Orgc._to_feed(),
|
||||
'Tag': list(filter(None, [tag._to_feed() for tag in self.tags])),
|
||||
'info': self.info,
|
||||
'date': self.date.isoformat(),
|
||||
'analysis': self.analysis,
|
||||
'threat_level_id': self.threat_level_id,
|
||||
'timestamp': self._datetime_to_timestamp(self.timestamp)
|
||||
}
|
||||
}
|
||||
|
||||
def attributes_hashes(self, algorithm='sha512'):
|
||||
to_return = []
|
||||
for attribute in self.attributes:
|
||||
to_return += attribute.hash_values(algorithm)
|
||||
for obj in self.objects:
|
||||
for attribute in obj.attributes:
|
||||
to_return += attribute.hash_values(algorithm)
|
||||
return to_return
|
||||
|
||||
def to_feed(self, valid_distributions=[0, 1, 2, 3, 4, 5], with_meta=False):
|
||||
""" Generate a json output for MISP Feed.
|
||||
Notes:
|
||||
* valid_distributions only makes sense if the distribution key is set (i.e. the event is exported from a MISP instance)
|
||||
"""
|
||||
required = ['info', 'Orgc']
|
||||
for r in required:
|
||||
if not hasattr(self, r):
|
||||
raise PyMISPError('The field {} is required to generate the event feed output.')
|
||||
|
||||
if (hasattr(self, 'distribution')
|
||||
and self.distribution is not None
|
||||
and int(self.distribution) not in valid_distributions):
|
||||
return
|
||||
|
||||
to_return = super(MISPEvent, self)._to_feed()
|
||||
if with_meta:
|
||||
to_return['_hashes'] = []
|
||||
to_return['_manifest'] = self.manifest
|
||||
|
||||
to_return['Orgc'] = self.Orgc._to_feed()
|
||||
to_return['Tag'] = list(filter(None, [tag._to_feed() for tag in self.tags]))
|
||||
if self.attributes:
|
||||
to_return['Attribute'] = []
|
||||
for attribute in self.attributes:
|
||||
if (valid_distributions and attribute.get('distribution') is not None and attribute.distribution not in valid_distributions):
|
||||
continue
|
||||
to_return['Attribute'].append(attribute._to_feed())
|
||||
if with_meta:
|
||||
to_return['_hashes'] += attribute.hash_values('md5')
|
||||
|
||||
if self.objects:
|
||||
to_return['Object'] = []
|
||||
for obj in self.objects:
|
||||
if (valid_distributions and obj.get('distribution') is not None and obj.distribution not in valid_distributions):
|
||||
continue
|
||||
obj_to_attach = obj._to_feed()
|
||||
obj_to_attach['Attribute'] = []
|
||||
for attribute in obj.attributes:
|
||||
if (valid_distributions and attribute.get('distribution') is not None and attribute.distribution not in valid_distributions):
|
||||
continue
|
||||
obj_to_attach['Attribute'].append(attribute._to_feed())
|
||||
if with_meta:
|
||||
to_return['_hashes'] += attribute.hash_values('md5')
|
||||
to_return['Object'].append(obj_to_attach)
|
||||
|
||||
return {'Event': to_return}
|
||||
|
||||
@property
|
||||
def known_types(self):
|
||||
return self.describe_types['types']
|
||||
|
|
@ -496,14 +633,14 @@ class MISPEvent(AbstractMISP):
|
|||
else:
|
||||
raise PyMISPError('All the attributes have to be of type MISPObject.')
|
||||
|
||||
def load_file(self, event_path):
|
||||
def load_file(self, event_path, validate=False, metadata_only=False):
|
||||
"""Load a JSON dump from a file on the disk"""
|
||||
if not os.path.exists(event_path):
|
||||
raise PyMISPError('Invalid path, unable to load the event.')
|
||||
with open(event_path, 'rb') as f:
|
||||
self.load(f)
|
||||
self.load(f, validate, metadata_only)
|
||||
|
||||
def load(self, json_event, validate=False):
|
||||
def load(self, json_event, validate=False, metadata_only=False):
|
||||
"""Load a JSON dump from a pseudo file or a JSON string"""
|
||||
if hasattr(json_event, 'read'):
|
||||
# python2 and python3 compatible to find if we have a file
|
||||
|
|
@ -518,6 +655,9 @@ class MISPEvent(AbstractMISP):
|
|||
event = json_event
|
||||
if not event:
|
||||
raise PyMISPError('Invalid event')
|
||||
if metadata_only:
|
||||
event.pop('Attribute', None)
|
||||
event.pop('Object', None)
|
||||
self.from_dict(**event)
|
||||
if validate:
|
||||
jsonschema.validate(json.loads(self.to_json()), self.__json_schema)
|
||||
|
|
@ -591,6 +731,11 @@ class MISPEvent(AbstractMISP):
|
|||
self.publish_timestamp = datetime.datetime.fromtimestamp(int(kwargs.pop('publish_timestamp')), datetime.timezone.utc)
|
||||
else:
|
||||
self.publish_timestamp = datetime.datetime.fromtimestamp(int(kwargs.pop('publish_timestamp')), UTC())
|
||||
if kwargs.get('sighting_timestamp'):
|
||||
if sys.version_info >= (3, 3):
|
||||
self.sighting_timestamp = datetime.datetime.fromtimestamp(int(kwargs.pop('sighting_timestamp')), datetime.timezone.utc)
|
||||
else:
|
||||
self.sighting_timestamp = datetime.datetime.fromtimestamp(int(kwargs.pop('sighting_timestamp')), UTC())
|
||||
if kwargs.get('sharing_group_id'):
|
||||
self.sharing_group_id = int(kwargs.pop('sharing_group_id'))
|
||||
if kwargs.get('RelatedEvent'):
|
||||
|
|
@ -620,6 +765,8 @@ class MISPEvent(AbstractMISP):
|
|||
to_return['date'] = self.date.isoformat()
|
||||
if to_return.get('publish_timestamp'):
|
||||
to_return['publish_timestamp'] = self._datetime_to_timestamp(self.publish_timestamp)
|
||||
if to_return.get('sighting_timestamp'):
|
||||
to_return['sighting_timestamp'] = self._datetime_to_timestamp(self.sighting_timestamp)
|
||||
|
||||
return to_return
|
||||
|
||||
|
|
@ -671,7 +818,7 @@ class MISPEvent(AbstractMISP):
|
|||
attributes.append(a)
|
||||
|
||||
if not attributes:
|
||||
raise Exception('No attribute with identifier {} found.'.format(attribute_identifier))
|
||||
raise PyMISPError('No attribute with identifier {} found.'.format(attribute_identifier))
|
||||
self.edited = True
|
||||
return attributes
|
||||
|
||||
|
|
@ -693,7 +840,7 @@ class MISPEvent(AbstractMISP):
|
|||
found = True
|
||||
break
|
||||
if not found:
|
||||
raise Exception('No attribute with UUID/ID {} found.'.format(attribute_id))
|
||||
raise PyMISPError('No attribute with UUID/ID {} found.'.format(attribute_id))
|
||||
|
||||
def add_attribute(self, type, value, **kwargs):
|
||||
"""Add an attribute. type and value are required but you can pass all
|
||||
|
|
@ -857,8 +1004,18 @@ class MISPEvent(AbstractMISP):
|
|||
|
||||
class MISPObjectReference(AbstractMISP):
|
||||
|
||||
_fields_for_feed = {'uuid', 'timestamp', 'relationship_type', 'comment',
|
||||
'object_uuid', 'referenced_uuid'}
|
||||
|
||||
def __init__(self):
|
||||
super(MISPObjectReference, self).__init__()
|
||||
self.uuid = str(uuid.uuid4())
|
||||
|
||||
def _set_default(self):
|
||||
if not hasattr(self, 'comment'):
|
||||
self.comment = ''
|
||||
if not hasattr(self, 'timestamp'):
|
||||
self.timestamp = datetime.datetime.timestamp(datetime.datetime.now())
|
||||
|
||||
def from_dict(self, **kwargs):
|
||||
if 'ObjectReference' in kwargs:
|
||||
|
|
@ -891,6 +1048,8 @@ class MISPUser(AbstractMISP):
|
|||
if 'User' in kwargs:
|
||||
kwargs = kwargs['User']
|
||||
super(MISPUser, self).from_dict(**kwargs)
|
||||
if hasattr(self, 'password') and set(self.password) == set(['*']):
|
||||
self.password = None
|
||||
|
||||
def __repr__(self):
|
||||
if hasattr(self, 'email'):
|
||||
|
|
@ -900,6 +1059,8 @@ class MISPUser(AbstractMISP):
|
|||
|
||||
class MISPOrganisation(AbstractMISP):
|
||||
|
||||
_fields_for_feed = {'name', 'uuid'}
|
||||
|
||||
def __init__(self):
|
||||
super(MISPOrganisation, self).__init__()
|
||||
|
||||
|
|
@ -1056,6 +1217,9 @@ class MISPSighting(AbstractMISP):
|
|||
|
||||
class MISPObjectAttribute(MISPAttribute):
|
||||
|
||||
_fields_for_feed = {'uuid', 'object_relation', 'value', 'category', 'type',
|
||||
'comment', 'data', 'timestamp', 'to_ids', 'disable_correlation'}
|
||||
|
||||
def __init__(self, definition):
|
||||
super(MISPObjectAttribute, self).__init__()
|
||||
self._definition = definition
|
||||
|
|
@ -1140,6 +1304,10 @@ class MISPUserSetting(AbstractMISP):
|
|||
|
||||
class MISPObject(AbstractMISP):
|
||||
|
||||
_fields_for_feed = {'name', 'meta-category', 'description', 'template_uuid',
|
||||
'template_version', 'uuid', 'timestamp', 'distribution',
|
||||
'sharing_group_id', 'comment'}
|
||||
|
||||
def __init__(self, name, strict=False, standalone=False, default_attributes_parameters={}, **kwargs):
|
||||
''' Master class representing a generic MISP object
|
||||
:name: Name of the object
|
||||
|
|
@ -1203,6 +1371,18 @@ class MISPObject(AbstractMISP):
|
|||
self.template_version = self._definition['version']
|
||||
return True
|
||||
|
||||
def _set_default(self):
|
||||
if not hasattr(self, 'comment'):
|
||||
self.comment = ''
|
||||
if not hasattr(self, 'timestamp'):
|
||||
self.timestamp = datetime.datetime.timestamp(datetime.datetime.now())
|
||||
|
||||
def _to_feed(self):
|
||||
to_return = super(MISPObject, self)._to_feed()
|
||||
if self.references:
|
||||
to_return['ObjectReference'] = [reference._to_feed() for reference in self.references]
|
||||
return to_return
|
||||
|
||||
def force_misp_objects_path_custom(self, misp_objects_path_custom, object_name=None):
|
||||
if object_name:
|
||||
self.name = object_name
|
||||
|
|
@ -1309,6 +1489,7 @@ class MISPObject(AbstractMISP):
|
|||
relationship_type=relationship_type, comment=comment, **kwargs)
|
||||
self.ObjectReference.append(reference)
|
||||
self.edited = True
|
||||
return reference
|
||||
|
||||
def get_attributes_by_relation(self, object_relation):
|
||||
'''Returns the list of attributes with the given object relation in the object'''
|
||||
|
|
@ -1370,10 +1551,10 @@ class MISPObject(AbstractMISP):
|
|||
self._validate()
|
||||
return super(MISPObject, self).to_dict()
|
||||
|
||||
def to_json(self, strict=False):
|
||||
def to_json(self, strict=False, sort_keys=False, indent=None):
|
||||
if strict or self._strict and self._known_template:
|
||||
self._validate()
|
||||
return super(MISPObject, self).to_json()
|
||||
return super(MISPObject, self).to_json(sort_keys=sort_keys, indent=indent)
|
||||
|
||||
def _validate(self):
|
||||
"""Make sure the object we're creating has the required fields"""
|
||||
|
|
|
|||
|
|
@ -21,3 +21,9 @@ if sys.version_info >= (3, 6):
|
|||
from .vehicleobject import VehicleObject # noqa
|
||||
from .csvloader import CSVLoader # noqa
|
||||
from .sshauthkeyobject import SSHAuthorizedKeysObject # noqa
|
||||
from .feed import feed_meta_generator # noqa
|
||||
try:
|
||||
from .urlobject import URLObject # noqa
|
||||
except ImportError:
|
||||
# Requires faup, which is a bit difficult to install
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -50,17 +50,30 @@ class EMailObject(AbstractMISPObjectGenerator):
|
|||
if 'Message-ID' in self.__email:
|
||||
self.add_attribute('message-id', value=self.__email['Message-ID'])
|
||||
if 'To' in self.__email:
|
||||
# TODO: split name and email address
|
||||
to_add = [to.strip() for to in self.__email['To'].split(',')]
|
||||
self.add_attributes('to', *to_add)
|
||||
if 'Cc' in self.__email:
|
||||
# TODO: split name and email address
|
||||
to_add = [to.strip() for to in self.__email['Cc'].split(',')]
|
||||
self.add_attributes('cc', *to_add)
|
||||
if 'Subject' in self.__email:
|
||||
self.add_attribute('subject', value=self.__email['Subject'])
|
||||
if 'From' in self.__email:
|
||||
# TODO: split name and email address
|
||||
to_add = [to.strip() for to in self.__email['From'].split(',')]
|
||||
self.add_attributes('from', *to_add)
|
||||
if 'Return-Path' in self.__email:
|
||||
# TODO: split name and email address
|
||||
self.add_attribute('return-path', value=self.__email['Return-Path'])
|
||||
if 'User-Agent' in self.__email:
|
||||
self.add_attribute('user-agent', value=self.__email['User-Agent'])
|
||||
if self.__email.get_boundary():
|
||||
self.add_attribute('mime-boundary', value=self.__email.get_boundary())
|
||||
if 'X-Mailer' in self.__email:
|
||||
self.add_attribute('x-mailer', value=self.__email['X-Mailer'])
|
||||
if 'Thread-Index' in self.__email:
|
||||
self.add_attribute('thread-index', value=self.__email['Thread-Index'])
|
||||
# TODO: email-header: all headers in one bloc
|
||||
# TODO: BCC?
|
||||
# TODO: received headers sometimes have TO email addresses
|
||||
|
|
|
|||
|
|
@ -0,0 +1,26 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from pathlib import Path
|
||||
from pymisp import MISPEvent
|
||||
import json
|
||||
|
||||
|
||||
def feed_meta_generator(path: Path):
|
||||
manifests = {}
|
||||
hashes = []
|
||||
|
||||
for f_name in path.glob('*.json'):
|
||||
if str(f_name.name) == 'manifest.json':
|
||||
continue
|
||||
event = MISPEvent()
|
||||
event.load_file(str(f_name))
|
||||
manifests.update(event.manifest)
|
||||
hashes += [f'{h},{event.uuid}' for h in event.attributes_hashes('md5')]
|
||||
|
||||
with (path / 'manifest.json').open('w') as f:
|
||||
json.dump(manifests, f)
|
||||
|
||||
with (path / 'hashes.csv').open('w') as f:
|
||||
for h in hashes:
|
||||
f.write(f'{h}\n')
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from .abstractgenerator import AbstractMISPObjectGenerator
|
||||
import logging
|
||||
from pyfaup.faup import Faup
|
||||
from urllib.parse import unquote_plus
|
||||
|
||||
logger = logging.getLogger('pymisp')
|
||||
|
||||
faup = Faup()
|
||||
|
||||
|
||||
class URLObject(AbstractMISPObjectGenerator):
|
||||
|
||||
def __init__(self, url, standalone=True, **kwargs):
|
||||
# PY3 way:
|
||||
# super().__init__('file')
|
||||
super(URLObject, self).__init__('url', standalone=standalone, **kwargs)
|
||||
faup.decode(unquote_plus(url))
|
||||
self.generate_attributes()
|
||||
|
||||
def generate_attributes(self):
|
||||
self.add_attribute('url', value=faup.url.decode())
|
||||
if faup.get_host():
|
||||
self.add_attribute('host', value=faup.get_host())
|
||||
if faup.get_domain():
|
||||
self.add_attribute('domain', value=faup.get_domain())
|
||||
2
setup.py
2
setup.py
|
|
@ -42,7 +42,7 @@ setup(
|
|||
install_requires=['six', 'requests', 'python-dateutil', 'jsonschema',
|
||||
'python-dateutil', 'enum34;python_version<"3.4"',
|
||||
'functools32;python_version<"3.0"', 'deprecated', 'cachetools;python_version<"3.0"'],
|
||||
extras_require={'fileobjects': ['lief>=0.8,<0.10;python_version<"3.5"', 'lief>=0.10.0.dev0;python_version>"3.5"', 'python-magic', 'pydeep'],
|
||||
extras_require={'fileobjects': ['lief>=0.8,<0.10;python_version<"3.5"', 'lief>=0.10.1;python_version>"3.5"', 'python-magic', 'pydeep'],
|
||||
'neo': ['py2neo'],
|
||||
'openioc': ['beautifulsoup4'],
|
||||
'virustotal': ['validators'],
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@
|
|||
"name": "file",
|
||||
"sharing_group_id": "0",
|
||||
"template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215",
|
||||
"template_version": "17",
|
||||
"template_version": "18",
|
||||
"uuid": "a"
|
||||
},
|
||||
{
|
||||
|
|
@ -50,7 +50,7 @@
|
|||
"name": "url",
|
||||
"sharing_group_id": "0",
|
||||
"template_uuid": "60efb77b-40b5-4c46-871b-ed1ed999fce5",
|
||||
"template_version": "7",
|
||||
"template_version": "8",
|
||||
"uuid": "b"
|
||||
}
|
||||
]
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@
|
|||
"name": "file",
|
||||
"sharing_group_id": "0",
|
||||
"template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215",
|
||||
"template_version": "17",
|
||||
"template_version": "18",
|
||||
"uuid": "a"
|
||||
},
|
||||
{
|
||||
|
|
@ -55,7 +55,7 @@
|
|||
"name": "file",
|
||||
"sharing_group_id": "0",
|
||||
"template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215",
|
||||
"template_version": "17",
|
||||
"template_version": "18",
|
||||
"uuid": "b"
|
||||
}
|
||||
]
|
||||
|
|
|
|||
|
|
@ -449,9 +449,9 @@
|
|||
]
|
||||
}
|
||||
},
|
||||
"version": 17,
|
||||
"version": 1,
|
||||
"description": "File object describing a file with meta-information",
|
||||
"meta-category": "file",
|
||||
"uuid": "688c46fb-5edb-40a3-8273-1af7923e2215",
|
||||
"name": "file"
|
||||
"uuid": "688c46fb-5edb-40a3-8273-1af7923e0000",
|
||||
"name": "overwrite_file"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import unittest
|
|||
import json
|
||||
import sys
|
||||
from io import BytesIO
|
||||
import glob
|
||||
|
||||
from pymisp import MISPEvent, MISPSighting, MISPTag
|
||||
from pymisp.exceptions import InvalidMISPObject
|
||||
|
|
@ -86,7 +87,8 @@ class TestMISPEvent(unittest.TestCase):
|
|||
del a.uuid
|
||||
self.mispevent.objects[0].uuid = 'a'
|
||||
self.mispevent.objects[1].uuid = 'b'
|
||||
self.mispevent.objects[0].add_reference(self.mispevent.objects[1], 'baz', comment='foo')
|
||||
reference = self.mispevent.objects[0].add_reference(self.mispevent.objects[1], 'baz', comment='foo')
|
||||
del reference.uuid
|
||||
self.assertEqual(self.mispevent.objects[0].references[0].relationship_type, 'baz')
|
||||
with open('tests/mispevent_testfiles/event_obj_attr_tag.json', 'r') as f:
|
||||
ref_json = json.load(f)
|
||||
|
|
@ -291,6 +293,28 @@ class TestMISPEvent(unittest.TestCase):
|
|||
ref_json = json.load(f)
|
||||
self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2))
|
||||
|
||||
'''
|
||||
# Reenable that the 1st of jan 2020.
|
||||
@unittest.skipIf(sys.version_info < (3, 6), 'Not supported on python < 3.6')
|
||||
def test_object_templates(self):
|
||||
me = MISPEvent()
|
||||
for template in glob.glob(str(me.misp_objects_path / '*' / 'definition.json')):
|
||||
with open(template) as f:
|
||||
t_json = json.load(f)
|
||||
if 'requiredOneOf' in t_json:
|
||||
obj_relations = set(t_json['attributes'].keys())
|
||||
subset = set(t_json['requiredOneOf']).issubset(obj_relations)
|
||||
self.assertTrue(subset, f'{t_json["name"]}')
|
||||
if 'required' in t_json:
|
||||
obj_relations = set(t_json['attributes'].keys())
|
||||
subset = set(t_json['required']).issubset(obj_relations)
|
||||
self.assertTrue(subset, f'{t_json["name"]}')
|
||||
for obj_relation, entry in t_json['attributes'].items():
|
||||
self.assertTrue(entry['misp-attribute'] in me.describe_types['types'])
|
||||
if 'categories' in entry:
|
||||
subset = set(entry['categories']).issubset(me.describe_types['categories'])
|
||||
self.assertTrue(subset, f'{t_json["name"]} - {obj_relation}')
|
||||
'''
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
|
|
|||
|
|
@ -23,6 +23,8 @@ from collections import defaultdict
|
|||
|
||||
import logging
|
||||
logging.disable(logging.CRITICAL)
|
||||
logger = logging.getLogger('pymisp')
|
||||
|
||||
|
||||
try:
|
||||
from pymisp import ExpandedPyMISP, MISPEvent, MISPOrganisation, MISPUser, Distribution, ThreatLevel, Analysis, MISPObject, MISPAttribute, MISPSighting, MISPShadowAttribute, MISPTag, MISPSharingGroup, MISPFeed, MISPServer, MISPUserSetting
|
||||
|
|
@ -75,7 +77,7 @@ class TestComprehensive(unittest.TestCase):
|
|||
user.email = 'testusr@user.local'
|
||||
user.org_id = cls.test_org.id
|
||||
cls.test_usr = cls.admin_misp_connector.add_user(user, pythonify=True)
|
||||
cls.user_misp_connector = ExpandedPyMISP(url, cls.test_usr.authkey, verifycert, debug=False)
|
||||
cls.user_misp_connector = ExpandedPyMISP(url, cls.test_usr.authkey, verifycert, debug=True)
|
||||
cls.user_misp_connector.toggle_global_pythonify()
|
||||
# Creates a publisher
|
||||
user = MISPUser()
|
||||
|
|
@ -839,6 +841,7 @@ class TestComprehensive(unittest.TestCase):
|
|||
second = self.user_misp_connector.add_event(second)
|
||||
|
||||
current_ts = int(time.time())
|
||||
time.sleep(5)
|
||||
r = self.user_misp_connector.add_sighting({'value': first.attributes[0].value})
|
||||
self.assertEqual(int(r.attribute_id), first.attributes[0].id)
|
||||
|
||||
|
|
@ -1072,8 +1075,24 @@ class TestComprehensive(unittest.TestCase):
|
|||
file_object = first.get_objects_by_name('file')[0]
|
||||
file_object.force_misp_objects_path_custom('tests/mispevent_testfiles', 'overwrite_file')
|
||||
file_object.add_attribute('test_overwrite', 'blah')
|
||||
obj = self.admin_misp_connector.update_object(file_object, pythonify=True)
|
||||
obj_json = self.admin_misp_connector.update_object(file_object)
|
||||
self.assertTrue('Object' in obj_json, obj_json)
|
||||
self.assertTrue('name' in obj_json['Object'], obj_json)
|
||||
obj = MISPObject(obj_json['Object']['name'])
|
||||
obj.from_dict(**obj_json)
|
||||
self.assertEqual(obj.get_attributes_by_relation('test_overwrite')[0].value, 'blah')
|
||||
|
||||
# FULL object add & update with custom template
|
||||
new_object = MISPObject('overwrite_file', misp_objects_path_custom='tests/mispevent_testfiles')
|
||||
new_object.add_attribute('test_overwrite', 'barbaz')
|
||||
new_object.add_attribute('filename', 'barbaz.exe')
|
||||
new_object = self.admin_misp_connector.add_object(first, new_object, pythonify=True)
|
||||
self.assertEqual(new_object.get_attributes_by_relation('test_overwrite')[0].value, 'barbaz', new_object)
|
||||
|
||||
new_object.force_misp_objects_path_custom('tests/mispevent_testfiles', 'overwrite_file')
|
||||
new_object.add_attribute('filename', 'foobar.exe')
|
||||
new_object = self.admin_misp_connector.update_object(new_object, pythonify=True)
|
||||
self.assertEqual(new_object.get_attributes_by_relation('filename')[1].value, 'foobar.exe', new_object)
|
||||
finally:
|
||||
# Delete event
|
||||
self.admin_misp_connector.delete_event(first)
|
||||
|
|
@ -1166,12 +1185,34 @@ class TestComprehensive(unittest.TestCase):
|
|||
self.assertFalse(non_exportable_tag.exportable)
|
||||
first = self.create_simple_event()
|
||||
first.attributes[0].add_tag('non-exportable tag')
|
||||
# Add tag restricted to an org
|
||||
tag = MISPTag()
|
||||
tag.name = f'restricted to org {self.test_org.id}'
|
||||
tag.org_id = self.test_org.id
|
||||
tag_org_restricted = self.admin_misp_connector.add_tag(tag, pythonify=True)
|
||||
self.assertEqual(tag_org_restricted.org_id, tag.org_id)
|
||||
# Add tag restricted to a user
|
||||
tag.name = f'restricted to user {self.test_usr.id}'
|
||||
tag.user_id = self.test_usr.id
|
||||
tag_user_restricted = self.admin_misp_connector.add_tag(tag, pythonify=True)
|
||||
self.assertEqual(tag_user_restricted.user_id, tag.user_id)
|
||||
try:
|
||||
first = self.user_misp_connector.add_event(first)
|
||||
self.assertFalse(first.attributes[0].tags)
|
||||
first = self.admin_misp_connector.get_event(first, pythonify=True)
|
||||
# Reference: https://github.com/MISP/MISP/issues/1394
|
||||
self.assertFalse(first.attributes[0].tags)
|
||||
# Reference: https://github.com/MISP/PyMISP/issues/483
|
||||
r = self.delegate_user_misp_connector.tag(first, tag_org_restricted)
|
||||
# FIXME: The error message changed and is unhelpful.
|
||||
# self.assertEqual(r['errors'][1]['message'], 'Invalid Tag. This tag can only be set by a fixed organisation.')
|
||||
self.assertEqual(r['errors'][1]['message'], 'Invalid Target.')
|
||||
r = self.user_misp_connector.tag(first, tag_org_restricted)
|
||||
self.assertEqual(r['name'], f'Global tag {tag_org_restricted.name}({tag_org_restricted.id}) successfully attached to Event({first.id}).')
|
||||
r = self.pub_misp_connector.tag(first.attributes[0], tag_user_restricted)
|
||||
self.assertEqual(r['errors'][1]['message'], 'Invalid Tag. This tag can only be set by a fixed user.')
|
||||
r = self.user_misp_connector.tag(first.attributes[0], tag_user_restricted)
|
||||
self.assertEqual(r['name'], f'Global tag {tag_user_restricted.name}({tag_user_restricted.id}) successfully attached to Attribute({first.attributes[0].id}).')
|
||||
finally:
|
||||
# Delete event
|
||||
self.admin_misp_connector.delete_event(first)
|
||||
|
|
@ -1181,6 +1222,8 @@ class TestComprehensive(unittest.TestCase):
|
|||
self.assertEqual(response['message'], 'Tag deleted.')
|
||||
response = self.admin_misp_connector.delete_tag(non_exportable_tag)
|
||||
self.assertEqual(response['message'], 'Tag deleted.')
|
||||
response = self.admin_misp_connector.delete_tag(tag_org_restricted)
|
||||
response = self.admin_misp_connector.delete_tag(tag_user_restricted)
|
||||
|
||||
def test_add_event_with_attachment_object_controller(self):
|
||||
first = self.create_simple_event()
|
||||
|
|
@ -1561,6 +1604,10 @@ class TestComprehensive(unittest.TestCase):
|
|||
self.assertEqual(entry.action, 'edit')
|
||||
r = self.admin_misp_connector.update_user({'email': 'testusr@user.local'}, self.test_usr)
|
||||
|
||||
def test_db_schema(self):
|
||||
diag = self.admin_misp_connector.db_schema_diagnostic()
|
||||
self.assertEqual(diag['actual_db_version'], diag['expected_db_version'], diag)
|
||||
|
||||
def test_live_acl(self):
|
||||
missing_acls = self.admin_misp_connector.remote_acl()
|
||||
self.assertEqual(missing_acls, [], msg=missing_acls)
|
||||
|
|
@ -1757,7 +1804,7 @@ class TestComprehensive(unittest.TestCase):
|
|||
break
|
||||
# Get
|
||||
botvrij = self.admin_misp_connector.get_feed(feed, pythonify=True)
|
||||
self.assertEqual(botvrij.url, "http://www.botvrij.eu/data/feed-osint")
|
||||
self.assertEqual(botvrij.url, "https://www.botvrij.eu/data/feed-osint")
|
||||
# Enable
|
||||
# MISP OSINT
|
||||
feed = self.admin_misp_connector.enable_feed(feeds[0].id, pythonify=True)
|
||||
|
|
|
|||
Loading…
Reference in New Issue