mirror of https://github.com/CIRCL/PyCIRCLean
new: Add typing everywhere, use pathlib when possible
Raphaël Vinot
parent
9989a0769a
commit
d0f7d1bf81
|
|
@ -4,6 +4,7 @@ python:
|
||||||
- 3.6
|
- 3.6
|
||||||
- "3.6-dev"
|
- "3.6-dev"
|
||||||
- "3.7-dev"
|
- "3.7-dev"
|
||||||
|
- "3.8-dev"
|
||||||
# - nightly
|
# - nightly
|
||||||
|
|
||||||
sudo: required
|
sudo: required
|
||||||
|
|
@ -25,8 +26,8 @@ install:
|
||||||
- sudo apt-get install -y p7zip-rar
|
- sudo apt-get install -y p7zip-rar
|
||||||
# filecheck.py dependencies
|
# filecheck.py dependencies
|
||||||
- sudo apt-get install libxml2-dev libxslt1-dev
|
- sudo apt-get install libxml2-dev libxslt1-dev
|
||||||
- wget https://didierstevens.com/files/software/pdfid_v0_2_5.zip
|
- wget https://didierstevens.com/files/software/pdfid_v0_2_7.zip
|
||||||
- unzip pdfid_v0_2_5.zip
|
- unzip pdfid_v0_2_7.zip
|
||||||
- pip install -U pip pipenv
|
- pip install -U pip pipenv
|
||||||
# PyCIRCLean dependencies
|
# PyCIRCLean dependencies
|
||||||
- pipenv install -d
|
- pipenv install -d
|
||||||
|
|
@ -73,6 +74,7 @@ install:
|
||||||
# - popd
|
# - popd
|
||||||
|
|
||||||
script:
|
script:
|
||||||
|
- pipenv run mypy kittengroomer/ filecheck/ tests/ scripts/ --ignore-missing-imports
|
||||||
- travis_wait 60 pipenv run py.test --cov=kittengroomer --cov=filecheck tests/
|
- travis_wait 60 pipenv run py.test --cov=kittengroomer --cov=filecheck tests/
|
||||||
|
|
||||||
notifications:
|
notifications:
|
||||||
|
|
|
||||||
1
Pipfile
1
Pipfile
|
|
@ -10,6 +10,7 @@ pytest-cov = "*"
|
||||||
tox = "*"
|
tox = "*"
|
||||||
PyYAML = "*"
|
PyYAML = "*"
|
||||||
codecov = "*"
|
codecov = "*"
|
||||||
|
mypy = "*"
|
||||||
|
|
||||||
[packages]
|
[packages]
|
||||||
lxml = "*"
|
lxml = "*"
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
{
|
{
|
||||||
"_meta": {
|
"_meta": {
|
||||||
"hash": {
|
"hash": {
|
||||||
"sha256": "b78c4dbc4788f0e9c8400519c1dfb8d4b45ea65b7ffb236967d8d402b34543b2"
|
"sha256": "3771d02efb26aefa7e58aaf26720703899f894e4b8e8854be484a154e64aa1dc"
|
||||||
},
|
},
|
||||||
"pipfile-spec": 6,
|
"pipfile-spec": 6,
|
||||||
"requires": {
|
"requires": {
|
||||||
|
|
@ -18,40 +18,41 @@
|
||||||
"default": {
|
"default": {
|
||||||
"cffi": {
|
"cffi": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
"sha256:00d890313797d9fe4420506613384b43099ad7d2b905c0752dbcc3a6f14d80fa",
|
"sha256:0b49274afc941c626b605fb59b59c3485c17dc776dc3cc7cc14aca74cc19cc42",
|
||||||
"sha256:0cf9e550ac6c5e57b713437e2f4ac2d7fd0cd10336525a27224f5fc1ec2ee59a",
|
"sha256:0e3ea92942cb1168e38c05c1d56b0527ce31f1a370f6117f1d490b8dcd6b3a04",
|
||||||
"sha256:0ea23c9c0cdd6778146a50d867d6405693ac3b80a68829966c98dd5e1bbae400",
|
"sha256:135f69aecbf4517d5b3d6429207b2dff49c876be724ac0c8bf8e1ea99df3d7e5",
|
||||||
"sha256:193697c2918ecdb3865acf6557cddf5076bb39f1f654975e087b67efdff83365",
|
"sha256:19db0cdd6e516f13329cba4903368bff9bb5a9331d3410b1b448daaadc495e54",
|
||||||
"sha256:1ae14b542bf3b35e5229439c35653d2ef7d8316c1fffb980f9b7647e544baa98",
|
"sha256:2781e9ad0e9d47173c0093321bb5435a9dfae0ed6a762aabafa13108f5f7b2ba",
|
||||||
"sha256:1e389e069450609c6ffa37f21f40cce36f9be7643bbe5051ab1de99d5a779526",
|
"sha256:291f7c42e21d72144bb1c1b2e825ec60f46d0a7468f5346841860454c7aa8f57",
|
||||||
"sha256:263242b6ace7f9cd4ea401428d2d45066b49a700852334fd55311bde36dcda14",
|
"sha256:2c5e309ec482556397cb21ede0350c5e82f0eb2621de04b2633588d118da4396",
|
||||||
"sha256:33142ae9807665fa6511cfa9857132b2c3ee6ddffb012b3f0933fc11e1e830d5",
|
"sha256:2e9c80a8c3344a92cb04661115898a9129c074f7ab82011ef4b612f645939f12",
|
||||||
"sha256:364f8404034ae1b232335d8c7f7b57deac566f148f7222cef78cf8ae28ef764e",
|
"sha256:32a262e2b90ffcfdd97c7a5e24a6012a43c61f1f5a57789ad80af1d26c6acd97",
|
||||||
"sha256:47368f69fe6529f8f49a5d146ddee713fc9057e31d61e8b6dc86a6a5e38cecc1",
|
"sha256:3c9fff570f13480b201e9ab69453108f6d98244a7f495e91b6c654a47486ba43",
|
||||||
"sha256:4895640844f17bec32943995dc8c96989226974dfeb9dd121cc45d36e0d0c434",
|
"sha256:415bdc7ca8c1c634a6d7163d43fb0ea885a07e9618a64bda407e04b04333b7db",
|
||||||
"sha256:558b3afef987cf4b17abd849e7bedf64ee12b28175d564d05b628a0f9355599b",
|
"sha256:42194f54c11abc8583417a7cf4eaff544ce0de8187abaf5d29029c91b1725ad3",
|
||||||
"sha256:5ba86e1d80d458b338bda676fd9f9d68cb4e7a03819632969cf6d46b01a26730",
|
"sha256:4424e42199e86b21fc4db83bd76909a6fc2a2aefb352cb5414833c030f6ed71b",
|
||||||
"sha256:63424daa6955e6b4c70dc2755897f5be1d719eabe71b2625948b222775ed5c43",
|
"sha256:4a43c91840bda5f55249413037b7a9b79c90b1184ed504883b72c4df70778579",
|
||||||
"sha256:6381a7d8b1ebd0bc27c3bc85bc1bfadbb6e6f756b4d4db0aa1425c3719ba26b4",
|
"sha256:599a1e8ff057ac530c9ad1778293c665cb81a791421f46922d80a86473c13346",
|
||||||
"sha256:6381ab708158c4e1639da1f2a7679a9bbe3e5a776fc6d1fd808076f0e3145331",
|
"sha256:5c4fae4e9cdd18c82ba3a134be256e98dc0596af1e7285a3d2602c97dcfa5159",
|
||||||
"sha256:6fd58366747debfa5e6163ada468a90788411f10c92597d3b0a912d07e580c36",
|
"sha256:5ecfa867dea6fabe2a58f03ac9186ea64da1386af2159196da51c4904e11d652",
|
||||||
"sha256:728ec653964655d65408949b07f9b2219df78badd601d6c49e28d604efe40599",
|
"sha256:62f2578358d3a92e4ab2d830cd1c2049c9c0d0e6d3c58322993cc341bdeac22e",
|
||||||
"sha256:7cfcfda59ef1f95b9f729c56fe8a4041899f96b72685d36ef16a3440a0f85da8",
|
"sha256:6471a82d5abea994e38d2c2abc77164b4f7fbaaf80261cb98394d5793f11b12a",
|
||||||
"sha256:819f8d5197c2684524637f940445c06e003c4a541f9983fd30d6deaa2a5487d8",
|
"sha256:6d4f18483d040e18546108eb13b1dfa1000a089bcf8529e30346116ea6240506",
|
||||||
"sha256:825ecffd9574557590e3225560a8a9d751f6ffe4a49e3c40918c9969b93395fa",
|
"sha256:71a608532ab3bd26223c8d841dde43f3516aa5d2bf37b50ac410bb5e99053e8f",
|
||||||
"sha256:8a2bcae2258d00fcfc96a9bde4a6177bc4274fe033f79311c5dd3d3148c26518",
|
"sha256:74a1d8c85fb6ff0b30fbfa8ad0ac23cd601a138f7509dc617ebc65ef305bb98d",
|
||||||
"sha256:9009e917d8f5ef780c2626e29b6bc126f4cb2a4d43ca67aa2b40f2a5d6385e78",
|
"sha256:7b93a885bb13073afb0aa73ad82059a4c41f4b7d8eb8368980448b52d4c7dc2c",
|
||||||
"sha256:9c77564a51d4d914ed5af096cd9843d90c45b784b511723bd46a8a9d09cf16fc",
|
"sha256:7d4751da932caaec419d514eaa4215eaf14b612cff66398dd51129ac22680b20",
|
||||||
"sha256:a19089fa74ed19c4fe96502a291cfdb89223a9705b1d73b3005df4256976142e",
|
"sha256:7f627141a26b551bdebbc4855c1157feeef18241b4b8366ed22a5c7d672ef858",
|
||||||
"sha256:a40ed527bffa2b7ebe07acc5a3f782da072e262ca994b4f2085100b5a444bbb2",
|
"sha256:8169cf44dd8f9071b2b9248c35fc35e8677451c52f795daa2bb4643f32a540bc",
|
||||||
"sha256:b8f09f21544b9899defb09afbdaeb200e6a87a2b8e604892940044cf94444644",
|
"sha256:aa00d66c0fab27373ae44ae26a66a9e43ff2a678bf63a9c7c1a9a4d61172827a",
|
||||||
"sha256:bb75ba21d5716abc41af16eac1145ab2e471deedde1f22c6f99bd9f995504df0",
|
"sha256:ccb032fda0873254380aa2bfad2582aedc2959186cce61e3a17abc1a55ff89c3",
|
||||||
"sha256:e22a00c0c81ffcecaf07c2bfb3672fa372c50e2bd1024ffee0da191c1b27fc71",
|
"sha256:d754f39e0d1603b5b24a7f8484b22d2904fa551fe865fd0d4c3332f078d20d4e",
|
||||||
"sha256:e55b5a746fb77f10c83e8af081979351722f6ea48facea79d470b3731c7b2891",
|
"sha256:d75c461e20e29afc0aee7172a0950157c704ff0dd51613506bd7d82b718e7410",
|
||||||
"sha256:ec2fa3ee81707a5232bf2dfbd6623fdb278e070d596effc7e2d788f2ada71a05",
|
"sha256:dcd65317dd15bc0451f3e01c80da2216a31916bdcffd6221ca1202d96584aa25",
|
||||||
"sha256:fd82eb4694be712fcae03c717ca2e0fc720657ac226b80bbb597e971fc6928c2"
|
"sha256:e570d3ab32e2c2861c4ebe6ffcad6a8abf9347432a37608fe1fbd157b3f0036b",
|
||||||
|
"sha256:fd43a88e045cf992ed09fa724b5315b790525f2676883a6ea64e3263bae6549d"
|
||||||
],
|
],
|
||||||
"version": "==1.13.1"
|
"version": "==1.13.2"
|
||||||
},
|
},
|
||||||
"colorclass": {
|
"colorclass": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
|
|
@ -106,35 +107,35 @@
|
||||||
},
|
},
|
||||||
"lxml": {
|
"lxml": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
"sha256:02ca7bf899da57084041bb0f6095333e4d239948ad3169443f454add9f4e9cb4",
|
"sha256:00ac0d64949fef6b3693813fe636a2d56d97a5a49b5bbb86e4cc4cc50ebc9ea2",
|
||||||
"sha256:096b82c5e0ea27ce9138bcbb205313343ee66a6e132f25c5ed67e2c8d960a1bc",
|
"sha256:0571e607558665ed42e450d7bf0e2941d542c18e117b1ebbf0ba72f287ad841c",
|
||||||
"sha256:0a920ff98cf1aac310470c644bc23b326402d3ef667ddafecb024e1713d485f1",
|
"sha256:0e3f04a7615fdac0be5e18b2406529521d6dbdb0167d2a690ee328bef7807487",
|
||||||
"sha256:1409b14bf83a7d729f92e2a7fbfe7ec929d4883ca071b06e95c539ceedb6497c",
|
"sha256:13cf89be53348d1c17b453867da68704802966c433b2bb4fa1f970daadd2ef70",
|
||||||
"sha256:17cae1730a782858a6e2758fd20dd0ef7567916c47757b694a06ffafdec20046",
|
"sha256:217262fcf6a4c2e1c7cb1efa08bd9ebc432502abc6c255c4abab611e8be0d14d",
|
||||||
"sha256:17e3950add54c882e032527795c625929613adbd2ce5162b94667334458b5a36",
|
"sha256:223e544828f1955daaf4cefbb4853bc416b2ec3fd56d4f4204a8b17007c21250",
|
||||||
"sha256:1f4f214337f6ee5825bf90a65d04d70aab05526c08191ab888cb5149501923c5",
|
"sha256:277cb61fede2f95b9c61912fefb3d43fbd5f18bf18a14fae4911b67984486f5d",
|
||||||
"sha256:2e8f77db25b0a96af679e64ff9bf9dddb27d379c9900c3272f3041c4d1327c9d",
|
"sha256:3213f753e8ae86c396e0e066866e64c6b04618e85c723b32ecb0909885211f74",
|
||||||
"sha256:4dffd405390a45ecb95ab5ab1c1b847553c18b0ef8ed01e10c1c8b1a76452916",
|
"sha256:4690984a4dee1033da0af6df0b7a6bde83f74e1c0c870623797cec77964de34d",
|
||||||
"sha256:6b899931a5648862c7b88c795eddff7588fb585e81cecce20f8d9da16eff96e0",
|
"sha256:4fcc472ef87f45c429d3b923b925704aa581f875d65bac80f8ab0c3296a63f78",
|
||||||
"sha256:726c17f3e0d7a7200718c9a890ccfeab391c9133e363a577a44717c85c71db27",
|
"sha256:61409bd745a265a742f2693e4600e4dbd45cc1daebe1d5fad6fcb22912d44145",
|
||||||
"sha256:760c12276fee05c36f95f8040180abc7fbebb9e5011447a97cdc289b5d6ab6fc",
|
"sha256:678f1963f755c5d9f5f6968dded7b245dd1ece8cf53c1aa9d80e6734a8c7f41d",
|
||||||
"sha256:796685d3969815a633827c818863ee199440696b0961e200b011d79b9394bbe7",
|
"sha256:6c6d03549d4e2734133badb9ab1c05d9f0ef4bcd31d83e5d2b4747c85cfa21da",
|
||||||
"sha256:891fe897b49abb7db470c55664b198b1095e4943b9f82b7dcab317a19116cd38",
|
"sha256:6e74d5f4d6ecd6942375c52ffcd35f4318a61a02328f6f1bd79fcb4ffedf969e",
|
||||||
"sha256:9277562f175d2334744ad297568677056861070399cec56ff06abbe2564d1232",
|
"sha256:7b4fc7b1ecc987ca7aaf3f4f0e71bbfbd81aaabf87002558f5bc95da3a865bcd",
|
||||||
"sha256:a471628e20f03dcdfde00770eeaf9c77811f0c331c8805219ca7b87ac17576c5",
|
"sha256:7ed386a40e172ddf44c061ad74881d8622f791d9af0b6f5be20023029129bc85",
|
||||||
"sha256:a63b4fd3e2cabdcc9d918ed280bdde3e8e9641e04f3c59a2a3109644a07b9832",
|
"sha256:8f54f0924d12c47a382c600c880770b5ebfc96c9fd94cf6f6bdc21caf6163ea7",
|
||||||
"sha256:ae88588d687bd476be588010cbbe551e9c2872b816f2da8f01f6f1fda74e1ef0",
|
"sha256:ad9b81351fdc236bda538efa6879315448411a81186c836d4b80d6ca8217cdb9",
|
||||||
"sha256:b0b84408d4eabc6de9dd1e1e0bc63e7731e890c0b378a62443e5741cfd0ae90a",
|
"sha256:bbd00e21ea17f7bcc58dccd13869d68441b32899e89cf6cfa90d624a9198ce85",
|
||||||
"sha256:be78485e5d5f3684e875dab60f40cddace2f5b2a8f7fede412358ab3214c3a6f",
|
"sha256:c3c289762cc09735e2a8f8a49571d0e8b4f57ea831ea11558247b5bdea0ac4db",
|
||||||
"sha256:c27eaed872185f047bb7f7da2d21a7d8913457678c9a100a50db6da890bc28b9",
|
"sha256:cf4650942de5e5685ad308e22bcafbccfe37c54aa7c0e30cd620c2ee5c93d336",
|
||||||
"sha256:c7fccd08b14aa437fe096c71c645c0f9be0655a9b1a4b7cffc77bcb23b3d61d2",
|
"sha256:cfcbc33c9c59c93776aa41ab02e55c288a042211708b72fdb518221cc803abc8",
|
||||||
"sha256:c81cb40bff373ab7a7446d6bbca0190bccc5be3448b47b51d729e37799bb5692",
|
"sha256:e301055deadfedbd80cf94f2f65ff23126b232b0d1fea28f332ce58137bcdb18",
|
||||||
"sha256:d11874b3c33ee441059464711cd365b89fa1a9cf19ae75b0c189b01fbf735b84",
|
"sha256:ebbfe24df7f7b5c6c7620702496b6419f6a9aa2fd7f005eb731cc80d7b4692b9",
|
||||||
"sha256:e9c028b5897901361d81a4718d1db217b716424a0283afe9d6735fe0caf70f79",
|
"sha256:eff69ddbf3ad86375c344339371168640951c302450c5d3e9936e98d6459db06",
|
||||||
"sha256:fe489d486cd00b739be826e8c1be188ddb74c7a1ca784d93d06fda882a6a1681"
|
"sha256:f6ed60a62c5f1c44e789d2cf14009423cb1646b44a43e40a9cf6a21f077678a1"
|
||||||
],
|
],
|
||||||
"index": "pypi",
|
"index": "pypi",
|
||||||
"version": "==4.4.1"
|
"version": "==4.4.2"
|
||||||
},
|
},
|
||||||
"msoffcrypto-tool": {
|
"msoffcrypto-tool": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
|
|
@ -156,46 +157,45 @@
|
||||||
},
|
},
|
||||||
"oletools": {
|
"oletools": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
"sha256:aad914cddbb84bd085607686293b3b0d97093b0b6f57785c956c94aa51885e06"
|
"sha256:edea57914c4040e7d0d64cfd88c84355d4305548d761d476fbac21ee26b25d8d"
|
||||||
],
|
],
|
||||||
"index": "pypi",
|
"index": "pypi",
|
||||||
"version": "==0.54.2"
|
"version": "==0.55.1"
|
||||||
|
},
|
||||||
|
"pcodedmp": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:025f8c809a126f45a082ffa820893e6a8d990d9d7ddb68694b5a9f0a6dbcd955",
|
||||||
|
"sha256:4441f7c0ab4cbda27bd4668db3b14f36261d86e5059ce06c0828602cbe1c4278"
|
||||||
|
],
|
||||||
|
"version": "==1.2.6"
|
||||||
},
|
},
|
||||||
"pillow": {
|
"pillow": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
"sha256:047d9473cf68af50ac85f8ee5d5f21a60f849bc17d348da7fc85711287a75031",
|
"sha256:0a628977ac2e01ca96aaae247ec2bd38e729631ddf2221b4b715446fd45505be",
|
||||||
"sha256:0f66dc6c8a3cc319561a633b6aa82c44107f12594643efa37210d8c924fc1c71",
|
"sha256:4d9ed9a64095e031435af120d3c910148067087541131e82b3e8db302f4c8946",
|
||||||
"sha256:12c9169c4e8fe0a7329e8658c7e488001f6b4c8e88740e76292c2b857af2e94c",
|
"sha256:54ebae163e8412aff0b9df1e88adab65788f5f5b58e625dc5c7f51eaf14a6837",
|
||||||
"sha256:248cffc168896982f125f5c13e9317c059f74fffdb4152893339f3be62a01340",
|
"sha256:5bfef0b1cdde9f33881c913af14e43db69815c7e8df429ceda4c70a5e529210f",
|
||||||
"sha256:27faf0552bf8c260a5cee21a76e031acaea68babb64daf7e8f2e2540745082aa",
|
"sha256:5f3546ceb08089cedb9e8ff7e3f6a7042bb5b37c2a95d392fb027c3e53a2da00",
|
||||||
"sha256:285edafad9bc60d96978ed24d77cdc0b91dace88e5da8c548ba5937c425bca8b",
|
"sha256:5f7ae9126d16194f114435ebb79cc536b5682002a4fa57fa7bb2cbcde65f2f4d",
|
||||||
"sha256:384b12c9aa8ef95558abdcb50aada56d74bc7cc131dd62d28c2d0e4d3aadd573",
|
"sha256:62a889aeb0a79e50ecf5af272e9e3c164148f4bd9636cc6bcfa182a52c8b0533",
|
||||||
"sha256:38950b3a707f6cef09cd3cbb142474357ad1a985ceb44d921bdf7b4647b3e13e",
|
"sha256:7406f5a9b2fd966e79e6abdaf700585a4522e98d6559ce37fc52e5c955fade0a",
|
||||||
"sha256:4aad1b88933fd6dc2846552b89ad0c74ddbba2f0884e2c162aa368374bf5abab",
|
"sha256:8453f914f4e5a3d828281a6628cf517832abfa13ff50679a4848926dac7c0358",
|
||||||
"sha256:4ac6148008c169603070c092e81f88738f1a0c511e07bd2bb0f9ef542d375da9",
|
"sha256:87269cc6ce1e3dee11f23fa515e4249ae678dbbe2704598a51cee76c52e19cda",
|
||||||
"sha256:4deb1d2a45861ae6f0b12ea0a786a03d19d29edcc7e05775b85ec2877cb54c5e",
|
"sha256:875358310ed7abd5320f21dd97351d62de4929b0426cdb1eaa904b64ac36b435",
|
||||||
"sha256:59aa2c124df72cc75ed72c8d6005c442d4685691a30c55321e00ed915ad1a291",
|
"sha256:8ac6ce7ff3892e5deaab7abaec763538ffd011f74dc1801d93d3c5fc541feee2",
|
||||||
"sha256:5a47d2123a9ec86660fe0e8d0ebf0aa6bc6a17edc63f338b73ea20ba11713f12",
|
"sha256:91b710e3353aea6fc758cdb7136d9bbdcb26b53cefe43e2cba953ac3ee1d3313",
|
||||||
"sha256:5cc901c2ab9409b4b7ac7b5bcc3e86ac14548627062463da0af3b6b7c555a871",
|
"sha256:9d2ba4ed13af381233e2d810ff3bab84ef9f18430a9b336ab69eaf3cd24299ff",
|
||||||
"sha256:6c1db03e8dff7b9f955a0fb9907eb9ca5da75b5ce056c0c93d33100a35050281",
|
"sha256:a62ec5e13e227399be73303ff301f2865bf68657d15ea50b038d25fc41097317",
|
||||||
"sha256:7ce80c0a65a6ea90ef9c1f63c8593fcd2929448613fc8da0adf3e6bfad669d08",
|
"sha256:ab76e5580b0ed647a8d8d2d2daee170e8e9f8aad225ede314f684e297e3643c2",
|
||||||
"sha256:809c19241c14433c5d6135e1b6c72da4e3b56d5c865ad5736ab99af8896b8f41",
|
"sha256:bf4003aa538af3f4205c5fac56eacaa67a6dd81e454ffd9e9f055fff9f1bc614",
|
||||||
"sha256:83792cb4e0b5af480588601467c0764242b9a483caea71ef12d22a0d0d6bdce2",
|
"sha256:bf598d2e37cf8edb1a2f26ed3fb255191f5232badea4003c16301cb94ac5bdd0",
|
||||||
"sha256:846fa202bd7ee0f6215c897a1d33238ef071b50766339186687bd9b7a6d26ac5",
|
"sha256:c18f70dc27cc5d236f10e7834236aff60aadc71346a5bc1f4f83a4b3abee6386",
|
||||||
"sha256:9f5529fc02009f96ba95bea48870173426879dc19eec49ca8e08cd63ecd82ddb",
|
"sha256:c5ed816632204a2fc9486d784d8e0d0ae754347aba99c811458d69fcdfd2a2f9",
|
||||||
"sha256:a423c2ea001c6265ed28700df056f75e26215fd28c001e93ef4380b0f05f9547",
|
"sha256:dc058b7833184970d1248135b8b0ab702e6daa833be14035179f2acb78ff5636",
|
||||||
"sha256:ac4428094b42907aba5879c7c000d01c8278d451a3b7cccd2103e21f6397ea75",
|
"sha256:ff3797f2f16bf9d17d53257612da84dd0758db33935777149b3334c01ff68865"
|
||||||
"sha256:b1ae48d87f10d1384e5beecd169c77502fcc04a2c00a4c02b85f0a94b419e5f9",
|
|
||||||
"sha256:bf4e972a88f8841d8fdc6db1a75e0f8d763e66e3754b03006cbc3854d89f1cb1",
|
|
||||||
"sha256:c6414f6aad598364aaf81068cabb077894eb88fed99c6a65e6e8217bab62ae7a",
|
|
||||||
"sha256:c710fcb7ee32f67baf25aa9ffede4795fd5d93b163ce95fdc724383e38c9df96",
|
|
||||||
"sha256:c7be4b8a09852291c3c48d3c25d1b876d2494a0a674980089ac9d5e0d78bd132",
|
|
||||||
"sha256:c9e5ffb910b14f090ac9c38599063e354887a5f6d7e6d26795e916b4514f2c1a",
|
|
||||||
"sha256:e0697b826da6c2472bb6488db4c0a7fa8af0d52fa08833ceb3681358914b14e5",
|
|
||||||
"sha256:e9a3edd5f714229d41057d56ac0f39ad9bdba6767e8c888c951869f0bdd129b0"
|
|
||||||
],
|
],
|
||||||
"index": "pypi",
|
"index": "pypi",
|
||||||
"version": "==6.2.1"
|
"version": "==7.0.0"
|
||||||
},
|
},
|
||||||
"pycparser": {
|
"pycparser": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
|
|
@ -205,10 +205,10 @@
|
||||||
},
|
},
|
||||||
"pyparsing": {
|
"pyparsing": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
"sha256:6f98a7b9397e206d78cc01df10131398f1c8b8510a2f4d97d9abd82e1aacdd80",
|
"sha256:4c830582a84fb022400b85429791bc551f1f4871c33f23e44f353119e92f969f",
|
||||||
"sha256:d9338df12903bbf5d65a0e4e87c2161968b10d2e489652bb47001d82a9b028b4"
|
"sha256:c342dccb5250c08d45fd6f8b4a559613ca603b57498511740e65cd11a2e7dcec"
|
||||||
],
|
],
|
||||||
"version": "==2.4.2"
|
"version": "==2.4.6"
|
||||||
},
|
},
|
||||||
"python-magic": {
|
"python-magic": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
|
|
@ -219,20 +219,13 @@
|
||||||
},
|
},
|
||||||
"six": {
|
"six": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
"sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c",
|
"sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd",
|
||||||
"sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73"
|
"sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66"
|
||||||
],
|
],
|
||||||
"version": "==1.12.0"
|
"version": "==1.13.0"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"develop": {
|
"develop": {
|
||||||
"atomicwrites": {
|
|
||||||
"hashes": [
|
|
||||||
"sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4",
|
|
||||||
"sha256:75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6"
|
|
||||||
],
|
|
||||||
"version": "==1.3.0"
|
|
||||||
},
|
|
||||||
"attrs": {
|
"attrs": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
"sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c",
|
"sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c",
|
||||||
|
|
@ -242,10 +235,10 @@
|
||||||
},
|
},
|
||||||
"certifi": {
|
"certifi": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
"sha256:e4f3620cfea4f83eedc95b24abd9cd56f3c4b146dd0177e83a21b4eb49e21e50",
|
"sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3",
|
||||||
"sha256:fd7c7c74727ddcf00e9acd26bba8da604ffec95bf1c2144e67aff7a8b50e6cef"
|
"sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f"
|
||||||
],
|
],
|
||||||
"version": "==2019.9.11"
|
"version": "==2019.11.28"
|
||||||
},
|
},
|
||||||
"chardet": {
|
"chardet": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
|
|
@ -264,40 +257,39 @@
|
||||||
},
|
},
|
||||||
"coverage": {
|
"coverage": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
"sha256:08907593569fe59baca0bf152c43f3863201efb6113ecb38ce7e97ce339805a6",
|
"sha256:15cf13a6896048d6d947bf7d222f36e4809ab926894beb748fc9caa14605d9c3",
|
||||||
"sha256:0be0f1ed45fc0c185cfd4ecc19a1d6532d72f86a2bac9de7e24541febad72650",
|
"sha256:1daa3eceed220f9fdb80d5ff950dd95112cd27f70d004c7918ca6dfc6c47054c",
|
||||||
"sha256:141f08ed3c4b1847015e2cd62ec06d35e67a3ac185c26f7635f4406b90afa9c5",
|
"sha256:1e44a022500d944d42f94df76727ba3fc0a5c0b672c358b61067abb88caee7a0",
|
||||||
"sha256:19e4df788a0581238e9390c85a7a09af39c7b539b29f25c89209e6c3e371270d",
|
"sha256:25dbf1110d70bab68a74b4b9d74f30e99b177cde3388e07cc7272f2168bd1477",
|
||||||
"sha256:23cc09ed395b03424d1ae30dcc292615c1372bfba7141eb85e11e50efaa6b351",
|
"sha256:3230d1003eec018ad4a472d254991e34241e0bbd513e97a29727c7c2f637bd2a",
|
||||||
"sha256:245388cda02af78276b479f299bbf3783ef0a6a6273037d7c60dc73b8d8d7755",
|
"sha256:3dbb72eaeea5763676a1a1efd9b427a048c97c39ed92e13336e726117d0b72bf",
|
||||||
"sha256:331cb5115673a20fb131dadd22f5bcaf7677ef758741312bee4937d71a14b2ef",
|
"sha256:5012d3b8d5a500834783689a5d2292fe06ec75dc86ee1ccdad04b6f5bf231691",
|
||||||
"sha256:386e2e4090f0bc5df274e720105c342263423e77ee8826002dcffe0c9533dbca",
|
"sha256:51bc7710b13a2ae0c726f69756cf7ffd4362f4ac36546e243136187cfcc8aa73",
|
||||||
"sha256:3a794ce50daee01c74a494919d5ebdc23d58873747fa0e288318728533a3e1ca",
|
"sha256:527b4f316e6bf7755082a783726da20671a0cc388b786a64417780b90565b987",
|
||||||
"sha256:60851187677b24c6085248f0a0b9b98d49cba7ecc7ec60ba6b9d2e5574ac1ee9",
|
"sha256:722e4557c8039aad9592c6a4213db75da08c2cd9945320220634f637251c3894",
|
||||||
"sha256:63a9a5fc43b58735f65ed63d2cf43508f462dc49857da70b8980ad78d41d52fc",
|
"sha256:76e2057e8ffba5472fd28a3a010431fd9e928885ff480cb278877c6e9943cc2e",
|
||||||
"sha256:6b62544bb68106e3f00b21c8930e83e584fdca005d4fffd29bb39fb3ffa03cb5",
|
"sha256:77afca04240c40450c331fa796b3eab6f1e15c5ecf8bf2b8bee9706cd5452fef",
|
||||||
"sha256:6ba744056423ef8d450cf627289166da65903885272055fb4b5e113137cfa14f",
|
"sha256:7afad9835e7a651d3551eab18cbc0fdb888f0a6136169fbef0662d9cdc9987cf",
|
||||||
"sha256:7494b0b0274c5072bddbfd5b4a6c6f18fbbe1ab1d22a41e99cd2d00c8f96ecfe",
|
"sha256:9bea19ac2f08672636350f203db89382121c9c2ade85d945953ef3c8cf9d2a68",
|
||||||
"sha256:826f32b9547c8091679ff292a82aca9c7b9650f9fda3e2ca6bf2ac905b7ce888",
|
"sha256:a8b8ac7876bc3598e43e2603f772d2353d9931709345ad6c1149009fd1bc81b8",
|
||||||
"sha256:93715dffbcd0678057f947f496484e906bf9509f5c1c38fc9ba3922893cda5f5",
|
"sha256:b0840b45187699affd4c6588286d429cd79a99d509fe3de0f209594669bb0954",
|
||||||
"sha256:9a334d6c83dfeadae576b4d633a71620d40d1c379129d587faa42ee3e2a85cce",
|
"sha256:b26aaf69713e5674efbde4d728fb7124e429c9466aeaf5f4a7e9e699b12c9fe2",
|
||||||
"sha256:af7ed8a8aa6957aac47b4268631fa1df984643f07ef00acd374e456364b373f5",
|
"sha256:b63dd43f455ba878e5e9f80ba4f748c0a2156dde6e0e6e690310e24d6e8caf40",
|
||||||
"sha256:bf0a7aed7f5521c7ca67febd57db473af4762b9622254291fbcbb8cd0ba5e33e",
|
"sha256:be18f4ae5a9e46edae3f329de2191747966a34a3d93046dbdf897319923923bc",
|
||||||
"sha256:bf1ef9eb901113a9805287e090452c05547578eaab1b62e4ad456fcc049a9b7e",
|
"sha256:c312e57847db2526bc92b9bfa78266bfbaabac3fdcd751df4d062cd4c23e46dc",
|
||||||
"sha256:c0afd27bc0e307a1ffc04ca5ec010a290e49e3afbe841c5cafc5c5a80ecd81c9",
|
"sha256:c60097190fe9dc2b329a0eb03393e2e0829156a589bd732e70794c0dd804258e",
|
||||||
"sha256:dd579709a87092c6dbee09d1b7cfa81831040705ffa12a1b248935274aee0437",
|
"sha256:c62a2143e1313944bf4a5ab34fd3b4be15367a02e9478b0ce800cb510e3bbb9d",
|
||||||
"sha256:df6712284b2e44a065097846488f66840445eb987eb81b3cc6e4149e7b6982e1",
|
"sha256:cc1109f54a14d940b8512ee9f1c3975c181bbb200306c6d8b87d93376538782f",
|
||||||
"sha256:e07d9f1a23e9e93ab5c62902833bf3e4b1f65502927379148b6622686223125c",
|
"sha256:cd60f507c125ac0ad83f05803063bed27e50fa903b9c2cfee3f8a6867ca600fc",
|
||||||
"sha256:e2ede7c1d45e65e209d6093b762e98e8318ddeff95317d07a27a2140b80cfd24",
|
"sha256:d513cc3db248e566e07a0da99c230aca3556d9b09ed02f420664e2da97eac301",
|
||||||
"sha256:e4ef9c164eb55123c62411f5936b5c2e521b12356037b6e1c2617cef45523d47",
|
"sha256:d649dc0bcace6fcdb446ae02b98798a856593b19b637c1b9af8edadf2b150bea",
|
||||||
"sha256:eca2b7343524e7ba246cab8ff00cab47a2d6d54ada3b02772e908a45675722e2",
|
"sha256:d7008a6796095a79544f4da1ee49418901961c97ca9e9d44904205ff7d6aa8cb",
|
||||||
"sha256:eee64c616adeff7db37cc37da4180a3a5b6177f5c46b187894e633f088fb5b28",
|
"sha256:da93027835164b8223e8e5af2cf902a4c80ed93cb0909417234f4a9df3bcd9af",
|
||||||
"sha256:ef824cad1f980d27f26166f86856efe11eff9912c4fed97d3804820d43fa550c",
|
"sha256:e69215621707119c6baf99bda014a45b999d37602cb7043d943c76a59b05bf52",
|
||||||
"sha256:efc89291bd5a08855829a3c522df16d856455297cf35ae827a37edac45f466a7",
|
"sha256:ea9525e0fef2de9208250d6c5aeeee0138921057cd67fcef90fbed49c4d62d37",
|
||||||
"sha256:fa964bae817babece5aa2e8c1af841bebb6d0b9add8e637548809d040443fee0",
|
"sha256:fca1669d464f0c9831fd10be2eef6b86f5ebd76c724d1e0706ebdff86bb4adf0"
|
||||||
"sha256:ff37757e068ae606659c28c3bd0d923f9d29a85de79bf25b2b34b148473b5025"
|
|
||||||
],
|
],
|
||||||
"version": "==4.5.4"
|
"version": "==5.0.3"
|
||||||
},
|
},
|
||||||
"filelock": {
|
"filelock": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
|
|
@ -315,54 +307,81 @@
|
||||||
},
|
},
|
||||||
"importlib-metadata": {
|
"importlib-metadata": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
"sha256:aa18d7378b00b40847790e7c27e11673d7fed219354109d0e7b9e5b25dc3ad26",
|
"sha256:bdd9b7c397c273bcc9a11d6629a38487cd07154fa255a467bf704cd2c258e359",
|
||||||
"sha256:d5f18a79777f3aa179c145737780282e27b508fc8fd688cb17c7a813e8bd39af"
|
"sha256:f17c015735e1a88296994c0697ecea7e11db24290941983b08c9feb30921e6d8"
|
||||||
],
|
],
|
||||||
"markers": "python_version < '3.8'",
|
"markers": "python_version < '3.8'",
|
||||||
"version": "==0.23"
|
"version": "==1.4.0"
|
||||||
},
|
},
|
||||||
"more-itertools": {
|
"more-itertools": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
"sha256:409cd48d4db7052af495b09dec721011634af3753ae1ef92d2b32f73a745f832",
|
"sha256:1a2a32c72400d365000412fe08eb4a24ebee89997c18d3d147544f70f5403b39",
|
||||||
"sha256:92b8c4b06dac4f0611c0729b2f2ede52b2e1bac1ab48f089c7ddc12e26bb60c4"
|
"sha256:c468adec578380b6281a114cb8a5db34eb1116277da92d7c46f904f0b52d3288"
|
||||||
],
|
],
|
||||||
"version": "==7.2.0"
|
"version": "==8.1.0"
|
||||||
|
},
|
||||||
|
"mypy": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:0a9a45157e532da06fe56adcfef8a74629566b607fa2c1ac0122d1ff995c748a",
|
||||||
|
"sha256:2c35cae79ceb20d47facfad51f952df16c2ae9f45db6cb38405a3da1cf8fc0a7",
|
||||||
|
"sha256:4b9365ade157794cef9685791032521233729cb00ce76b0ddc78749abea463d2",
|
||||||
|
"sha256:53ea810ae3f83f9c9b452582261ea859828a9ed666f2e1ca840300b69322c474",
|
||||||
|
"sha256:634aef60b4ff0f650d3e59d4374626ca6153fcaff96ec075b215b568e6ee3cb0",
|
||||||
|
"sha256:7e396ce53cacd5596ff6d191b47ab0ea18f8e0ec04e15d69728d530e86d4c217",
|
||||||
|
"sha256:7eadc91af8270455e0d73565b8964da1642fe226665dd5c9560067cd64d56749",
|
||||||
|
"sha256:7f672d02fffcbace4db2b05369142e0506cdcde20cea0e07c7c2171c4fd11dd6",
|
||||||
|
"sha256:85baab8d74ec601e86134afe2bcccd87820f79d2f8d5798c889507d1088287bf",
|
||||||
|
"sha256:87c556fb85d709dacd4b4cb6167eecc5bbb4f0a9864b69136a0d4640fdc76a36",
|
||||||
|
"sha256:a6bd44efee4dc8c3324c13785a9dc3519b3ee3a92cada42d2b57762b7053b49b",
|
||||||
|
"sha256:c6d27bd20c3ba60d5b02f20bd28e20091d6286a699174dfad515636cb09b5a72",
|
||||||
|
"sha256:e2bb577d10d09a2d8822a042a23b8d62bc3b269667c9eb8e60a6edfa000211b1",
|
||||||
|
"sha256:f97a605d7c8bc2c6d1172c2f0d5a65b24142e11a58de689046e62c2d632ca8c1"
|
||||||
|
],
|
||||||
|
"index": "pypi",
|
||||||
|
"version": "==0.761"
|
||||||
|
},
|
||||||
|
"mypy-extensions": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d",
|
||||||
|
"sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"
|
||||||
|
],
|
||||||
|
"version": "==0.4.3"
|
||||||
},
|
},
|
||||||
"packaging": {
|
"packaging": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
"sha256:28b924174df7a2fa32c1953825ff29c61e2f5e082343165438812f00d3a7fc47",
|
"sha256:aec3fdbb8bc9e4bb65f0634b9f551ced63983a529d6a8931817d52fdd0816ddb",
|
||||||
"sha256:d9551545c6d761f3def1677baf08ab2a3ca17c56879e70fecba2fc4dde4ed108"
|
"sha256:fe1d8331dfa7cc0a883b49d75fc76380b2ab2734b220fbb87d774e4fd4b851f8"
|
||||||
],
|
],
|
||||||
"version": "==19.2"
|
"version": "==20.0"
|
||||||
},
|
},
|
||||||
"pluggy": {
|
"pluggy": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
"sha256:0db4b7601aae1d35b4a033282da476845aa19185c1e6964b25cf324b5e4ec3e6",
|
"sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0",
|
||||||
"sha256:fa5fa1622fa6dd5c030e9cad086fa19ef6a0cf6d7a2d12318e10cb49d6d68f34"
|
"sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"
|
||||||
],
|
],
|
||||||
"version": "==0.13.0"
|
"version": "==0.13.1"
|
||||||
},
|
},
|
||||||
"py": {
|
"py": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
"sha256:64f65755aee5b381cea27766a3a147c3f15b9b6b9ac88676de66ba2ae36793fa",
|
"sha256:5e27081401262157467ad6e7f851b7aa402c5852dbcb3dae06768434de5752aa",
|
||||||
"sha256:dc639b046a6e2cff5bbe40194ad65936d6ba360b52b3c3fe1d08a82dd50b5e53"
|
"sha256:c20fdd83a5dbc0af9efd622bee9a5564e278f6380fffcacc43ba6f43db2813b0"
|
||||||
],
|
],
|
||||||
"version": "==1.8.0"
|
"version": "==1.8.1"
|
||||||
},
|
},
|
||||||
"pyparsing": {
|
"pyparsing": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
"sha256:6f98a7b9397e206d78cc01df10131398f1c8b8510a2f4d97d9abd82e1aacdd80",
|
"sha256:4c830582a84fb022400b85429791bc551f1f4871c33f23e44f353119e92f969f",
|
||||||
"sha256:d9338df12903bbf5d65a0e4e87c2161968b10d2e489652bb47001d82a9b028b4"
|
"sha256:c342dccb5250c08d45fd6f8b4a559613ca603b57498511740e65cd11a2e7dcec"
|
||||||
],
|
],
|
||||||
"version": "==2.4.2"
|
"version": "==2.4.6"
|
||||||
},
|
},
|
||||||
"pytest": {
|
"pytest": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
"sha256:27abc3fef618a01bebb1f0d6d303d2816a99aa87a5968ebc32fe971be91eb1e6",
|
"sha256:6b571215b5a790f9b41f19f3531c53a45cf6bb8ef2988bc1ff9afb38270b25fa",
|
||||||
"sha256:58cee9e09242937e136dbb3dab466116ba20d6b7828c7620f23947f37eb4dae4"
|
"sha256:e41d489ff43948babd0fad7ad5e49b8735d5d55e26628a58673c39ff61d95de4"
|
||||||
],
|
],
|
||||||
"index": "pypi",
|
"index": "pypi",
|
||||||
"version": "==5.2.2"
|
"version": "==5.3.2"
|
||||||
},
|
},
|
||||||
"pytest-cov": {
|
"pytest-cov": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
|
|
@ -381,22 +400,20 @@
|
||||||
},
|
},
|
||||||
"pyyaml": {
|
"pyyaml": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
"sha256:0113bc0ec2ad727182326b61326afa3d1d8280ae1122493553fd6f4397f33df9",
|
"sha256:059b2ee3194d718896c0ad077dd8c043e5e909d9180f387ce42012662a4946d6",
|
||||||
"sha256:01adf0b6c6f61bd11af6e10ca52b7d4057dd0be0343eb9283c878cf3af56aee4",
|
"sha256:1cf708e2ac57f3aabc87405f04b86354f66799c8e62c28c5fc5f88b5521b2dbf",
|
||||||
"sha256:5124373960b0b3f4aa7df1707e63e9f109b5263eca5976c66e08b1c552d4eaf8",
|
"sha256:24521fa2890642614558b492b473bee0ac1f8057a7263156b02e8b14c88ce6f5",
|
||||||
"sha256:5ca4f10adbddae56d824b2c09668e91219bb178a1eee1faa56af6f99f11bf696",
|
"sha256:4fee71aa5bc6ed9d5f116327c04273e25ae31a3020386916905767ec4fc5317e",
|
||||||
"sha256:7907be34ffa3c5a32b60b95f4d95ea25361c951383a894fec31be7252b2b6f34",
|
"sha256:70024e02197337533eef7b85b068212420f950319cc8c580261963aefc75f811",
|
||||||
"sha256:7ec9b2a4ed5cad025c2278a1e6a19c011c80a3caaac804fd2d329e9cc2c287c9",
|
"sha256:74782fbd4d4f87ff04159e986886931456a1894c61229be9eaf4de6f6e44b99e",
|
||||||
"sha256:87ae4c829bb25b9fe99cf71fbb2140c448f534e24c998cc60f39ae4f94396a73",
|
"sha256:940532b111b1952befd7db542c370887a8611660d2b9becff75d39355303d82d",
|
||||||
"sha256:9de9919becc9cc2ff03637872a440195ac4241c80536632fffeb6a1e25a74299",
|
"sha256:cb1f2f5e426dc9f07a7681419fe39cee823bb74f723f36f70399123f439e9b20",
|
||||||
"sha256:a5a85b10e450c66b49f98846937e8cfca1db3127a9d5d1e31ca45c3d0bef4c5b",
|
"sha256:dbbb2379c19ed6042e8f11f2a2c66d39cceb8aeace421bfc29d085d93eda3689",
|
||||||
"sha256:b0997827b4f6a7c286c01c5f60384d218dca4ed7d9efa945c3e1aa623d5709ae",
|
"sha256:e3a057b7a64f1222b56e47bcff5e4b94c4f61faac04c7c4ecb1985e18caa3994",
|
||||||
"sha256:b631ef96d3222e62861443cc89d6563ba3eeb816eeb96b2629345ab795e53681",
|
"sha256:e9f45bd5b92c7974e59bcd2dcc8631a6b6cc380a904725fce7bc08872e691615"
|
||||||
"sha256:bf47c0607522fdbca6c9e817a6e81b08491de50f3766a7a0e6a5be7905961b41",
|
|
||||||
"sha256:f81025eddd0327c7d4cfe9b62cf33190e1e736cc6e97502b3ec425f574b3e7a8"
|
|
||||||
],
|
],
|
||||||
"index": "pypi",
|
"index": "pypi",
|
||||||
"version": "==5.1.2"
|
"version": "==5.3"
|
||||||
},
|
},
|
||||||
"requests": {
|
"requests": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
|
|
@ -407,10 +424,10 @@
|
||||||
},
|
},
|
||||||
"six": {
|
"six": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
"sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c",
|
"sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd",
|
||||||
"sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73"
|
"sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66"
|
||||||
],
|
],
|
||||||
"version": "==1.12.0"
|
"version": "==1.13.0"
|
||||||
},
|
},
|
||||||
"toml": {
|
"toml": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
|
|
@ -421,32 +438,65 @@
|
||||||
},
|
},
|
||||||
"tox": {
|
"tox": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
"sha256:0bc216b6a2e6afe764476b4a07edf2c1dab99ed82bb146a1130b2e828f5bff5e",
|
"sha256:06ba73b149bf838d5cd25dc30c2dd2671ae5b2757cf98e5c41a35fe449f131b3",
|
||||||
"sha256:c4f6b319c20ba4913dbfe71ebfd14ff95d1853c4231493608182f66e566ecfe1"
|
"sha256:806d0a9217584558cc93747a945a9d9bff10b141a5287f0c8429a08828a22192"
|
||||||
],
|
],
|
||||||
"index": "pypi",
|
"index": "pypi",
|
||||||
"version": "==3.14.0"
|
"version": "==3.14.3"
|
||||||
|
},
|
||||||
|
"typed-ast": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:1170afa46a3799e18b4c977777ce137bb53c7485379d9706af8a59f2ea1aa161",
|
||||||
|
"sha256:18511a0b3e7922276346bcb47e2ef9f38fb90fd31cb9223eed42c85d1312344e",
|
||||||
|
"sha256:262c247a82d005e43b5b7f69aff746370538e176131c32dda9cb0f324d27141e",
|
||||||
|
"sha256:2b907eb046d049bcd9892e3076c7a6456c93a25bebfe554e931620c90e6a25b0",
|
||||||
|
"sha256:354c16e5babd09f5cb0ee000d54cfa38401d8b8891eefa878ac772f827181a3c",
|
||||||
|
"sha256:48e5b1e71f25cfdef98b013263a88d7145879fbb2d5185f2a0c79fa7ebbeae47",
|
||||||
|
"sha256:4e0b70c6fc4d010f8107726af5fd37921b666f5b31d9331f0bd24ad9a088e631",
|
||||||
|
"sha256:630968c5cdee51a11c05a30453f8cd65e0cc1d2ad0d9192819df9978984529f4",
|
||||||
|
"sha256:66480f95b8167c9c5c5c87f32cf437d585937970f3fc24386f313a4c97b44e34",
|
||||||
|
"sha256:71211d26ffd12d63a83e079ff258ac9d56a1376a25bc80b1cdcdf601b855b90b",
|
||||||
|
"sha256:7954560051331d003b4e2b3eb822d9dd2e376fa4f6d98fee32f452f52dd6ebb2",
|
||||||
|
"sha256:838997f4310012cf2e1ad3803bce2f3402e9ffb71ded61b5ee22617b3a7f6b6e",
|
||||||
|
"sha256:95bd11af7eafc16e829af2d3df510cecfd4387f6453355188342c3e79a2ec87a",
|
||||||
|
"sha256:bc6c7d3fa1325a0c6613512a093bc2a2a15aeec350451cbdf9e1d4bffe3e3233",
|
||||||
|
"sha256:cc34a6f5b426748a507dd5d1de4c1978f2eb5626d51326e43280941206c209e1",
|
||||||
|
"sha256:d755f03c1e4a51e9b24d899561fec4ccaf51f210d52abdf8c07ee2849b212a36",
|
||||||
|
"sha256:d7c45933b1bdfaf9f36c579671fec15d25b06c8398f113dab64c18ed1adda01d",
|
||||||
|
"sha256:d896919306dd0aa22d0132f62a1b78d11aaf4c9fc5b3410d3c666b818191630a",
|
||||||
|
"sha256:fdc1c9bbf79510b76408840e009ed65958feba92a88833cdceecff93ae8fff66",
|
||||||
|
"sha256:ffde2fbfad571af120fcbfbbc61c72469e72f550d676c3342492a9dfdefb8f12"
|
||||||
|
],
|
||||||
|
"version": "==1.4.0"
|
||||||
|
},
|
||||||
|
"typing-extensions": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:091ecc894d5e908ac75209f10d5b4f118fbdb2eb1ede6a63544054bb1edb41f2",
|
||||||
|
"sha256:910f4656f54de5993ad9304959ce9bb903f90aadc7c67a0bef07e678014e892d",
|
||||||
|
"sha256:cf8b63fedea4d89bab840ecbb93e75578af28f76f66c35889bd7065f5af88575"
|
||||||
|
],
|
||||||
|
"version": "==3.7.4.1"
|
||||||
},
|
},
|
||||||
"urllib3": {
|
"urllib3": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
"sha256:3de946ffbed6e6746608990594d08faac602528ac7015ac28d33cee6a45b7398",
|
"sha256:a8a318824cc77d1fd4b2bec2ded92646630d7fe8619497b142c84a9e6f5a7293",
|
||||||
"sha256:9a107b99a5393caf59c7aa3c1249c16e6879447533d0887f4336dde834c7be86"
|
"sha256:f3c5fd51747d450d4dcf6f923c81f78f811aab8205fda64b0aba34a4e48b0745"
|
||||||
],
|
],
|
||||||
"version": "==1.25.6"
|
"version": "==1.25.7"
|
||||||
},
|
},
|
||||||
"virtualenv": {
|
"virtualenv": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
"sha256:11cb4608930d5fd3afb545ecf8db83fa50e1f96fc4fca80c94b07d2c83146589",
|
"sha256:0d62c70883c0342d59c11d0ddac0d954d0431321a41ab20851facf2b222598f3",
|
||||||
"sha256:d257bb3773e48cac60e475a19b608996c73f4d333b3ba2e4e57d5ac6134e0136"
|
"sha256:55059a7a676e4e19498f1aad09b8313a38fcc0cdbe4fdddc0e9b06946d21b4bb"
|
||||||
],
|
],
|
||||||
"version": "==16.7.7"
|
"version": "==16.7.9"
|
||||||
},
|
},
|
||||||
"wcwidth": {
|
"wcwidth": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
"sha256:3df37372226d6e63e1b1e1eda15c594bca98a22d33a23832a90998faa96bc65e",
|
"sha256:8fd29383f539be45b20bd4df0dc29c20ba48654a41e661925e612311e9f3c603",
|
||||||
"sha256:f4ebe71925af7b40a864553f761ed559b43544f8f71746c2d756c7fe788ade7c"
|
"sha256:f28b3e8a6483e5d49e7f8949ac1a78314e740333ae305b4ba5defd3e74fb37a8"
|
||||||
],
|
],
|
||||||
"version": "==0.1.7"
|
"version": "==0.1.8"
|
||||||
},
|
},
|
||||||
"zipp": {
|
"zipp": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
|
|
|
||||||
|
|
@ -10,14 +10,16 @@ import random
|
||||||
import shutil
|
import shutil
|
||||||
import time
|
import time
|
||||||
import hashlib
|
import hashlib
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Dict, List, Tuple, Callable, Optional
|
||||||
|
|
||||||
import oletools.oleid
|
import oletools.oleid # type: ignore
|
||||||
import olefile
|
import olefile # type: ignore
|
||||||
import officedissector
|
import officedissector # type: ignore
|
||||||
import warnings
|
import warnings
|
||||||
import exifread
|
import exifread # type: ignore
|
||||||
from PIL import Image
|
from PIL import Image # type: ignore
|
||||||
from pdfid import PDFiD, cPDFiD
|
from pdfid import PDFiD, cPDFiD # type: ignore
|
||||||
|
|
||||||
from kittengroomer import FileBase, KittenGroomerBase, Logging
|
from kittengroomer import FileBase, KittenGroomerBase, Logging
|
||||||
|
|
||||||
|
|
@ -26,27 +28,27 @@ class Config:
|
||||||
"""Configuration information for filecheck.py."""
|
"""Configuration information for filecheck.py."""
|
||||||
# MIMES
|
# MIMES
|
||||||
# Application subtypes (mimetype: 'application/<subtype>')
|
# Application subtypes (mimetype: 'application/<subtype>')
|
||||||
mimes_ooxml = ('vnd.openxmlformats-officedocument.',)
|
mimes_ooxml: Tuple[str, ...] = ('vnd.openxmlformats-officedocument.',)
|
||||||
mimes_office = ('msword', 'vnd.ms-',)
|
mimes_office: Tuple[str, ...] = ('msword', 'vnd.ms-',)
|
||||||
mimes_libreoffice = ('vnd.oasis.opendocument',)
|
mimes_libreoffice: Tuple[str, ...] = ('vnd.oasis.opendocument',)
|
||||||
mimes_rtf = ('rtf', 'richtext',)
|
mimes_rtf: Tuple[str, ...] = ('rtf', 'richtext',)
|
||||||
mimes_pdf = ('pdf', 'postscript',)
|
mimes_pdf: Tuple[str, ...] = ('pdf', 'postscript',)
|
||||||
mimes_xml = ('xml',)
|
mimes_xml: Tuple[str, ...] = ('xml',)
|
||||||
mimes_ms = ('dosexec',)
|
mimes_ms: Tuple[str, ...] = ('dosexec',)
|
||||||
mimes_compressed = ('zip', 'rar', 'x-rar', 'bzip2', 'lzip', 'lzma', 'lzop',
|
mimes_compressed: Tuple[str, ...] = ('zip', 'rar', 'x-rar', 'bzip2', 'lzip', 'lzma', 'lzop',
|
||||||
'xz', 'compress', 'gzip', 'tar',)
|
'xz', 'compress', 'gzip', 'tar',)
|
||||||
mimes_data = ('octet-stream',)
|
mimes_data: Tuple[str, ...] = ('octet-stream',)
|
||||||
mimes_audio = ('ogg',)
|
mimes_audio: Tuple[str, ...] = ('ogg',)
|
||||||
|
|
||||||
# Image subtypes
|
# Image subtypes
|
||||||
mimes_exif = ('image/jpeg', 'image/tiff',)
|
mimes_exif: Tuple[str, ...] = ('image/jpeg', 'image/tiff',)
|
||||||
mimes_png = ('image/png',)
|
mimes_png: Tuple[str, ...] = ('image/png',)
|
||||||
|
|
||||||
# Mimetypes with metadata
|
# Mimetypes with metadata
|
||||||
mimes_metadata = ('image/jpeg', 'image/tiff', 'image/png',)
|
mimes_metadata: Tuple[str, ...] = ('image/jpeg', 'image/tiff', 'image/png',)
|
||||||
|
|
||||||
# Mimetype aliases
|
# Mimetype aliases
|
||||||
aliases = {
|
aliases: Dict[str, str] = {
|
||||||
# Win executables
|
# Win executables
|
||||||
'application/x-msdos-program': 'application/x-dosexec',
|
'application/x-msdos-program': 'application/x-dosexec',
|
||||||
'application/x-dosexec': 'application/x-msdos-program',
|
'application/x-dosexec': 'application/x-msdos-program',
|
||||||
|
|
@ -61,7 +63,7 @@ class Config:
|
||||||
# Commonly used malicious extensions
|
# Commonly used malicious extensions
|
||||||
# Sources: http://www.howtogeek.com/137270/50-file-extensions-that-are-potentially-dangerous-on-windows/
|
# Sources: http://www.howtogeek.com/137270/50-file-extensions-that-are-potentially-dangerous-on-windows/
|
||||||
# https://github.com/wiregit/wirecode/blob/master/components/core-settings/src/main/java/org/limewire/core/settings/FilterSettings.java
|
# https://github.com/wiregit/wirecode/blob/master/components/core-settings/src/main/java/org/limewire/core/settings/FilterSettings.java
|
||||||
malicious_exts = (
|
malicious_exts: Tuple[str, ...] = (
|
||||||
# Applications
|
# Applications
|
||||||
".exe", ".pif", ".application", ".gadget", ".msi", ".msp", ".com", ".scr",
|
".exe", ".pif", ".application", ".gadget", ".msi", ".msp", ".com", ".scr",
|
||||||
".hta", ".cpl", ".msc", ".jar",
|
".hta", ".cpl", ".msc", ".jar",
|
||||||
|
|
@ -116,7 +118,7 @@ class Config:
|
||||||
# In [12]: mimetypes.guess_type('toot.tar.gz', strict=False)
|
# In [12]: mimetypes.guess_type('toot.tar.gz', strict=False)
|
||||||
# Out[12]: ('application/x-tar', 'gzip')
|
# Out[12]: ('application/x-tar', 'gzip')
|
||||||
# It works as expected if you do mimetypes.guess_type('application/gzip', strict=False)
|
# It works as expected if you do mimetypes.guess_type('application/gzip', strict=False)
|
||||||
override_ext = {'.gz': 'application/gzip'}
|
override_ext: Dict[str, str] = {'.gz': 'application/gzip'}
|
||||||
|
|
||||||
|
|
||||||
SEVENZ_PATH = '/usr/bin/7z'
|
SEVENZ_PATH = '/usr/bin/7z'
|
||||||
|
|
@ -130,12 +132,12 @@ class File(FileBase):
|
||||||
filetype-specific processing methods.
|
filetype-specific processing methods.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, src_path, dst_path):
|
def __init__(self, src_path: Path, dst_path: Path):
|
||||||
super(File, self).__init__(src_path, dst_path)
|
super(File, self).__init__(src_path, dst_path)
|
||||||
self.is_archive = False
|
self.is_archive: bool = False
|
||||||
self.tempdir_path = self.dst_path + '_temp'
|
self.tempdir_path: Path = Path(str(self.dst_path) + '_temp')
|
||||||
|
|
||||||
subtypes_apps = (
|
subtypes_apps: Tuple[Tuple[Tuple[str, ...], Callable], ...] = (
|
||||||
(Config.mimes_office, self._winoffice),
|
(Config.mimes_office, self._winoffice),
|
||||||
(Config.mimes_ooxml, self._ooxml),
|
(Config.mimes_ooxml, self._ooxml),
|
||||||
(Config.mimes_rtf, self.text),
|
(Config.mimes_rtf, self.text),
|
||||||
|
|
@ -147,15 +149,15 @@ class File(FileBase):
|
||||||
(Config.mimes_data, self._binary_app),
|
(Config.mimes_data, self._binary_app),
|
||||||
(Config.mimes_audio, self.audio)
|
(Config.mimes_audio, self.audio)
|
||||||
)
|
)
|
||||||
self.app_subtype_methods = self._make_method_dict(subtypes_apps)
|
self.app_subtype_methods: Dict[str, Callable] = self._make_method_dict(subtypes_apps)
|
||||||
|
|
||||||
types_metadata = (
|
types_metadata: Tuple[Tuple[Tuple[str, ...], Callable], ...] = (
|
||||||
(Config.mimes_exif, self._metadata_exif),
|
(Config.mimes_exif, self._metadata_exif),
|
||||||
(Config.mimes_png, self._metadata_png),
|
(Config.mimes_png, self._metadata_png),
|
||||||
)
|
)
|
||||||
self.metadata_mimetype_methods = self._make_method_dict(types_metadata)
|
self.metadata_mimetype_methods: Dict[str, Callable] = self._make_method_dict(types_metadata)
|
||||||
|
|
||||||
self.mime_processing_options = {
|
self.mime_processing_options: Dict[str, Callable] = {
|
||||||
'text': self.text,
|
'text': self.text,
|
||||||
'audio': self.audio,
|
'audio': self.audio,
|
||||||
'image': self.image,
|
'image': self.image,
|
||||||
|
|
@ -199,7 +201,7 @@ class File(FileBase):
|
||||||
|
|
||||||
is_known_extension = self.extension in mimetypes.types_map.keys()
|
is_known_extension = self.extension in mimetypes.types_map.keys()
|
||||||
if is_known_extension and self.mimetype not in expected_mimetypes and not is_empty_file:
|
if is_known_extension and self.mimetype not in expected_mimetypes and not is_empty_file:
|
||||||
self.make_dangerous('Mimetype does not match expected mimetypes ({}) for this extension'.format(expected_mimetypes))
|
self.make_dangerous('Mimetype does not match expected mimetypes ({expected_mimetypes}) for this extension')
|
||||||
|
|
||||||
def _check_mimetype(self):
|
def _check_mimetype(self):
|
||||||
"""
|
"""
|
||||||
|
|
@ -273,7 +275,7 @@ class File(FileBase):
|
||||||
self.random_hashes.append((start_pos, hashed))
|
self.random_hashes.append((start_pos, hashed))
|
||||||
time.sleep(random.uniform(0.1, 0.5)) # Add a random sleep length
|
time.sleep(random.uniform(0.1, 0.5)) # Add a random sleep length
|
||||||
|
|
||||||
def _validate_random_hashes(self):
|
def _validate_random_hashes(self) -> bool:
|
||||||
"""Validate hashes computed by _compute_random_hashes"""
|
"""Validate hashes computed by _compute_random_hashes"""
|
||||||
if not os.path.exists(self.src_path) or os.path.isdir(self.src_path) or self.maintype == 'image':
|
if not os.path.exists(self.src_path) or os.path.isdir(self.src_path) or self.maintype == 'image':
|
||||||
# Images are converted, we don't have to fear TOCTOU
|
# Images are converted, we don't have to fear TOCTOU
|
||||||
|
|
@ -306,7 +308,7 @@ class File(FileBase):
|
||||||
self.mime_processing_options.get(self.maintype, self.unknown)()
|
self.mime_processing_options.get(self.maintype, self.unknown)()
|
||||||
|
|
||||||
# ##### Helper functions #####
|
# ##### Helper functions #####
|
||||||
def _make_method_dict(self, list_of_tuples):
|
def _make_method_dict(self, list_of_tuples: Tuple) -> Dict[str, Callable]:
|
||||||
"""Returns a dictionary with mimetype: method pairs."""
|
"""Returns a dictionary with mimetype: method pairs."""
|
||||||
dict_to_return = {}
|
dict_to_return = {}
|
||||||
for list_of_subtypes, method in list_of_tuples:
|
for list_of_subtypes, method in list_of_tuples:
|
||||||
|
|
@ -315,16 +317,16 @@ class File(FileBase):
|
||||||
return dict_to_return
|
return dict_to_return
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def has_metadata(self):
|
def has_metadata(self) -> bool:
|
||||||
"""True if filetype typically contains metadata, else False."""
|
"""True if filetype typically contains metadata, else False."""
|
||||||
if self.mimetype in Config.mimes_metadata:
|
if self.mimetype in Config.mimes_metadata:
|
||||||
return True
|
return True
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def make_tempdir(self):
|
def make_tempdir(self) -> Path:
|
||||||
"""Make a temporary directory at self.tempdir_path."""
|
"""Make a temporary directory at self.tempdir_path."""
|
||||||
if not os.path.exists(self.tempdir_path):
|
if not self.tempdir_path.exists():
|
||||||
os.makedirs(self.tempdir_path)
|
self.tempdir_path.mkdir()
|
||||||
return self.tempdir_path
|
return self.tempdir_path
|
||||||
|
|
||||||
#######################
|
#######################
|
||||||
|
|
@ -498,7 +500,7 @@ class File(FileBase):
|
||||||
|
|
||||||
#######################
|
#######################
|
||||||
# Metadata extractors
|
# Metadata extractors
|
||||||
def _metadata_exif(self, metadata_file_path):
|
def _metadata_exif(self, metadata_file_path) -> bool:
|
||||||
"""Read exif metadata from a jpg or tiff file using exifread."""
|
"""Read exif metadata from a jpg or tiff file using exifread."""
|
||||||
# TODO: can we shorten this method somehow?
|
# TODO: can we shorten this method somehow?
|
||||||
with open(self.src_path, 'rb') as img:
|
with open(self.src_path, 'rb') as img:
|
||||||
|
|
@ -527,7 +529,7 @@ class File(FileBase):
|
||||||
self.set_property('metadata', 'exif')
|
self.set_property('metadata', 'exif')
|
||||||
return True
|
return True
|
||||||
|
|
||||||
def _metadata_png(self, metadata_file_path):
|
def _metadata_png(self, metadata_file_path) -> bool:
|
||||||
"""Extract metadata from a png file using PIL/Pillow."""
|
"""Extract metadata from a png file using PIL/Pillow."""
|
||||||
warnings.simplefilter('error', Image.DecompressionBombWarning)
|
warnings.simplefilter('error', Image.DecompressionBombWarning)
|
||||||
try:
|
try:
|
||||||
|
|
@ -539,6 +541,7 @@ class File(FileBase):
|
||||||
metadata_file.write("Key: {}\tValue: {}\n".format(tag, img.info[tag]))
|
metadata_file.write("Key: {}\tValue: {}\n".format(tag, img.info[tag]))
|
||||||
# LOG: handle metadata
|
# LOG: handle metadata
|
||||||
self.set_property('metadata', 'png')
|
self.set_property('metadata', 'png')
|
||||||
|
return True
|
||||||
except Exception as e: # Catch decompression bombs
|
except Exception as e: # Catch decompression bombs
|
||||||
# TODO: only catch DecompressionBombWarnings here?
|
# TODO: only catch DecompressionBombWarnings here?
|
||||||
self.add_error(e, "Caught exception processing metadata for {}".format(self.src_path))
|
self.add_error(e, "Caught exception processing metadata for {}".format(self.src_path))
|
||||||
|
|
@ -582,7 +585,7 @@ class File(FileBase):
|
||||||
if self.has_metadata:
|
if self.has_metadata:
|
||||||
self.extract_metadata()
|
self.extract_metadata()
|
||||||
tempdir_path = self.make_tempdir()
|
tempdir_path = self.make_tempdir()
|
||||||
tempfile_path = os.path.join(tempdir_path, self.filename)
|
tempfile_path = tempdir_path / self.filename
|
||||||
warnings.simplefilter('error', Image.DecompressionBombWarning)
|
warnings.simplefilter('error', Image.DecompressionBombWarning)
|
||||||
try: # Do image conversions
|
try: # Do image conversions
|
||||||
with Image.open(self.src_path) as img_in:
|
with Image.open(self.src_path) as img_in:
|
||||||
|
|
@ -600,37 +603,37 @@ class File(FileBase):
|
||||||
class GroomerLogger(object):
|
class GroomerLogger(object):
|
||||||
"""Groomer logging interface."""
|
"""Groomer logging interface."""
|
||||||
|
|
||||||
def __init__(self, src_root_path, dst_root_path, debug=False):
|
def __init__(self, src_root_path: Path, dst_root_path: Path, debug: bool=False):
|
||||||
self._src_root_path = src_root_path
|
self._src_root_path: Path = src_root_path
|
||||||
self._dst_root_path = dst_root_path
|
self._dst_root_path: Path = dst_root_path
|
||||||
self._log_dir_path = self._make_log_dir(dst_root_path)
|
self._log_dir_path: Path = self._make_log_dir(dst_root_path)
|
||||||
self.log_path = os.path.join(self._log_dir_path, 'circlean_log.txt')
|
self.log_path: Path = self._log_dir_path / 'circlean_log.txt'
|
||||||
self._add_root_dir(src_root_path)
|
self._add_root_dir(src_root_path)
|
||||||
if debug:
|
if debug:
|
||||||
self.log_debug_err = os.path.join(self._log_dir_path, 'debug_stderr.log')
|
self.log_debug_err: Path = self._log_dir_path / 'debug_stderr.log'
|
||||||
self.log_debug_out = os.path.join(self._log_dir_path, 'debug_stdout.log')
|
self.log_debug_out: Path = self._log_dir_path / 'debug_stdout.log'
|
||||||
else:
|
else:
|
||||||
self.log_debug_err = os.devnull
|
self.log_debug_err = Path(os.devnull)
|
||||||
self.log_debug_out = os.devnull
|
self.log_debug_out = Path(os.devnull)
|
||||||
|
|
||||||
def _make_log_dir(self, root_dir_path):
|
def _make_log_dir(self, root_dir_path: Path) -> Path:
|
||||||
"""Create the directory in the dest dir that will hold the logs"""
|
"""Create the directory in the dest dir that will hold the logs"""
|
||||||
log_dir_path = os.path.join(root_dir_path, 'logs')
|
log_dir_path = root_dir_path / 'logs'
|
||||||
if os.path.exists(log_dir_path):
|
if os.path.exists(log_dir_path):
|
||||||
shutil.rmtree(log_dir_path)
|
shutil.rmtree(log_dir_path)
|
||||||
os.makedirs(log_dir_path)
|
os.makedirs(log_dir_path)
|
||||||
return log_dir_path
|
return log_dir_path
|
||||||
|
|
||||||
def _add_root_dir(self, root_path):
|
def _add_root_dir(self, root_path: Path):
|
||||||
"""Add the root directory to the log"""
|
"""Add the root directory to the log"""
|
||||||
dirname = os.path.split(root_path)[1] + '/'
|
dirname = os.path.split(root_path)[1] + '/'
|
||||||
with open(self.log_path, mode='ab') as lf:
|
with open(self.log_path, mode='ab') as lf:
|
||||||
lf.write(bytes(dirname, 'utf-8'))
|
lf.write(bytes(dirname, 'utf-8'))
|
||||||
lf.write(b'\n')
|
lf.write(b'\n')
|
||||||
|
|
||||||
def add_file(self, file_path, file_props, in_tempdir=False):
|
def add_file(self, file_path: Path, file_props: dict, in_tempdir: bool=False):
|
||||||
"""Add a file to the log. Takes a path and a dict of file properties."""
|
"""Add a file to the log. Takes a path and a dict of file properties."""
|
||||||
depth = self._get_path_depth(file_path)
|
depth = self._get_path_depth(str(file_path))
|
||||||
try:
|
try:
|
||||||
file_hash = Logging.computehash(file_path)[:6]
|
file_hash = Logging.computehash(file_path)[:6]
|
||||||
except IsADirectoryError:
|
except IsADirectoryError:
|
||||||
|
|
@ -671,34 +674,34 @@ class GroomerLogger(object):
|
||||||
depth -= 1
|
depth -= 1
|
||||||
self._write_line_to_log(log_string, depth)
|
self._write_line_to_log(log_string, depth)
|
||||||
|
|
||||||
def add_dir(self, dir_path):
|
def add_dir(self, dir_path: Path):
|
||||||
"""Add a directory to the log"""
|
"""Add a directory to the log"""
|
||||||
path_depth = self._get_path_depth(dir_path)
|
path_depth = self._get_path_depth(str(dir_path))
|
||||||
dirname = os.path.split(dir_path)[1] + '/'
|
dirname = os.path.split(str(dir_path))[1] + '/'
|
||||||
log_line = '+- ' + dirname
|
log_line = '+- ' + dirname
|
||||||
self._write_line_to_log(log_line, path_depth)
|
self._write_line_to_log(log_line, path_depth)
|
||||||
|
|
||||||
def _format_file_size(self, size):
|
def _format_file_size(self, size: int) -> str:
|
||||||
"""Returns a string with the file size and appropriate unit"""
|
"""Returns a string with the file size and appropriate unit"""
|
||||||
file_size = size
|
file_size = size
|
||||||
for unit in ('B', 'KB', 'MB', 'GB'):
|
for unit in ('B', 'KB', 'MB', 'GB'):
|
||||||
if file_size < 1024:
|
if file_size < 1024:
|
||||||
return str(int(file_size)) + unit
|
return str(int(file_size)) + unit
|
||||||
else:
|
else:
|
||||||
file_size = file_size / 1024
|
file_size = int(file_size / 1024)
|
||||||
return str(int(file_size)) + 'GB'
|
return str(int(file_size)) + 'GB'
|
||||||
|
|
||||||
def _get_path_depth(self, path):
|
def _get_path_depth(self, path: str) -> int:
|
||||||
"""Returns the relative path depth compared to root directory"""
|
"""Returns the relative path depth compared to root directory"""
|
||||||
if self._dst_root_path in path:
|
if str(self._dst_root_path) in path:
|
||||||
base_path = self._dst_root_path
|
base_path = str(self._dst_root_path)
|
||||||
elif self._src_root_path in path:
|
elif str(self._src_root_path) in path:
|
||||||
base_path = self._src_root_path
|
base_path = str(self._src_root_path)
|
||||||
relpath = os.path.relpath(path, base_path)
|
relpath = os.path.relpath(path, base_path)
|
||||||
path_depth = relpath.count(os.path.sep)
|
path_depth = relpath.count(os.path.sep)
|
||||||
return path_depth
|
return path_depth
|
||||||
|
|
||||||
def _write_line_to_log(self, line, indentation_depth):
|
def _write_line_to_log(self, line: str, indentation_depth: int):
|
||||||
"""
|
"""
|
||||||
Write a line to the log
|
Write a line to the log
|
||||||
|
|
||||||
|
|
@ -715,28 +718,28 @@ class GroomerLogger(object):
|
||||||
|
|
||||||
class KittenGroomerFileCheck(KittenGroomerBase):
|
class KittenGroomerFileCheck(KittenGroomerBase):
|
||||||
|
|
||||||
def __init__(self, root_src, root_dst, max_recursive_depth=2, debug=False):
|
def __init__(self, root_src: str, root_dst: str, max_recursive_depth: int=2, debug: bool=False):
|
||||||
super(KittenGroomerFileCheck, self).__init__(root_src, root_dst)
|
super(KittenGroomerFileCheck, self).__init__(root_src, root_dst)
|
||||||
self.recursive_archive_depth = 0
|
self.recursive_archive_depth = 0
|
||||||
self.max_recursive_depth = max_recursive_depth
|
self.max_recursive_depth = max_recursive_depth
|
||||||
self.logger = GroomerLogger(root_src, root_dst, debug)
|
self.logger = GroomerLogger(self.src_root_path, self.dst_root_path, debug)
|
||||||
|
|
||||||
def __repr__(self):
|
def __repr__(self):
|
||||||
return "filecheck.KittenGroomerFileCheck object: {{{}}}".format(
|
return "filecheck.KittenGroomerFileCheck object: {{{}}}".format(
|
||||||
os.path.basename(self.src_root_path)
|
os.path.basename(self.src_root_path)
|
||||||
)
|
)
|
||||||
|
|
||||||
def process_dir(self, src_dir, dst_dir):
|
def process_dir(self, src_dir: Path, dst_dir: Path):
|
||||||
"""Process a directory on the source key."""
|
"""Process a directory on the source key."""
|
||||||
for srcpath in self.list_files_dirs(src_dir):
|
for srcpath in self.list_files_dirs(src_dir):
|
||||||
if not os.path.islink(srcpath) and os.path.isdir(srcpath):
|
if not srcpath.is_symlink() and srcpath.is_dir():
|
||||||
self.logger.add_dir(srcpath)
|
self.logger.add_dir(srcpath)
|
||||||
else:
|
else:
|
||||||
dstpath = os.path.join(dst_dir, os.path.basename(srcpath))
|
dstpath = dst_dir / srcpath.name
|
||||||
cur_file = File(srcpath, dstpath)
|
cur_file = File(srcpath, dstpath)
|
||||||
self.process_file(cur_file)
|
self.process_file(cur_file)
|
||||||
|
|
||||||
def process_file(self, file):
|
def process_file(self, file: File):
|
||||||
"""
|
"""
|
||||||
Process an individual file.
|
Process an individual file.
|
||||||
|
|
||||||
|
|
@ -753,13 +756,13 @@ class KittenGroomerFileCheck(KittenGroomerBase):
|
||||||
if not file._validate_random_hashes():
|
if not file._validate_random_hashes():
|
||||||
# Something's fucked up.
|
# Something's fucked up.
|
||||||
file.make_dangerous('The copied file is different from the one checked, removing.')
|
file.make_dangerous('The copied file is different from the one checked, removing.')
|
||||||
os.remove(file.dst_path)
|
file.dst_path.unlink()
|
||||||
self.write_file_to_log(file)
|
self.write_file_to_log(file)
|
||||||
# TODO: Can probably handle cleaning up the tempdir better
|
# TODO: Can probably handle cleaning up the tempdir better
|
||||||
if hasattr(file, 'tempdir_path'):
|
if hasattr(file, 'tempdir_path'):
|
||||||
self.safe_rmtree(file.tempdir_path)
|
self.safe_rmtree(file.tempdir_path)
|
||||||
|
|
||||||
def process_archive(self, file):
|
def process_archive(self, file: File):
|
||||||
"""
|
"""
|
||||||
Unpack an archive using 7zip and process contents using process_dir.
|
Unpack an archive using 7zip and process contents using process_dir.
|
||||||
|
|
||||||
|
|
@ -781,25 +784,25 @@ class KittenGroomerFileCheck(KittenGroomerBase):
|
||||||
self.safe_rmtree(tempdir_path)
|
self.safe_rmtree(tempdir_path)
|
||||||
self.recursive_archive_depth -= 1
|
self.recursive_archive_depth -= 1
|
||||||
|
|
||||||
def _run_process(self, command_string, timeout=None):
|
def _run_process(self, command_string: str, timeout: Optional[int]=None) -> bool:
|
||||||
"""Run command_string in a subprocess, wait until it finishes."""
|
"""Run command_string in a subprocess, wait until it finishes."""
|
||||||
args = shlex.split(command_string)
|
args = shlex.split(command_string)
|
||||||
with open(self.logger.log_debug_err, 'ab') as stderr, open(self.logger.log_debug_out, 'ab') as stdout:
|
with open(self.logger.log_debug_err, 'ab') as stderr, open(self.logger.log_debug_out, 'ab') as stdout:
|
||||||
try:
|
try:
|
||||||
subprocess.check_call(args, stdout=stdout, stderr=stderr, timeout=timeout)
|
subprocess.check_call(args, stdout=stdout, stderr=stderr, timeout=timeout)
|
||||||
except (subprocess.TimeoutExpired, subprocess.CalledProcessError):
|
except (subprocess.TimeoutExpired, subprocess.CalledProcessError):
|
||||||
return
|
return False
|
||||||
return True
|
return True
|
||||||
|
|
||||||
def write_file_to_log(self, file):
|
def write_file_to_log(self, file: File):
|
||||||
"""Pass information about `file` to self.logger."""
|
"""Pass information about `file` to self.logger."""
|
||||||
props = file.get_all_props()
|
props = file.get_all_props()
|
||||||
if not file.is_archive:
|
if not file.is_archive:
|
||||||
# FIXME: in_tempdir is a hack to make image files appear at the correct tree depth in log
|
# FIXME: in_tempdir is a hack to make image files appear at the correct tree depth in log
|
||||||
in_tempdir = os.path.exists(file.tempdir_path)
|
in_tempdir = file.tempdir_path.exists()
|
||||||
self.logger.add_file(file.src_path, props, in_tempdir)
|
self.logger.add_file(file.src_path, props, in_tempdir)
|
||||||
|
|
||||||
def list_files_dirs(self, root_dir_path):
|
def list_files_dirs(self, root_dir_path: Path) -> List[Path]:
|
||||||
"""
|
"""
|
||||||
Returns a list of all files and directories
|
Returns a list of all files and directories
|
||||||
|
|
||||||
|
|
@ -807,14 +810,14 @@ class KittenGroomerFileCheck(KittenGroomerBase):
|
||||||
"""
|
"""
|
||||||
queue = []
|
queue = []
|
||||||
for path in sorted(os.listdir(root_dir_path), key=lambda x: str.lower(x)):
|
for path in sorted(os.listdir(root_dir_path), key=lambda x: str.lower(x)):
|
||||||
full_path = os.path.join(root_dir_path, path)
|
full_path = root_dir_path / path
|
||||||
# check for symlinks first to prevent getting trapped in infinite symlink recursion
|
# check for symlinks first to prevent getting trapped in infinite symlink recursion
|
||||||
if os.path.islink(full_path):
|
if full_path.is_symlink():
|
||||||
queue.append(full_path)
|
queue.append(full_path)
|
||||||
elif os.path.isdir(full_path):
|
elif full_path.is_dir():
|
||||||
queue.append(full_path)
|
queue.append(full_path)
|
||||||
queue += self.list_files_dirs(full_path)
|
queue += self.list_files_dirs(full_path)
|
||||||
elif os.path.isfile(full_path):
|
elif full_path.is_file():
|
||||||
queue.append(full_path)
|
queue.append(full_path)
|
||||||
return queue
|
return queue
|
||||||
|
|
||||||
|
|
@ -822,7 +825,7 @@ class KittenGroomerFileCheck(KittenGroomerBase):
|
||||||
self.process_dir(self.src_root_path, self.dst_root_path)
|
self.process_dir(self.src_root_path, self.dst_root_path)
|
||||||
|
|
||||||
|
|
||||||
def main(kg_implementation, description):
|
def main(kg_implementation, description: str):
|
||||||
parser = argparse.ArgumentParser(prog='KittenGroomer', description=description)
|
parser = argparse.ArgumentParser(prog='KittenGroomer', description=description)
|
||||||
parser.add_argument('-s', '--source', type=str, help='Source directory')
|
parser.add_argument('-s', '--source', type=str, help='Source directory')
|
||||||
parser.add_argument('-d', '--destination', type=str, help='Destination directory')
|
parser.add_argument('-d', '--destination', type=str, help='Destination directory')
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
#!/usr/bin/env python
|
#!/usr/bin/env python3
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
|
@ -13,8 +13,10 @@ import hashlib
|
||||||
import shutil
|
import shutil
|
||||||
import argparse
|
import argparse
|
||||||
import stat
|
import stat
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Union, Optional, List, Dict, Any, Tuple, Iterator
|
||||||
|
|
||||||
import magic
|
import magic # type: ignore
|
||||||
|
|
||||||
|
|
||||||
class FileBase(object):
|
class FileBase(object):
|
||||||
|
|
@ -24,49 +26,49 @@ class FileBase(object):
|
||||||
Contains file attributes and various helper methods.
|
Contains file attributes and various helper methods.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, src_path, dst_path):
|
def __init__(self, src_path: Path, dst_path: Path):
|
||||||
"""
|
"""
|
||||||
Initialized with the source path and expected destination path.
|
Initialized with the source path and expected destination path.
|
||||||
|
|
||||||
Create various properties and determine the file's mimetype.
|
Create various properties and determine the file's mimetype.
|
||||||
"""
|
"""
|
||||||
self.src_path = src_path
|
self.src_path: Path = src_path
|
||||||
self.dst_dir = os.path.dirname(dst_path)
|
self.dst_dir: Path = dst_path.parent
|
||||||
self.filename = os.path.basename(src_path)
|
self.filename: str = src_path.name
|
||||||
self.size = self._get_size(src_path)
|
self.size: int = self._get_size(src_path)
|
||||||
self.is_dangerous = False
|
self.is_dangerous: bool = False
|
||||||
self.copied = False
|
self.copied: bool = False
|
||||||
self.symlink_path = None
|
self.symlink_path = None
|
||||||
self._description_string = [] # array of descriptions to be joined
|
self._description_string: List[str] = [] # array of descriptions to be joined
|
||||||
self._errors = {}
|
self._errors: Dict[Exception, str] = {}
|
||||||
self._user_defined = {}
|
self._user_defined: Dict[str, str] = {}
|
||||||
self.should_copy = True
|
self.should_copy: bool = True
|
||||||
self.mimetype = self._determine_mimetype(src_path)
|
self.mimetype = self._determine_mimetype(str(src_path))
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def dst_path(self):
|
def dst_path(self) -> Path:
|
||||||
return os.path.join(self.dst_dir, self.filename)
|
return self.dst_dir / self.filename
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def extension(self):
|
def extension(self) -> Union[None, str]:
|
||||||
_, ext = os.path.splitext(self.filename)
|
ext = self.src_path.suffix
|
||||||
if ext == '':
|
if ext == '':
|
||||||
return None
|
return None
|
||||||
else:
|
else:
|
||||||
return ext.lower()
|
return ext.lower()
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def maintype(self):
|
def maintype(self) -> Optional[str]:
|
||||||
main, _ = self._split_mimetype(self.mimetype)
|
main, _ = self._split_mimetype(self.mimetype)
|
||||||
return main
|
return main
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def subtype(self):
|
def subtype(self) -> Optional[str]:
|
||||||
_, sub = self._split_mimetype(self.mimetype)
|
_, sub = self._split_mimetype(self.mimetype)
|
||||||
return sub
|
return sub
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def has_mimetype(self):
|
def has_mimetype(self) -> bool:
|
||||||
"""True if file has a main and sub mimetype, else False."""
|
"""True if file has a main and sub mimetype, else False."""
|
||||||
if not self.maintype or not self.subtype:
|
if not self.maintype or not self.subtype:
|
||||||
return False
|
return False
|
||||||
|
|
@ -74,7 +76,7 @@ class FileBase(object):
|
||||||
return True
|
return True
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def has_extension(self):
|
def has_extension(self) -> bool:
|
||||||
"""True if self.extension is set, else False."""
|
"""True if self.extension is set, else False."""
|
||||||
if self.extension is None:
|
if self.extension is None:
|
||||||
return False
|
return False
|
||||||
|
|
@ -82,7 +84,7 @@ class FileBase(object):
|
||||||
return True
|
return True
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def is_symlink(self):
|
def is_symlink(self) -> bool:
|
||||||
"""True if file is a symlink, else False."""
|
"""True if file is a symlink, else False."""
|
||||||
if self.symlink_path is None:
|
if self.symlink_path is None:
|
||||||
return False
|
return False
|
||||||
|
|
@ -90,27 +92,23 @@ class FileBase(object):
|
||||||
return True
|
return True
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def description_string(self):
|
def description_string(self) -> str:
|
||||||
if len(self._description_string) == 0:
|
if len(self._description_string) == 0:
|
||||||
return 'No description'
|
return 'No description'
|
||||||
elif len(self._description_string) == 1:
|
elif len(self._description_string) == 1:
|
||||||
return self._description_string[0]
|
return self._description_string[0]
|
||||||
else:
|
else:
|
||||||
ret_string = ', '.join(self._description_string)
|
ret_string = ', '.join(self._description_string)
|
||||||
return ret_string.strip(', ')
|
return ret_string.strip(', ') # NOTE: why strip?
|
||||||
|
|
||||||
@description_string.setter
|
@description_string.setter
|
||||||
def description_string(self, value):
|
def description_string(self, value: str):
|
||||||
if hasattr(self, 'description_string'):
|
if not isinstance(value, str):
|
||||||
if isinstance(value, str):
|
raise TypeError(f"value ({value}) must be a 'str' and not a {type(value)}")
|
||||||
if value not in self._description_string:
|
if value not in self._description_string:
|
||||||
self._description_string.append(value)
|
self._description_string.append(value)
|
||||||
else:
|
|
||||||
raise TypeError("Description_string can only include strings")
|
|
||||||
else:
|
|
||||||
self._description_string = value
|
|
||||||
|
|
||||||
def set_property(self, prop_string, value):
|
def set_property(self, prop_string: str, value: Any):
|
||||||
"""
|
"""
|
||||||
Take a property and a value and add them to the file's stored props.
|
Take a property and a value and add them to the file's stored props.
|
||||||
|
|
||||||
|
|
@ -123,7 +121,7 @@ class FileBase(object):
|
||||||
else:
|
else:
|
||||||
self._user_defined[prop_string] = value
|
self._user_defined[prop_string] = value
|
||||||
|
|
||||||
def get_property(self, prop_string):
|
def get_property(self, prop_string: str) -> Any:
|
||||||
"""
|
"""
|
||||||
Get the value for a property stored on the file.
|
Get the value for a property stored on the file.
|
||||||
|
|
||||||
|
|
@ -134,7 +132,7 @@ class FileBase(object):
|
||||||
except AttributeError:
|
except AttributeError:
|
||||||
return self._user_defined.get(prop_string, None)
|
return self._user_defined.get(prop_string, None)
|
||||||
|
|
||||||
def get_all_props(self):
|
def get_all_props(self) -> dict:
|
||||||
"""Return a dict containing all stored properties of this file."""
|
"""Return a dict containing all stored properties of this file."""
|
||||||
# Maybe move this onto the logger? I think that makes more sense
|
# Maybe move this onto the logger? I think that makes more sense
|
||||||
props_dict = {
|
props_dict = {
|
||||||
|
|
@ -155,11 +153,11 @@ class FileBase(object):
|
||||||
}
|
}
|
||||||
return props_dict
|
return props_dict
|
||||||
|
|
||||||
def add_error(self, error, info_string):
|
def add_error(self, error: Exception, info_string: str):
|
||||||
"""Add an `error`: `info_string` pair to the file."""
|
"""Add an `error`: `info_string` pair to the file."""
|
||||||
self._errors.update({error: info_string})
|
self._errors.update({error: info_string})
|
||||||
|
|
||||||
def add_description(self, description_string):
|
def add_description(self, description_string: str):
|
||||||
"""
|
"""
|
||||||
Add a description string to the file.
|
Add a description string to the file.
|
||||||
|
|
||||||
|
|
@ -167,7 +165,7 @@ class FileBase(object):
|
||||||
"""
|
"""
|
||||||
self.set_property('description_string', description_string)
|
self.set_property('description_string', description_string)
|
||||||
|
|
||||||
def make_dangerous(self, reason_string=None):
|
def make_dangerous(self, reason_string: Optional[str]=None):
|
||||||
"""
|
"""
|
||||||
Mark file as dangerous.
|
Mark file as dangerous.
|
||||||
|
|
||||||
|
|
@ -180,37 +178,33 @@ class FileBase(object):
|
||||||
if reason_string:
|
if reason_string:
|
||||||
self.add_description(reason_string)
|
self.add_description(reason_string)
|
||||||
|
|
||||||
def safe_copy(self, src=None, dst=None):
|
def safe_copy(self):
|
||||||
"""
|
"""
|
||||||
Copy file and create destination directories if needed.
|
Copy file and create destination directories if needed.
|
||||||
|
|
||||||
Sets all exec bits to '0'.
|
Sets all exec bits to '0'.
|
||||||
"""
|
"""
|
||||||
if src is None:
|
src = self.src_path
|
||||||
src = self.src_path
|
dst = self.dst_path
|
||||||
if dst is None:
|
|
||||||
dst = self.dst_path
|
|
||||||
try:
|
try:
|
||||||
os.makedirs(self.dst_dir, exist_ok=True)
|
self.dst_dir.mkdir(exist_ok=True)
|
||||||
shutil.copy(src, dst)
|
shutil.copy(str(src), str(dst))
|
||||||
current_perms = self._get_file_permissions(dst)
|
current_perms = self._get_file_permissions(dst)
|
||||||
only_exec_bits = 0o0111
|
only_exec_bits = 0o0111
|
||||||
perms_no_exec = current_perms & (~only_exec_bits)
|
perms_no_exec = current_perms & (~only_exec_bits)
|
||||||
os.chmod(dst, perms_no_exec)
|
dst.chmod(perms_no_exec)
|
||||||
except IOError as e:
|
except IOError as e:
|
||||||
# Probably means we can't write in the dest dir
|
# Probably means we can't write in the dest dir
|
||||||
self.add_error(e, '')
|
self.add_error(e, '')
|
||||||
|
|
||||||
def force_ext(self, extension):
|
def force_ext(self, extension: str):
|
||||||
"""If dst_path does not end in `extension`, append .ext to it."""
|
"""If dst_path does not end in `extension`, append .ext to it."""
|
||||||
new_ext = self._check_leading_dot(extension)
|
new_ext = self._check_leading_dot(extension)
|
||||||
if not self.filename.endswith(new_ext):
|
if not self.filename.endswith(new_ext):
|
||||||
# TODO: log that the extension was changed
|
# TODO: log that the extension was changed
|
||||||
self.filename += new_ext
|
self.filename += new_ext
|
||||||
if not self.get_property('extension') == new_ext:
|
|
||||||
self.set_property('extension', new_ext)
|
|
||||||
|
|
||||||
def create_metadata_file(self, extension):
|
def create_metadata_file(self, extension) -> Union[Path, bool]:
|
||||||
# TODO: this method name is confusing
|
# TODO: this method name is confusing
|
||||||
"""
|
"""
|
||||||
Create a separate file to hold extracted metadata.
|
Create a separate file to hold extracted metadata.
|
||||||
|
|
@ -220,29 +214,26 @@ class FileBase(object):
|
||||||
ext = self._check_leading_dot(extension)
|
ext = self._check_leading_dot(extension)
|
||||||
try:
|
try:
|
||||||
# Prevent using the same path as another file from src_path
|
# Prevent using the same path as another file from src_path
|
||||||
if os.path.exists(self.src_path + ext):
|
if Path(f'{self.src_path}{ext}').exists():
|
||||||
raise KittenGroomerError(
|
raise KittenGroomerError(f'Could not create metadata file for "{self.filename}": a file with that path exists.')
|
||||||
"Could not create metadata file for \"" +
|
|
||||||
self.filename +
|
|
||||||
"\": a file with that path exists.")
|
|
||||||
else:
|
else:
|
||||||
os.makedirs(self.dst_dir, exist_ok=True)
|
self.dst_dir.mkdir(exist_ok=True)
|
||||||
# TODO: shouldn't mutate state and also return something
|
# TODO: shouldn't mutate state and also return something
|
||||||
self.metadata_file_path = self.dst_path + ext
|
self.metadata_file_path = Path(f'{self.dst_path}{ext}')
|
||||||
return self.metadata_file_path
|
return self.metadata_file_path
|
||||||
# TODO: can probably let this exception bubble up
|
# TODO: can probably let this exception bubble up
|
||||||
except KittenGroomerError as e:
|
except KittenGroomerError as e:
|
||||||
self.add_error(e, '')
|
self.add_error(e, '')
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def _check_leading_dot(self, ext):
|
def _check_leading_dot(self, ext: str) -> str:
|
||||||
# TODO: this method name is confusing
|
# TODO: this method name is confusing
|
||||||
if len(ext) > 0:
|
if len(ext) > 0:
|
||||||
if not ext.startswith('.'):
|
if not ext.startswith('.'):
|
||||||
return '.' + ext
|
return '.' + ext
|
||||||
return ext
|
return ext
|
||||||
|
|
||||||
def _determine_mimetype(self, file_path):
|
def _determine_mimetype(self, file_path: str) -> str:
|
||||||
if os.path.islink(file_path):
|
if os.path.islink(file_path):
|
||||||
# libmagic will throw an IOError on a broken symlink
|
# libmagic will throw an IOError on a broken symlink
|
||||||
mimetype = 'inode/symlink'
|
mimetype = 'inode/symlink'
|
||||||
|
|
@ -256,19 +247,18 @@ class FileBase(object):
|
||||||
mt = None
|
mt = None
|
||||||
try:
|
try:
|
||||||
mimetype = mt.decode("utf-8")
|
mimetype = mt.decode("utf-8")
|
||||||
except:
|
except Exception:
|
||||||
# FIXME: what should the exception be if mimetype isn't utf-8?
|
# FIXME: what should the exception be if mimetype isn't utf-8?
|
||||||
mimetype = mt
|
mimetype = mt
|
||||||
return mimetype
|
return mimetype
|
||||||
|
|
||||||
def _split_mimetype(self, mimetype):
|
def _split_mimetype(self, mimetype: str) -> Tuple[Union[str, None], Union[str, None]]:
|
||||||
|
main_type, sub_type = None, None
|
||||||
if mimetype and '/' in mimetype:
|
if mimetype and '/' in mimetype:
|
||||||
main_type, sub_type = mimetype.split('/')
|
main_type, sub_type = mimetype.split('/')
|
||||||
else:
|
|
||||||
main_type, sub_type = None, None
|
|
||||||
return main_type, sub_type
|
return main_type, sub_type
|
||||||
|
|
||||||
def _get_size(self, file_path):
|
def _get_size(self, file_path: Path) -> int:
|
||||||
"""Filesize in bytes as an int, 0 if file does not exist."""
|
"""Filesize in bytes as an int, 0 if file does not exist."""
|
||||||
try:
|
try:
|
||||||
size = os.path.getsize(file_path)
|
size = os.path.getsize(file_path)
|
||||||
|
|
@ -276,23 +266,23 @@ class FileBase(object):
|
||||||
size = 0
|
size = 0
|
||||||
return size
|
return size
|
||||||
|
|
||||||
def _remove_exec_bit(self, file_path):
|
def _remove_exec_bit(self, file_path: Path):
|
||||||
current_perms = self._get_file_permissions(file_path)
|
current_perms = self._get_file_permissions(file_path)
|
||||||
perms_no_exec = current_perms & (~stat.S_IEXEC)
|
perms_no_exec = current_perms & (~stat.S_IEXEC)
|
||||||
os.chmod(file_path, perms_no_exec)
|
os.chmod(file_path, perms_no_exec)
|
||||||
|
|
||||||
def _get_file_permissions(self, file_path):
|
def _get_file_permissions(self, file_path: Path):
|
||||||
full_mode = os.stat(file_path, follow_symlinks=False).st_mode
|
full_mode = file_path.lstat().st_mode
|
||||||
return stat.S_IMODE(full_mode)
|
return stat.S_IMODE(full_mode)
|
||||||
|
|
||||||
|
|
||||||
class Logging(object):
|
class Logging(object):
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def computehash(path):
|
def computehash(path: Path) -> str:
|
||||||
"""Return the sha256 hash of a file at a given path."""
|
"""Return the sha256 hash of a file at a given path."""
|
||||||
s = hashlib.sha256()
|
s = hashlib.sha256()
|
||||||
with open(path, 'rb') as f:
|
with path.open('rb') as f:
|
||||||
while True:
|
while True:
|
||||||
buf = f.read(0x100000)
|
buf = f.read(0x100000)
|
||||||
if not buf:
|
if not buf:
|
||||||
|
|
@ -304,36 +294,35 @@ class Logging(object):
|
||||||
class KittenGroomerBase(object):
|
class KittenGroomerBase(object):
|
||||||
"""Base object responsible for copy/sanitization process."""
|
"""Base object responsible for copy/sanitization process."""
|
||||||
|
|
||||||
def __init__(self, src_root_path, dst_root_path):
|
def __init__(self, src_root_path: str, dst_root_path: str):
|
||||||
"""Initialized with path to source and dest directories."""
|
"""Initialized with path to source and dest directories."""
|
||||||
self.src_root_path = os.path.abspath(src_root_path)
|
self.src_root_path: Path = Path(os.path.abspath(src_root_path))
|
||||||
self.dst_root_path = os.path.abspath(dst_root_path)
|
self.dst_root_path: Path = Path(os.path.abspath(dst_root_path))
|
||||||
|
|
||||||
def safe_rmtree(self, directory_path):
|
def safe_rmtree(self, directory_path: Path):
|
||||||
"""Remove a directory tree if it exists."""
|
"""Remove a directory tree if it exists."""
|
||||||
if os.path.exists(directory_path):
|
if directory_path.is_dir():
|
||||||
shutil.rmtree(directory_path)
|
shutil.rmtree(directory_path)
|
||||||
|
|
||||||
def safe_remove(self, file_path):
|
def safe_remove(self, file_path: Path):
|
||||||
"""Remove file at file_path if it exists."""
|
"""Remove file at file_path if it exists."""
|
||||||
if os.path.exists(file_path):
|
if file_path.is_file():
|
||||||
os.remove(file_path)
|
os.remove(file_path)
|
||||||
|
|
||||||
def safe_mkdir(self, directory_path):
|
def safe_mkdir(self, directory_path: Path):
|
||||||
"""Make a directory if it does not exist."""
|
"""Make a directory if it does not exist."""
|
||||||
if not os.path.exists(directory_path):
|
if not directory_path.exists():
|
||||||
os.makedirs(directory_path)
|
os.makedirs(directory_path)
|
||||||
|
|
||||||
def list_all_files(self, directory_path):
|
def list_all_files(self, directory_path: Path) -> Iterator[Path]:
|
||||||
"""Generator yielding path to all of the files in a directory tree."""
|
"""Generator yielding path to all of the files in a directory tree."""
|
||||||
for root, dirs, files in os.walk(directory_path):
|
for root, dirs, files in os.walk(directory_path):
|
||||||
for filename in files:
|
for filename in files:
|
||||||
filepath = os.path.join(root, filename)
|
yield Path(root) / filename
|
||||||
yield filepath
|
|
||||||
|
|
||||||
#######################
|
#######################
|
||||||
|
|
||||||
def processdir(self, src_dir, dst_dir):
|
def processdir(self, src_dir: Path, dst_dir: Path):
|
||||||
"""Implement this function to define file processing behavior."""
|
"""Implement this function to define file processing behavior."""
|
||||||
raise ImplementationRequired('Please implement processdir.')
|
raise ImplementationRequired('Please implement processdir.')
|
||||||
|
|
||||||
|
|
@ -341,7 +330,7 @@ class KittenGroomerBase(object):
|
||||||
class KittenGroomerError(Exception):
|
class KittenGroomerError(Exception):
|
||||||
"""Base KittenGroomer exception handler."""
|
"""Base KittenGroomer exception handler."""
|
||||||
|
|
||||||
def __init__(self, message):
|
def __init__(self, message: str):
|
||||||
super(KittenGroomerError, self).__init__(message)
|
super(KittenGroomerError, self).__init__(message)
|
||||||
self.message = message
|
self.message = message
|
||||||
|
|
||||||
|
|
|
||||||
2
setup.py
2
setup.py
|
|
@ -1,6 +1,6 @@
|
||||||
#!/usr/bin/python
|
#!/usr/bin/python
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
from setuptools import setup
|
from setuptools import setup # type: ignore
|
||||||
|
|
||||||
setup(
|
setup(
|
||||||
name='kittengroomer',
|
name='kittengroomer',
|
||||||
|
|
|
||||||
|
|
@ -2,8 +2,9 @@
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
import pytest
|
import pytest # type: ignore
|
||||||
import yaml
|
import yaml
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
|
@ -19,16 +20,16 @@ skip = pytest.mark.skip
|
||||||
parametrize = pytest.mark.parametrize
|
parametrize = pytest.mark.parametrize
|
||||||
|
|
||||||
|
|
||||||
NORMAL_FILES_PATH = 'tests/normal/'
|
NORMAL_FILES_PATH = Path('tests/normal/')
|
||||||
DANGEROUS_FILES_PATH = 'tests/dangerous/'
|
DANGEROUS_FILES_PATH = Path('tests/dangerous/')
|
||||||
UNCATEGORIZED_FILES_PATH = 'tests/uncategorized'
|
UNCATEGORIZED_FILES_PATH = Path('tests/uncategorized')
|
||||||
CATALOG_PATH = 'tests/file_catalog.yaml'
|
CATALOG_PATH = Path('tests/file_catalog.yaml')
|
||||||
|
|
||||||
|
|
||||||
class SampleFile():
|
class SampleFile():
|
||||||
def __init__(self, path, exp_dangerous):
|
def __init__(self, path, exp_dangerous):
|
||||||
self.path = path
|
self.path = Path(path)
|
||||||
self.filename = os.path.basename(path)
|
self.filename = self.path.name
|
||||||
self.exp_dangerous = exp_dangerous
|
self.exp_dangerous = exp_dangerous
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -91,13 +92,13 @@ def get_filename(sample_file):
|
||||||
|
|
||||||
|
|
||||||
@fixture(scope='module')
|
@fixture(scope='module')
|
||||||
def src_dir_path(tmpdir_factory):
|
def src_dir_path(tmp_path_factory):
|
||||||
return tmpdir_factory.mktemp('src').strpath
|
return tmp_path_factory.mktemp('src')
|
||||||
|
|
||||||
|
|
||||||
@fixture(scope='module')
|
@fixture(scope='module')
|
||||||
def dest_dir_path(tmpdir_factory):
|
def dest_dir_path(tmp_path_factory):
|
||||||
return tmpdir_factory.mktemp('dest').strpath
|
return tmp_path_factory.mktemp('dest')
|
||||||
|
|
||||||
|
|
||||||
@fixture
|
@fixture
|
||||||
|
|
@ -113,7 +114,7 @@ def groomer(dest_dir_path):
|
||||||
def test_sample_files(sample_file, groomer, dest_dir_path):
|
def test_sample_files(sample_file, groomer, dest_dir_path):
|
||||||
if sample_file.xfail:
|
if sample_file.xfail:
|
||||||
pytest.xfail(reason='Marked xfail in file catalog')
|
pytest.xfail(reason='Marked xfail in file catalog')
|
||||||
file_dest_path = os.path.join(dest_dir_path, sample_file.filename)
|
file_dest_path = dest_dir_path / sample_file.filename
|
||||||
file = File(sample_file.path, file_dest_path)
|
file = File(sample_file.path, file_dest_path)
|
||||||
groomer.process_file(file)
|
groomer.process_file(file)
|
||||||
print(file.description_string)
|
print(file.description_string)
|
||||||
|
|
|
||||||
|
|
@ -4,7 +4,7 @@
|
||||||
import os
|
import os
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
import pytest
|
import pytest # type: ignore
|
||||||
|
|
||||||
try:
|
try:
|
||||||
from filecheck.filecheck import KittenGroomerFileCheck
|
from filecheck.filecheck import KittenGroomerFileCheck
|
||||||
|
|
|
||||||
|
|
@ -2,9 +2,10 @@
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
from pathlib import Path
|
||||||
import unittest.mock as mock
|
import unittest.mock as mock
|
||||||
|
|
||||||
import pytest
|
import pytest # type: ignore
|
||||||
|
|
||||||
from kittengroomer import FileBase, KittenGroomerBase
|
from kittengroomer import FileBase, KittenGroomerBase
|
||||||
from kittengroomer.helpers import ImplementationRequired
|
from kittengroomer.helpers import ImplementationRequired
|
||||||
|
|
@ -17,35 +18,34 @@ fixture = pytest.fixture
|
||||||
class TestFileBase:
|
class TestFileBase:
|
||||||
|
|
||||||
@fixture(scope='class')
|
@fixture(scope='class')
|
||||||
def src_dir_path(self, tmpdir_factory):
|
def src_dir_path(self, tmp_path_factory):
|
||||||
return tmpdir_factory.mktemp('src').strpath
|
return tmp_path_factory.mktemp('src')
|
||||||
|
|
||||||
@fixture(scope='class')
|
@fixture(scope='class')
|
||||||
def dest_dir_path(self, tmpdir_factory):
|
def dest_dir_path(self, tmp_path_factory):
|
||||||
return tmpdir_factory.mktemp('dest').strpath
|
return tmp_path_factory.mktemp('dest')
|
||||||
|
|
||||||
@fixture
|
@fixture(scope='class')
|
||||||
def tmpfile_path(self, tmpdir):
|
def tmpfile_path(self, src_dir_path):
|
||||||
file_path = tmpdir.join('test.txt')
|
path_src_file = src_dir_path / 'test.txt'
|
||||||
file_path.write('testing')
|
with path_src_file.open('w') as f:
|
||||||
return file_path.strpath
|
f.write('testing')
|
||||||
|
return path_src_file
|
||||||
|
|
||||||
@fixture
|
@fixture(scope='class')
|
||||||
def symlink_file_path(self, tmpdir, tmpfile_path):
|
def symlink_file_path(self, src_dir_path, tmpfile_path):
|
||||||
symlink_path = tmpdir.join('symlinked')
|
symlink_path = src_dir_path / 'symlinked'
|
||||||
symlink_path = symlink_path.strpath
|
symlink_path.symlink_to(tmpfile_path)
|
||||||
os.symlink(tmpfile_path, symlink_path)
|
|
||||||
return symlink_path
|
return symlink_path
|
||||||
|
|
||||||
@fixture
|
@fixture
|
||||||
def text_file(self):
|
def text_file(self, tmpfile_path, dest_dir_path):
|
||||||
with mock.patch(
|
with mock.patch(
|
||||||
'kittengroomer.helpers.magic.from_file',
|
'kittengroomer.helpers.magic.from_file',
|
||||||
return_value='text/plain'
|
return_value='text/plain'
|
||||||
):
|
):
|
||||||
src_path = 'src/test.txt'
|
dst_path = dest_dir_path / 'test.txt'
|
||||||
dst_path = 'dst/test.txt'
|
file = FileBase(tmpfile_path, dst_path)
|
||||||
file = FileBase(src_path, dst_path)
|
|
||||||
return file
|
return file
|
||||||
|
|
||||||
# Constructor behavior
|
# Constructor behavior
|
||||||
|
|
@ -53,63 +53,63 @@ class TestFileBase:
|
||||||
@mock.patch('kittengroomer.helpers.magic')
|
@mock.patch('kittengroomer.helpers.magic')
|
||||||
def test_init_identify_filename(self, mock_libmagic):
|
def test_init_identify_filename(self, mock_libmagic):
|
||||||
"""Init should identify the filename correctly for src_path."""
|
"""Init should identify the filename correctly for src_path."""
|
||||||
src_path = 'src/test.txt'
|
src_path = Path('src/test.txt')
|
||||||
dst_path = 'dst/test.txt'
|
dst_path = Path('dst/test.txt')
|
||||||
file = FileBase(src_path, dst_path)
|
file = FileBase(src_path, dst_path)
|
||||||
assert file.filename == 'test.txt'
|
assert file.filename == 'test.txt'
|
||||||
|
|
||||||
@mock.patch('kittengroomer.helpers.magic')
|
@mock.patch('kittengroomer.helpers.magic')
|
||||||
def test_init_identify_extension(self, mock_libmagic):
|
def test_init_identify_extension(self, mock_libmagic):
|
||||||
"""Init should identify the extension for src_path."""
|
"""Init should identify the extension for src_path."""
|
||||||
src_path = 'src/test.txt'
|
src_path = Path('src/test.txt')
|
||||||
dst_path = 'dst/test.txt'
|
dst_path = Path('dst/test.txt')
|
||||||
file = FileBase(src_path, dst_path)
|
file = FileBase(src_path, dst_path)
|
||||||
assert file.extension == '.txt'
|
assert file.extension == '.txt'
|
||||||
|
|
||||||
@mock.patch('kittengroomer.helpers.magic')
|
@mock.patch('kittengroomer.helpers.magic')
|
||||||
def test_init_uppercase_extension(self, mock_libmagic):
|
def test_init_uppercase_extension(self, mock_libmagic):
|
||||||
"""Init should coerce uppercase extension to lowercase"""
|
"""Init should coerce uppercase extension to lowercase"""
|
||||||
src_path = 'src/TEST.TXT'
|
src_path = Path('src/test.txt')
|
||||||
dst_path = 'dst/TEST.TXT'
|
dst_path = Path('dst/test.txt')
|
||||||
file = FileBase(src_path, dst_path)
|
file = FileBase(src_path, dst_path)
|
||||||
assert file.extension == '.txt'
|
assert file.extension == '.txt'
|
||||||
|
|
||||||
@mock.patch('kittengroomer.helpers.magic')
|
@mock.patch('kittengroomer.helpers.magic')
|
||||||
def test_has_extension_true(self, mock_libmagic):
|
def test_has_extension_true(self, mock_libmagic):
|
||||||
"""If the file has an extension, has_extension should == True."""
|
"""If the file has an extension, has_extension should == True."""
|
||||||
src_path = 'src/test.txt'
|
src_path = Path('src/test.txt')
|
||||||
dst_path = 'dst/test.txt'
|
dst_path = Path('dst/test.txt')
|
||||||
file = FileBase(src_path, dst_path)
|
file = FileBase(src_path, dst_path)
|
||||||
assert file.has_extension is True
|
assert file.has_extension is True
|
||||||
|
|
||||||
@mock.patch('kittengroomer.helpers.magic')
|
@mock.patch('kittengroomer.helpers.magic')
|
||||||
def test_has_extension_false(self, mock_libmagic):
|
def test_has_extension_false(self, mock_libmagic):
|
||||||
"""If the file has no extension, has_extensions should == False."""
|
"""If the file has no extension, has_extensions should == False."""
|
||||||
src_path = 'src/test'
|
src_path = Path('src/test')
|
||||||
dst_path = 'dst/test'
|
dst_path = Path('dst/test')
|
||||||
file = FileBase(src_path, dst_path)
|
file = FileBase(src_path, dst_path)
|
||||||
assert file.has_extension is False
|
assert file.has_extension is False
|
||||||
|
|
||||||
def test_init_file_doesnt_exist(self):
|
def test_init_file_doesnt_exist(self):
|
||||||
"""Init should raise an exception if the file doesn't exist."""
|
"""Init should raise an exception if the file doesn't exist."""
|
||||||
with pytest.raises(FileNotFoundError):
|
with pytest.raises(FileNotFoundError):
|
||||||
FileBase('', '')
|
FileBase(Path('non_existent'), Path('non_existent'))
|
||||||
|
|
||||||
def test_init_srcpath_is_directory(self, tmpdir):
|
def test_init_srcpath_is_directory(self, tmpdir):
|
||||||
"""Init should raise an exception if given a path to a directory."""
|
"""Init should raise an exception if given a path to a directory."""
|
||||||
with pytest.raises(IsADirectoryError):
|
with pytest.raises(IsADirectoryError):
|
||||||
FileBase(tmpdir.strpath, tmpdir.strpath)
|
FileBase(Path(tmpdir.strpath), Path(tmpdir.strpath))
|
||||||
|
|
||||||
@mock.patch('kittengroomer.helpers.magic')
|
@mock.patch('kittengroomer.helpers.magic')
|
||||||
def test_init_symlink(self, mock_libmagic, symlink_file_path):
|
def test_init_symlink(self, mock_libmagic, symlink_file_path, tmpdir):
|
||||||
"""Init should properly identify symlinks."""
|
"""Init should properly identify symlinks."""
|
||||||
file = FileBase(symlink_file_path, '')
|
file = FileBase(symlink_file_path, Path(tmpdir.strpath))
|
||||||
assert file.mimetype == 'inode/symlink'
|
assert file.mimetype == 'inode/symlink'
|
||||||
|
|
||||||
@mock.patch('kittengroomer.helpers.magic')
|
@mock.patch('kittengroomer.helpers.magic')
|
||||||
def test_is_symlink_attribute(self, mock_libmagic, symlink_file_path):
|
def test_is_symlink_attribute(self, mock_libmagic, symlink_file_path, tmpdir):
|
||||||
"""If a file is a symlink, is_symlink should return True."""
|
"""If a file is a symlink, is_symlink should return True."""
|
||||||
file = FileBase(symlink_file_path, '')
|
file = FileBase(symlink_file_path, Path(tmpdir.strpath))
|
||||||
assert file.is_symlink is True
|
assert file.is_symlink is True
|
||||||
|
|
||||||
def test_init_mimetype_attribute_assigned_correctly(self):
|
def test_init_mimetype_attribute_assigned_correctly(self):
|
||||||
|
|
@ -117,7 +117,7 @@ class TestFileBase:
|
||||||
assigned properly."""
|
assigned properly."""
|
||||||
with mock.patch('kittengroomer.helpers.magic.from_file',
|
with mock.patch('kittengroomer.helpers.magic.from_file',
|
||||||
return_value='text/plain'):
|
return_value='text/plain'):
|
||||||
file = FileBase('', '')
|
file = FileBase(Path('non_existent'), Path('non_existent'))
|
||||||
assert file.mimetype == 'text/plain'
|
assert file.mimetype == 'text/plain'
|
||||||
|
|
||||||
def test_maintype_and_subtype_attributes(self):
|
def test_maintype_and_subtype_attributes(self):
|
||||||
|
|
@ -125,7 +125,7 @@ class TestFileBase:
|
||||||
the appropriate values."""
|
the appropriate values."""
|
||||||
with mock.patch('kittengroomer.helpers.magic.from_file',
|
with mock.patch('kittengroomer.helpers.magic.from_file',
|
||||||
return_value='text/plain'):
|
return_value='text/plain'):
|
||||||
file = FileBase('', '')
|
file = FileBase(Path('non_existent'), Path('non_existent'))
|
||||||
assert file.maintype == 'text'
|
assert file.maintype == 'text'
|
||||||
assert file.subtype == 'plain'
|
assert file.subtype == 'plain'
|
||||||
|
|
||||||
|
|
@ -133,14 +133,14 @@ class TestFileBase:
|
||||||
"""If a file doesn't have a full mimetype has_mimetype should == False."""
|
"""If a file doesn't have a full mimetype has_mimetype should == False."""
|
||||||
with mock.patch('kittengroomer.helpers.magic.from_file',
|
with mock.patch('kittengroomer.helpers.magic.from_file',
|
||||||
return_value='data'):
|
return_value='data'):
|
||||||
file = FileBase('', '')
|
file = FileBase(Path('non_existent'), Path('non_existent'))
|
||||||
assert file.has_mimetype is False
|
assert file.has_mimetype is False
|
||||||
|
|
||||||
def test_has_mimetype_mimetype_is_none(self):
|
def test_has_mimetype_mimetype_is_none(self):
|
||||||
"""If a file doesn't have a full mimetype has_mimetype should == False."""
|
"""If a file doesn't have a full mimetype has_mimetype should == False."""
|
||||||
with mock.patch('kittengroomer.helpers.FileBase._determine_mimetype',
|
with mock.patch('kittengroomer.helpers.FileBase._determine_mimetype',
|
||||||
return_value=None):
|
return_value=None):
|
||||||
file = FileBase('', '')
|
file = FileBase(Path('non_existent'), Path('non_existent'))
|
||||||
assert file.has_mimetype is False
|
assert file.has_mimetype is False
|
||||||
|
|
||||||
# File properties
|
# File properties
|
||||||
|
|
@ -223,38 +223,38 @@ class TestFileBase:
|
||||||
"""Force_ext should modify the path of the file to end in the
|
"""Force_ext should modify the path of the file to end in the
|
||||||
new extension."""
|
new extension."""
|
||||||
text_file.force_ext('.test')
|
text_file.force_ext('.test')
|
||||||
assert text_file.dst_path.endswith('.test')
|
assert text_file.dst_path.name.endswith('.test')
|
||||||
|
|
||||||
def test_force_ext_add_dot(self, text_file):
|
def test_force_ext_add_dot(self, text_file):
|
||||||
"""Force_ext should add a dot to an extension given without one."""
|
"""Force_ext should add a dot to an extension given without one."""
|
||||||
text_file.force_ext('test')
|
text_file.force_ext('test')
|
||||||
assert text_file.dst_path.endswith('.test')
|
assert text_file.dst_path.name.endswith('.test')
|
||||||
|
|
||||||
def test_force_ext_change_extension_attr(self, text_file):
|
def test_force_ext_change_extension_attr(self, text_file):
|
||||||
"""Force_ext should modify the extension attribute"""
|
"""Force_ext should only modify the extension of the destination file"""
|
||||||
text_file.force_ext('.thing')
|
text_file.force_ext('.thing')
|
||||||
assert text_file.extension == '.thing'
|
assert text_file.extension == '.txt'
|
||||||
|
|
||||||
def test_force_ext_no_change(self, text_file):
|
def test_force_ext_no_change(self, text_file):
|
||||||
"""Force_ext should do nothing if the current extension is the same
|
"""Force_ext should do nothing if the current extension is the same
|
||||||
as the new extension."""
|
as the new extension."""
|
||||||
text_file.force_ext('.txt')
|
text_file.force_ext('.txt')
|
||||||
assert text_file.extension == '.txt'
|
assert text_file.extension == '.txt'
|
||||||
assert '.txt.txt' not in text_file.dst_path
|
assert '.txt.txt' not in text_file.dst_path.name
|
||||||
|
|
||||||
def test_safe_copy_calls_copy(self, src_dir_path, dest_dir_path):
|
def test_safe_copy_calls_copy(self, src_dir_path, dest_dir_path):
|
||||||
"""Calling safe_copy should copy the file from the correct path to
|
"""Calling safe_copy should copy the file from the correct path to
|
||||||
the correct destination path."""
|
the correct destination path."""
|
||||||
file_path = os.path.join(src_dir_path, 'test.txt')
|
file_path = src_dir_path / 'test.txt'
|
||||||
with open(file_path, 'w+') as file:
|
with open(file_path, 'w+') as file:
|
||||||
file.write('')
|
file.write('')
|
||||||
dst_path = os.path.join(dest_dir_path, 'test.txt')
|
dst_path = dest_dir_path / 'test.txt'
|
||||||
with mock.patch('kittengroomer.helpers.magic.from_file',
|
with mock.patch('kittengroomer.helpers.magic.from_file',
|
||||||
return_value='text/plain'):
|
return_value='text/plain'):
|
||||||
file = FileBase(file_path, dst_path)
|
file = FileBase(file_path, dst_path)
|
||||||
with mock.patch('kittengroomer.helpers.shutil.copy') as mock_copy:
|
with mock.patch('kittengroomer.helpers.shutil.copy') as mock_copy:
|
||||||
file.safe_copy()
|
file.safe_copy()
|
||||||
mock_copy.assert_called_once_with(file_path, dst_path)
|
mock_copy.assert_called_once_with(str(file_path), str(dst_path))
|
||||||
|
|
||||||
def test_safe_copy_removes_exec_perms(self):
|
def test_safe_copy_removes_exec_perms(self):
|
||||||
"""`safe_copy` should create a file that doesn't have any of the
|
"""`safe_copy` should create a file that doesn't have any of the
|
||||||
|
|
@ -289,22 +289,27 @@ class TestKittenGroomerBase:
|
||||||
|
|
||||||
@fixture(scope='class')
|
@fixture(scope='class')
|
||||||
def src_dir_path(self, tmpdir_factory):
|
def src_dir_path(self, tmpdir_factory):
|
||||||
return tmpdir_factory.mktemp('src').strpath
|
return Path(tmpdir_factory.mktemp('src').strpath)
|
||||||
|
|
||||||
@fixture(scope='class')
|
@fixture(scope='class')
|
||||||
def dest_dir_path(self, tmpdir_factory):
|
def dest_dir_path(self, tmpdir_factory):
|
||||||
return tmpdir_factory.mktemp('dest').strpath
|
return Path(tmpdir_factory.mktemp('dest').strpath)
|
||||||
|
|
||||||
|
@fixture(scope='class')
|
||||||
|
def tmpfile_path(self, src_dir_path):
|
||||||
|
path_src_file = src_dir_path / 'test.txt'
|
||||||
|
with path_src_file.open('w') as f:
|
||||||
|
f.write('testing')
|
||||||
|
return path_src_file
|
||||||
|
|
||||||
@fixture
|
@fixture
|
||||||
def groomer(self, src_dir_path, dest_dir_path):
|
def groomer(self, src_dir_path, dest_dir_path):
|
||||||
return KittenGroomerBase(src_dir_path, dest_dir_path)
|
return KittenGroomerBase(src_dir_path, dest_dir_path)
|
||||||
|
|
||||||
def test_list_all_files_includes_file(self, tmpdir, groomer):
|
def test_list_all_files_includes_file(self, src_dir_path, tmpfile_path, groomer):
|
||||||
"""Calling list_all_files should include files in the given path."""
|
"""Calling list_all_files should include files in the given path."""
|
||||||
file = tmpdir.join('test.txt')
|
files = groomer.list_all_files(src_dir_path)
|
||||||
file.write('testing')
|
assert tmpfile_path in files
|
||||||
files = groomer.list_all_files(tmpdir.strpath)
|
|
||||||
assert file.strpath in files
|
|
||||||
|
|
||||||
def test_list_all_files_excludes_dir(self, tmpdir, groomer):
|
def test_list_all_files_excludes_dir(self, tmpdir, groomer):
|
||||||
"""Calling list_all_files shouldn't include directories in the given
|
"""Calling list_all_files shouldn't include directories in the given
|
||||||
|
|
@ -317,12 +322,12 @@ class TestKittenGroomerBase:
|
||||||
def test_safe_remove(self, groomer, src_dir_path):
|
def test_safe_remove(self, groomer, src_dir_path):
|
||||||
"""Calling safe_remove should not raise an Exception if trying to
|
"""Calling safe_remove should not raise an Exception if trying to
|
||||||
remove a file that doesn't exist."""
|
remove a file that doesn't exist."""
|
||||||
groomer.safe_remove(os.path.join(src_dir_path, 'thing'))
|
groomer.safe_remove(src_dir_path / 'thing')
|
||||||
|
|
||||||
def test_safe_mkdir_file_exists(self, groomer, dest_dir_path):
|
def test_safe_mkdir_file_exists(self, groomer, dest_dir_path):
|
||||||
"""Calling safe_mkdir should not overwrite an existing directory."""
|
"""Calling safe_mkdir should not overwrite an existing directory."""
|
||||||
filepath = os.path.join(dest_dir_path, 'thing')
|
filepath = dest_dir_path / 'thing'
|
||||||
os.mkdir(filepath)
|
filepath.mkdir()
|
||||||
groomer.safe_mkdir(filepath)
|
groomer.safe_mkdir(filepath)
|
||||||
|
|
||||||
def test_processdir_not_implemented(self, groomer):
|
def test_processdir_not_implemented(self, groomer):
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue