From d0f7d1bf8119d53b2fbc5e9fcb2d1b19569e582b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 13 Jan 2020 16:04:51 +0100 Subject: [PATCH] new: Add typing everywhere, use pathlib when possible --- .travis.yml | 6 +- Pipfile | 1 + Pipfile.lock | 436 ++++++++++++++++++-------------- filecheck/filecheck.py | 171 +++++++------ kittengroomer/helpers.py | 159 ++++++------ setup.py | 2 +- tests/test_filecheck.py | 25 +- tests/test_filecheck_logging.py | 2 +- tests/test_kittengroomer.py | 119 ++++----- 9 files changed, 486 insertions(+), 435 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4f10de2..82137a8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,7 @@ python: - 3.6 - "3.6-dev" - "3.7-dev" + - "3.8-dev" # - nightly sudo: required @@ -25,8 +26,8 @@ install: - sudo apt-get install -y p7zip-rar # filecheck.py dependencies - sudo apt-get install libxml2-dev libxslt1-dev - - wget https://didierstevens.com/files/software/pdfid_v0_2_5.zip - - unzip pdfid_v0_2_5.zip + - wget https://didierstevens.com/files/software/pdfid_v0_2_7.zip + - unzip pdfid_v0_2_7.zip - pip install -U pip pipenv # PyCIRCLean dependencies - pipenv install -d @@ -73,6 +74,7 @@ install: # - popd script: + - pipenv run mypy kittengroomer/ filecheck/ tests/ scripts/ --ignore-missing-imports - travis_wait 60 pipenv run py.test --cov=kittengroomer --cov=filecheck tests/ notifications: diff --git a/Pipfile b/Pipfile index 61bca89..faa6b5d 100644 --- a/Pipfile +++ b/Pipfile @@ -10,6 +10,7 @@ pytest-cov = "*" tox = "*" PyYAML = "*" codecov = "*" +mypy = "*" [packages] lxml = "*" diff --git a/Pipfile.lock b/Pipfile.lock index ff575e5..b119efc 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "b78c4dbc4788f0e9c8400519c1dfb8d4b45ea65b7ffb236967d8d402b34543b2" + "sha256": "3771d02efb26aefa7e58aaf26720703899f894e4b8e8854be484a154e64aa1dc" }, "pipfile-spec": 6, "requires": { @@ -18,40 +18,41 @@ "default": { "cffi": { "hashes": [ - "sha256:00d890313797d9fe4420506613384b43099ad7d2b905c0752dbcc3a6f14d80fa", - "sha256:0cf9e550ac6c5e57b713437e2f4ac2d7fd0cd10336525a27224f5fc1ec2ee59a", - "sha256:0ea23c9c0cdd6778146a50d867d6405693ac3b80a68829966c98dd5e1bbae400", - "sha256:193697c2918ecdb3865acf6557cddf5076bb39f1f654975e087b67efdff83365", - "sha256:1ae14b542bf3b35e5229439c35653d2ef7d8316c1fffb980f9b7647e544baa98", - "sha256:1e389e069450609c6ffa37f21f40cce36f9be7643bbe5051ab1de99d5a779526", - "sha256:263242b6ace7f9cd4ea401428d2d45066b49a700852334fd55311bde36dcda14", - "sha256:33142ae9807665fa6511cfa9857132b2c3ee6ddffb012b3f0933fc11e1e830d5", - "sha256:364f8404034ae1b232335d8c7f7b57deac566f148f7222cef78cf8ae28ef764e", - "sha256:47368f69fe6529f8f49a5d146ddee713fc9057e31d61e8b6dc86a6a5e38cecc1", - "sha256:4895640844f17bec32943995dc8c96989226974dfeb9dd121cc45d36e0d0c434", - "sha256:558b3afef987cf4b17abd849e7bedf64ee12b28175d564d05b628a0f9355599b", - "sha256:5ba86e1d80d458b338bda676fd9f9d68cb4e7a03819632969cf6d46b01a26730", - "sha256:63424daa6955e6b4c70dc2755897f5be1d719eabe71b2625948b222775ed5c43", - "sha256:6381a7d8b1ebd0bc27c3bc85bc1bfadbb6e6f756b4d4db0aa1425c3719ba26b4", - "sha256:6381ab708158c4e1639da1f2a7679a9bbe3e5a776fc6d1fd808076f0e3145331", - "sha256:6fd58366747debfa5e6163ada468a90788411f10c92597d3b0a912d07e580c36", - "sha256:728ec653964655d65408949b07f9b2219df78badd601d6c49e28d604efe40599", - "sha256:7cfcfda59ef1f95b9f729c56fe8a4041899f96b72685d36ef16a3440a0f85da8", - "sha256:819f8d5197c2684524637f940445c06e003c4a541f9983fd30d6deaa2a5487d8", - "sha256:825ecffd9574557590e3225560a8a9d751f6ffe4a49e3c40918c9969b93395fa", - "sha256:8a2bcae2258d00fcfc96a9bde4a6177bc4274fe033f79311c5dd3d3148c26518", - "sha256:9009e917d8f5ef780c2626e29b6bc126f4cb2a4d43ca67aa2b40f2a5d6385e78", - "sha256:9c77564a51d4d914ed5af096cd9843d90c45b784b511723bd46a8a9d09cf16fc", - "sha256:a19089fa74ed19c4fe96502a291cfdb89223a9705b1d73b3005df4256976142e", - "sha256:a40ed527bffa2b7ebe07acc5a3f782da072e262ca994b4f2085100b5a444bbb2", - "sha256:b8f09f21544b9899defb09afbdaeb200e6a87a2b8e604892940044cf94444644", - "sha256:bb75ba21d5716abc41af16eac1145ab2e471deedde1f22c6f99bd9f995504df0", - "sha256:e22a00c0c81ffcecaf07c2bfb3672fa372c50e2bd1024ffee0da191c1b27fc71", - "sha256:e55b5a746fb77f10c83e8af081979351722f6ea48facea79d470b3731c7b2891", - "sha256:ec2fa3ee81707a5232bf2dfbd6623fdb278e070d596effc7e2d788f2ada71a05", - "sha256:fd82eb4694be712fcae03c717ca2e0fc720657ac226b80bbb597e971fc6928c2" + "sha256:0b49274afc941c626b605fb59b59c3485c17dc776dc3cc7cc14aca74cc19cc42", + "sha256:0e3ea92942cb1168e38c05c1d56b0527ce31f1a370f6117f1d490b8dcd6b3a04", + "sha256:135f69aecbf4517d5b3d6429207b2dff49c876be724ac0c8bf8e1ea99df3d7e5", + "sha256:19db0cdd6e516f13329cba4903368bff9bb5a9331d3410b1b448daaadc495e54", + "sha256:2781e9ad0e9d47173c0093321bb5435a9dfae0ed6a762aabafa13108f5f7b2ba", + "sha256:291f7c42e21d72144bb1c1b2e825ec60f46d0a7468f5346841860454c7aa8f57", + "sha256:2c5e309ec482556397cb21ede0350c5e82f0eb2621de04b2633588d118da4396", + "sha256:2e9c80a8c3344a92cb04661115898a9129c074f7ab82011ef4b612f645939f12", + "sha256:32a262e2b90ffcfdd97c7a5e24a6012a43c61f1f5a57789ad80af1d26c6acd97", + "sha256:3c9fff570f13480b201e9ab69453108f6d98244a7f495e91b6c654a47486ba43", + "sha256:415bdc7ca8c1c634a6d7163d43fb0ea885a07e9618a64bda407e04b04333b7db", + "sha256:42194f54c11abc8583417a7cf4eaff544ce0de8187abaf5d29029c91b1725ad3", + "sha256:4424e42199e86b21fc4db83bd76909a6fc2a2aefb352cb5414833c030f6ed71b", + "sha256:4a43c91840bda5f55249413037b7a9b79c90b1184ed504883b72c4df70778579", + "sha256:599a1e8ff057ac530c9ad1778293c665cb81a791421f46922d80a86473c13346", + "sha256:5c4fae4e9cdd18c82ba3a134be256e98dc0596af1e7285a3d2602c97dcfa5159", + "sha256:5ecfa867dea6fabe2a58f03ac9186ea64da1386af2159196da51c4904e11d652", + "sha256:62f2578358d3a92e4ab2d830cd1c2049c9c0d0e6d3c58322993cc341bdeac22e", + "sha256:6471a82d5abea994e38d2c2abc77164b4f7fbaaf80261cb98394d5793f11b12a", + "sha256:6d4f18483d040e18546108eb13b1dfa1000a089bcf8529e30346116ea6240506", + "sha256:71a608532ab3bd26223c8d841dde43f3516aa5d2bf37b50ac410bb5e99053e8f", + "sha256:74a1d8c85fb6ff0b30fbfa8ad0ac23cd601a138f7509dc617ebc65ef305bb98d", + "sha256:7b93a885bb13073afb0aa73ad82059a4c41f4b7d8eb8368980448b52d4c7dc2c", + "sha256:7d4751da932caaec419d514eaa4215eaf14b612cff66398dd51129ac22680b20", + "sha256:7f627141a26b551bdebbc4855c1157feeef18241b4b8366ed22a5c7d672ef858", + "sha256:8169cf44dd8f9071b2b9248c35fc35e8677451c52f795daa2bb4643f32a540bc", + "sha256:aa00d66c0fab27373ae44ae26a66a9e43ff2a678bf63a9c7c1a9a4d61172827a", + "sha256:ccb032fda0873254380aa2bfad2582aedc2959186cce61e3a17abc1a55ff89c3", + "sha256:d754f39e0d1603b5b24a7f8484b22d2904fa551fe865fd0d4c3332f078d20d4e", + "sha256:d75c461e20e29afc0aee7172a0950157c704ff0dd51613506bd7d82b718e7410", + "sha256:dcd65317dd15bc0451f3e01c80da2216a31916bdcffd6221ca1202d96584aa25", + "sha256:e570d3ab32e2c2861c4ebe6ffcad6a8abf9347432a37608fe1fbd157b3f0036b", + "sha256:fd43a88e045cf992ed09fa724b5315b790525f2676883a6ea64e3263bae6549d" ], - "version": "==1.13.1" + "version": "==1.13.2" }, "colorclass": { "hashes": [ @@ -106,35 +107,35 @@ }, "lxml": { "hashes": [ - "sha256:02ca7bf899da57084041bb0f6095333e4d239948ad3169443f454add9f4e9cb4", - "sha256:096b82c5e0ea27ce9138bcbb205313343ee66a6e132f25c5ed67e2c8d960a1bc", - "sha256:0a920ff98cf1aac310470c644bc23b326402d3ef667ddafecb024e1713d485f1", - "sha256:1409b14bf83a7d729f92e2a7fbfe7ec929d4883ca071b06e95c539ceedb6497c", - "sha256:17cae1730a782858a6e2758fd20dd0ef7567916c47757b694a06ffafdec20046", - "sha256:17e3950add54c882e032527795c625929613adbd2ce5162b94667334458b5a36", - "sha256:1f4f214337f6ee5825bf90a65d04d70aab05526c08191ab888cb5149501923c5", - "sha256:2e8f77db25b0a96af679e64ff9bf9dddb27d379c9900c3272f3041c4d1327c9d", - "sha256:4dffd405390a45ecb95ab5ab1c1b847553c18b0ef8ed01e10c1c8b1a76452916", - "sha256:6b899931a5648862c7b88c795eddff7588fb585e81cecce20f8d9da16eff96e0", - "sha256:726c17f3e0d7a7200718c9a890ccfeab391c9133e363a577a44717c85c71db27", - "sha256:760c12276fee05c36f95f8040180abc7fbebb9e5011447a97cdc289b5d6ab6fc", - "sha256:796685d3969815a633827c818863ee199440696b0961e200b011d79b9394bbe7", - "sha256:891fe897b49abb7db470c55664b198b1095e4943b9f82b7dcab317a19116cd38", - "sha256:9277562f175d2334744ad297568677056861070399cec56ff06abbe2564d1232", - "sha256:a471628e20f03dcdfde00770eeaf9c77811f0c331c8805219ca7b87ac17576c5", - "sha256:a63b4fd3e2cabdcc9d918ed280bdde3e8e9641e04f3c59a2a3109644a07b9832", - "sha256:ae88588d687bd476be588010cbbe551e9c2872b816f2da8f01f6f1fda74e1ef0", - "sha256:b0b84408d4eabc6de9dd1e1e0bc63e7731e890c0b378a62443e5741cfd0ae90a", - "sha256:be78485e5d5f3684e875dab60f40cddace2f5b2a8f7fede412358ab3214c3a6f", - "sha256:c27eaed872185f047bb7f7da2d21a7d8913457678c9a100a50db6da890bc28b9", - "sha256:c7fccd08b14aa437fe096c71c645c0f9be0655a9b1a4b7cffc77bcb23b3d61d2", - "sha256:c81cb40bff373ab7a7446d6bbca0190bccc5be3448b47b51d729e37799bb5692", - "sha256:d11874b3c33ee441059464711cd365b89fa1a9cf19ae75b0c189b01fbf735b84", - "sha256:e9c028b5897901361d81a4718d1db217b716424a0283afe9d6735fe0caf70f79", - "sha256:fe489d486cd00b739be826e8c1be188ddb74c7a1ca784d93d06fda882a6a1681" + "sha256:00ac0d64949fef6b3693813fe636a2d56d97a5a49b5bbb86e4cc4cc50ebc9ea2", + "sha256:0571e607558665ed42e450d7bf0e2941d542c18e117b1ebbf0ba72f287ad841c", + "sha256:0e3f04a7615fdac0be5e18b2406529521d6dbdb0167d2a690ee328bef7807487", + "sha256:13cf89be53348d1c17b453867da68704802966c433b2bb4fa1f970daadd2ef70", + "sha256:217262fcf6a4c2e1c7cb1efa08bd9ebc432502abc6c255c4abab611e8be0d14d", + "sha256:223e544828f1955daaf4cefbb4853bc416b2ec3fd56d4f4204a8b17007c21250", + "sha256:277cb61fede2f95b9c61912fefb3d43fbd5f18bf18a14fae4911b67984486f5d", + "sha256:3213f753e8ae86c396e0e066866e64c6b04618e85c723b32ecb0909885211f74", + "sha256:4690984a4dee1033da0af6df0b7a6bde83f74e1c0c870623797cec77964de34d", + "sha256:4fcc472ef87f45c429d3b923b925704aa581f875d65bac80f8ab0c3296a63f78", + "sha256:61409bd745a265a742f2693e4600e4dbd45cc1daebe1d5fad6fcb22912d44145", + "sha256:678f1963f755c5d9f5f6968dded7b245dd1ece8cf53c1aa9d80e6734a8c7f41d", + "sha256:6c6d03549d4e2734133badb9ab1c05d9f0ef4bcd31d83e5d2b4747c85cfa21da", + "sha256:6e74d5f4d6ecd6942375c52ffcd35f4318a61a02328f6f1bd79fcb4ffedf969e", + "sha256:7b4fc7b1ecc987ca7aaf3f4f0e71bbfbd81aaabf87002558f5bc95da3a865bcd", + "sha256:7ed386a40e172ddf44c061ad74881d8622f791d9af0b6f5be20023029129bc85", + "sha256:8f54f0924d12c47a382c600c880770b5ebfc96c9fd94cf6f6bdc21caf6163ea7", + "sha256:ad9b81351fdc236bda538efa6879315448411a81186c836d4b80d6ca8217cdb9", + "sha256:bbd00e21ea17f7bcc58dccd13869d68441b32899e89cf6cfa90d624a9198ce85", + "sha256:c3c289762cc09735e2a8f8a49571d0e8b4f57ea831ea11558247b5bdea0ac4db", + "sha256:cf4650942de5e5685ad308e22bcafbccfe37c54aa7c0e30cd620c2ee5c93d336", + "sha256:cfcbc33c9c59c93776aa41ab02e55c288a042211708b72fdb518221cc803abc8", + "sha256:e301055deadfedbd80cf94f2f65ff23126b232b0d1fea28f332ce58137bcdb18", + "sha256:ebbfe24df7f7b5c6c7620702496b6419f6a9aa2fd7f005eb731cc80d7b4692b9", + "sha256:eff69ddbf3ad86375c344339371168640951c302450c5d3e9936e98d6459db06", + "sha256:f6ed60a62c5f1c44e789d2cf14009423cb1646b44a43e40a9cf6a21f077678a1" ], "index": "pypi", - "version": "==4.4.1" + "version": "==4.4.2" }, "msoffcrypto-tool": { "hashes": [ @@ -156,46 +157,45 @@ }, "oletools": { "hashes": [ - "sha256:aad914cddbb84bd085607686293b3b0d97093b0b6f57785c956c94aa51885e06" + "sha256:edea57914c4040e7d0d64cfd88c84355d4305548d761d476fbac21ee26b25d8d" ], "index": "pypi", - "version": "==0.54.2" + "version": "==0.55.1" + }, + "pcodedmp": { + "hashes": [ + "sha256:025f8c809a126f45a082ffa820893e6a8d990d9d7ddb68694b5a9f0a6dbcd955", + "sha256:4441f7c0ab4cbda27bd4668db3b14f36261d86e5059ce06c0828602cbe1c4278" + ], + "version": "==1.2.6" }, "pillow": { "hashes": [ - "sha256:047d9473cf68af50ac85f8ee5d5f21a60f849bc17d348da7fc85711287a75031", - "sha256:0f66dc6c8a3cc319561a633b6aa82c44107f12594643efa37210d8c924fc1c71", - "sha256:12c9169c4e8fe0a7329e8658c7e488001f6b4c8e88740e76292c2b857af2e94c", - "sha256:248cffc168896982f125f5c13e9317c059f74fffdb4152893339f3be62a01340", - "sha256:27faf0552bf8c260a5cee21a76e031acaea68babb64daf7e8f2e2540745082aa", - "sha256:285edafad9bc60d96978ed24d77cdc0b91dace88e5da8c548ba5937c425bca8b", - "sha256:384b12c9aa8ef95558abdcb50aada56d74bc7cc131dd62d28c2d0e4d3aadd573", - "sha256:38950b3a707f6cef09cd3cbb142474357ad1a985ceb44d921bdf7b4647b3e13e", - "sha256:4aad1b88933fd6dc2846552b89ad0c74ddbba2f0884e2c162aa368374bf5abab", - "sha256:4ac6148008c169603070c092e81f88738f1a0c511e07bd2bb0f9ef542d375da9", - "sha256:4deb1d2a45861ae6f0b12ea0a786a03d19d29edcc7e05775b85ec2877cb54c5e", - "sha256:59aa2c124df72cc75ed72c8d6005c442d4685691a30c55321e00ed915ad1a291", - "sha256:5a47d2123a9ec86660fe0e8d0ebf0aa6bc6a17edc63f338b73ea20ba11713f12", - "sha256:5cc901c2ab9409b4b7ac7b5bcc3e86ac14548627062463da0af3b6b7c555a871", - "sha256:6c1db03e8dff7b9f955a0fb9907eb9ca5da75b5ce056c0c93d33100a35050281", - "sha256:7ce80c0a65a6ea90ef9c1f63c8593fcd2929448613fc8da0adf3e6bfad669d08", - "sha256:809c19241c14433c5d6135e1b6c72da4e3b56d5c865ad5736ab99af8896b8f41", - "sha256:83792cb4e0b5af480588601467c0764242b9a483caea71ef12d22a0d0d6bdce2", - "sha256:846fa202bd7ee0f6215c897a1d33238ef071b50766339186687bd9b7a6d26ac5", - "sha256:9f5529fc02009f96ba95bea48870173426879dc19eec49ca8e08cd63ecd82ddb", - "sha256:a423c2ea001c6265ed28700df056f75e26215fd28c001e93ef4380b0f05f9547", - "sha256:ac4428094b42907aba5879c7c000d01c8278d451a3b7cccd2103e21f6397ea75", - "sha256:b1ae48d87f10d1384e5beecd169c77502fcc04a2c00a4c02b85f0a94b419e5f9", - "sha256:bf4e972a88f8841d8fdc6db1a75e0f8d763e66e3754b03006cbc3854d89f1cb1", - "sha256:c6414f6aad598364aaf81068cabb077894eb88fed99c6a65e6e8217bab62ae7a", - "sha256:c710fcb7ee32f67baf25aa9ffede4795fd5d93b163ce95fdc724383e38c9df96", - "sha256:c7be4b8a09852291c3c48d3c25d1b876d2494a0a674980089ac9d5e0d78bd132", - "sha256:c9e5ffb910b14f090ac9c38599063e354887a5f6d7e6d26795e916b4514f2c1a", - "sha256:e0697b826da6c2472bb6488db4c0a7fa8af0d52fa08833ceb3681358914b14e5", - "sha256:e9a3edd5f714229d41057d56ac0f39ad9bdba6767e8c888c951869f0bdd129b0" + "sha256:0a628977ac2e01ca96aaae247ec2bd38e729631ddf2221b4b715446fd45505be", + "sha256:4d9ed9a64095e031435af120d3c910148067087541131e82b3e8db302f4c8946", + "sha256:54ebae163e8412aff0b9df1e88adab65788f5f5b58e625dc5c7f51eaf14a6837", + "sha256:5bfef0b1cdde9f33881c913af14e43db69815c7e8df429ceda4c70a5e529210f", + "sha256:5f3546ceb08089cedb9e8ff7e3f6a7042bb5b37c2a95d392fb027c3e53a2da00", + "sha256:5f7ae9126d16194f114435ebb79cc536b5682002a4fa57fa7bb2cbcde65f2f4d", + "sha256:62a889aeb0a79e50ecf5af272e9e3c164148f4bd9636cc6bcfa182a52c8b0533", + "sha256:7406f5a9b2fd966e79e6abdaf700585a4522e98d6559ce37fc52e5c955fade0a", + "sha256:8453f914f4e5a3d828281a6628cf517832abfa13ff50679a4848926dac7c0358", + "sha256:87269cc6ce1e3dee11f23fa515e4249ae678dbbe2704598a51cee76c52e19cda", + "sha256:875358310ed7abd5320f21dd97351d62de4929b0426cdb1eaa904b64ac36b435", + "sha256:8ac6ce7ff3892e5deaab7abaec763538ffd011f74dc1801d93d3c5fc541feee2", + "sha256:91b710e3353aea6fc758cdb7136d9bbdcb26b53cefe43e2cba953ac3ee1d3313", + "sha256:9d2ba4ed13af381233e2d810ff3bab84ef9f18430a9b336ab69eaf3cd24299ff", + "sha256:a62ec5e13e227399be73303ff301f2865bf68657d15ea50b038d25fc41097317", + "sha256:ab76e5580b0ed647a8d8d2d2daee170e8e9f8aad225ede314f684e297e3643c2", + "sha256:bf4003aa538af3f4205c5fac56eacaa67a6dd81e454ffd9e9f055fff9f1bc614", + "sha256:bf598d2e37cf8edb1a2f26ed3fb255191f5232badea4003c16301cb94ac5bdd0", + "sha256:c18f70dc27cc5d236f10e7834236aff60aadc71346a5bc1f4f83a4b3abee6386", + "sha256:c5ed816632204a2fc9486d784d8e0d0ae754347aba99c811458d69fcdfd2a2f9", + "sha256:dc058b7833184970d1248135b8b0ab702e6daa833be14035179f2acb78ff5636", + "sha256:ff3797f2f16bf9d17d53257612da84dd0758db33935777149b3334c01ff68865" ], "index": "pypi", - "version": "==6.2.1" + "version": "==7.0.0" }, "pycparser": { "hashes": [ @@ -205,10 +205,10 @@ }, "pyparsing": { "hashes": [ - "sha256:6f98a7b9397e206d78cc01df10131398f1c8b8510a2f4d97d9abd82e1aacdd80", - "sha256:d9338df12903bbf5d65a0e4e87c2161968b10d2e489652bb47001d82a9b028b4" + "sha256:4c830582a84fb022400b85429791bc551f1f4871c33f23e44f353119e92f969f", + "sha256:c342dccb5250c08d45fd6f8b4a559613ca603b57498511740e65cd11a2e7dcec" ], - "version": "==2.4.2" + "version": "==2.4.6" }, "python-magic": { "hashes": [ @@ -219,20 +219,13 @@ }, "six": { "hashes": [ - "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", - "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" + "sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd", + "sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66" ], - "version": "==1.12.0" + "version": "==1.13.0" } }, "develop": { - "atomicwrites": { - "hashes": [ - "sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4", - "sha256:75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6" - ], - "version": "==1.3.0" - }, "attrs": { "hashes": [ "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c", @@ -242,10 +235,10 @@ }, "certifi": { "hashes": [ - "sha256:e4f3620cfea4f83eedc95b24abd9cd56f3c4b146dd0177e83a21b4eb49e21e50", - "sha256:fd7c7c74727ddcf00e9acd26bba8da604ffec95bf1c2144e67aff7a8b50e6cef" + "sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3", + "sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f" ], - "version": "==2019.9.11" + "version": "==2019.11.28" }, "chardet": { "hashes": [ @@ -264,40 +257,39 @@ }, "coverage": { "hashes": [ - "sha256:08907593569fe59baca0bf152c43f3863201efb6113ecb38ce7e97ce339805a6", - "sha256:0be0f1ed45fc0c185cfd4ecc19a1d6532d72f86a2bac9de7e24541febad72650", - "sha256:141f08ed3c4b1847015e2cd62ec06d35e67a3ac185c26f7635f4406b90afa9c5", - "sha256:19e4df788a0581238e9390c85a7a09af39c7b539b29f25c89209e6c3e371270d", - "sha256:23cc09ed395b03424d1ae30dcc292615c1372bfba7141eb85e11e50efaa6b351", - "sha256:245388cda02af78276b479f299bbf3783ef0a6a6273037d7c60dc73b8d8d7755", - "sha256:331cb5115673a20fb131dadd22f5bcaf7677ef758741312bee4937d71a14b2ef", - "sha256:386e2e4090f0bc5df274e720105c342263423e77ee8826002dcffe0c9533dbca", - "sha256:3a794ce50daee01c74a494919d5ebdc23d58873747fa0e288318728533a3e1ca", - "sha256:60851187677b24c6085248f0a0b9b98d49cba7ecc7ec60ba6b9d2e5574ac1ee9", - "sha256:63a9a5fc43b58735f65ed63d2cf43508f462dc49857da70b8980ad78d41d52fc", - "sha256:6b62544bb68106e3f00b21c8930e83e584fdca005d4fffd29bb39fb3ffa03cb5", - "sha256:6ba744056423ef8d450cf627289166da65903885272055fb4b5e113137cfa14f", - "sha256:7494b0b0274c5072bddbfd5b4a6c6f18fbbe1ab1d22a41e99cd2d00c8f96ecfe", - "sha256:826f32b9547c8091679ff292a82aca9c7b9650f9fda3e2ca6bf2ac905b7ce888", - "sha256:93715dffbcd0678057f947f496484e906bf9509f5c1c38fc9ba3922893cda5f5", - "sha256:9a334d6c83dfeadae576b4d633a71620d40d1c379129d587faa42ee3e2a85cce", - "sha256:af7ed8a8aa6957aac47b4268631fa1df984643f07ef00acd374e456364b373f5", - "sha256:bf0a7aed7f5521c7ca67febd57db473af4762b9622254291fbcbb8cd0ba5e33e", - "sha256:bf1ef9eb901113a9805287e090452c05547578eaab1b62e4ad456fcc049a9b7e", - "sha256:c0afd27bc0e307a1ffc04ca5ec010a290e49e3afbe841c5cafc5c5a80ecd81c9", - "sha256:dd579709a87092c6dbee09d1b7cfa81831040705ffa12a1b248935274aee0437", - "sha256:df6712284b2e44a065097846488f66840445eb987eb81b3cc6e4149e7b6982e1", - "sha256:e07d9f1a23e9e93ab5c62902833bf3e4b1f65502927379148b6622686223125c", - "sha256:e2ede7c1d45e65e209d6093b762e98e8318ddeff95317d07a27a2140b80cfd24", - "sha256:e4ef9c164eb55123c62411f5936b5c2e521b12356037b6e1c2617cef45523d47", - "sha256:eca2b7343524e7ba246cab8ff00cab47a2d6d54ada3b02772e908a45675722e2", - "sha256:eee64c616adeff7db37cc37da4180a3a5b6177f5c46b187894e633f088fb5b28", - "sha256:ef824cad1f980d27f26166f86856efe11eff9912c4fed97d3804820d43fa550c", - "sha256:efc89291bd5a08855829a3c522df16d856455297cf35ae827a37edac45f466a7", - "sha256:fa964bae817babece5aa2e8c1af841bebb6d0b9add8e637548809d040443fee0", - "sha256:ff37757e068ae606659c28c3bd0d923f9d29a85de79bf25b2b34b148473b5025" + "sha256:15cf13a6896048d6d947bf7d222f36e4809ab926894beb748fc9caa14605d9c3", + "sha256:1daa3eceed220f9fdb80d5ff950dd95112cd27f70d004c7918ca6dfc6c47054c", + "sha256:1e44a022500d944d42f94df76727ba3fc0a5c0b672c358b61067abb88caee7a0", + "sha256:25dbf1110d70bab68a74b4b9d74f30e99b177cde3388e07cc7272f2168bd1477", + "sha256:3230d1003eec018ad4a472d254991e34241e0bbd513e97a29727c7c2f637bd2a", + "sha256:3dbb72eaeea5763676a1a1efd9b427a048c97c39ed92e13336e726117d0b72bf", + "sha256:5012d3b8d5a500834783689a5d2292fe06ec75dc86ee1ccdad04b6f5bf231691", + "sha256:51bc7710b13a2ae0c726f69756cf7ffd4362f4ac36546e243136187cfcc8aa73", + "sha256:527b4f316e6bf7755082a783726da20671a0cc388b786a64417780b90565b987", + "sha256:722e4557c8039aad9592c6a4213db75da08c2cd9945320220634f637251c3894", + "sha256:76e2057e8ffba5472fd28a3a010431fd9e928885ff480cb278877c6e9943cc2e", + "sha256:77afca04240c40450c331fa796b3eab6f1e15c5ecf8bf2b8bee9706cd5452fef", + "sha256:7afad9835e7a651d3551eab18cbc0fdb888f0a6136169fbef0662d9cdc9987cf", + "sha256:9bea19ac2f08672636350f203db89382121c9c2ade85d945953ef3c8cf9d2a68", + "sha256:a8b8ac7876bc3598e43e2603f772d2353d9931709345ad6c1149009fd1bc81b8", + "sha256:b0840b45187699affd4c6588286d429cd79a99d509fe3de0f209594669bb0954", + "sha256:b26aaf69713e5674efbde4d728fb7124e429c9466aeaf5f4a7e9e699b12c9fe2", + "sha256:b63dd43f455ba878e5e9f80ba4f748c0a2156dde6e0e6e690310e24d6e8caf40", + "sha256:be18f4ae5a9e46edae3f329de2191747966a34a3d93046dbdf897319923923bc", + "sha256:c312e57847db2526bc92b9bfa78266bfbaabac3fdcd751df4d062cd4c23e46dc", + "sha256:c60097190fe9dc2b329a0eb03393e2e0829156a589bd732e70794c0dd804258e", + "sha256:c62a2143e1313944bf4a5ab34fd3b4be15367a02e9478b0ce800cb510e3bbb9d", + "sha256:cc1109f54a14d940b8512ee9f1c3975c181bbb200306c6d8b87d93376538782f", + "sha256:cd60f507c125ac0ad83f05803063bed27e50fa903b9c2cfee3f8a6867ca600fc", + "sha256:d513cc3db248e566e07a0da99c230aca3556d9b09ed02f420664e2da97eac301", + "sha256:d649dc0bcace6fcdb446ae02b98798a856593b19b637c1b9af8edadf2b150bea", + "sha256:d7008a6796095a79544f4da1ee49418901961c97ca9e9d44904205ff7d6aa8cb", + "sha256:da93027835164b8223e8e5af2cf902a4c80ed93cb0909417234f4a9df3bcd9af", + "sha256:e69215621707119c6baf99bda014a45b999d37602cb7043d943c76a59b05bf52", + "sha256:ea9525e0fef2de9208250d6c5aeeee0138921057cd67fcef90fbed49c4d62d37", + "sha256:fca1669d464f0c9831fd10be2eef6b86f5ebd76c724d1e0706ebdff86bb4adf0" ], - "version": "==4.5.4" + "version": "==5.0.3" }, "filelock": { "hashes": [ @@ -315,54 +307,81 @@ }, "importlib-metadata": { "hashes": [ - "sha256:aa18d7378b00b40847790e7c27e11673d7fed219354109d0e7b9e5b25dc3ad26", - "sha256:d5f18a79777f3aa179c145737780282e27b508fc8fd688cb17c7a813e8bd39af" + "sha256:bdd9b7c397c273bcc9a11d6629a38487cd07154fa255a467bf704cd2c258e359", + "sha256:f17c015735e1a88296994c0697ecea7e11db24290941983b08c9feb30921e6d8" ], "markers": "python_version < '3.8'", - "version": "==0.23" + "version": "==1.4.0" }, "more-itertools": { "hashes": [ - "sha256:409cd48d4db7052af495b09dec721011634af3753ae1ef92d2b32f73a745f832", - "sha256:92b8c4b06dac4f0611c0729b2f2ede52b2e1bac1ab48f089c7ddc12e26bb60c4" + "sha256:1a2a32c72400d365000412fe08eb4a24ebee89997c18d3d147544f70f5403b39", + "sha256:c468adec578380b6281a114cb8a5db34eb1116277da92d7c46f904f0b52d3288" ], - "version": "==7.2.0" + "version": "==8.1.0" + }, + "mypy": { + "hashes": [ + "sha256:0a9a45157e532da06fe56adcfef8a74629566b607fa2c1ac0122d1ff995c748a", + "sha256:2c35cae79ceb20d47facfad51f952df16c2ae9f45db6cb38405a3da1cf8fc0a7", + "sha256:4b9365ade157794cef9685791032521233729cb00ce76b0ddc78749abea463d2", + "sha256:53ea810ae3f83f9c9b452582261ea859828a9ed666f2e1ca840300b69322c474", + "sha256:634aef60b4ff0f650d3e59d4374626ca6153fcaff96ec075b215b568e6ee3cb0", + "sha256:7e396ce53cacd5596ff6d191b47ab0ea18f8e0ec04e15d69728d530e86d4c217", + "sha256:7eadc91af8270455e0d73565b8964da1642fe226665dd5c9560067cd64d56749", + "sha256:7f672d02fffcbace4db2b05369142e0506cdcde20cea0e07c7c2171c4fd11dd6", + "sha256:85baab8d74ec601e86134afe2bcccd87820f79d2f8d5798c889507d1088287bf", + "sha256:87c556fb85d709dacd4b4cb6167eecc5bbb4f0a9864b69136a0d4640fdc76a36", + "sha256:a6bd44efee4dc8c3324c13785a9dc3519b3ee3a92cada42d2b57762b7053b49b", + "sha256:c6d27bd20c3ba60d5b02f20bd28e20091d6286a699174dfad515636cb09b5a72", + "sha256:e2bb577d10d09a2d8822a042a23b8d62bc3b269667c9eb8e60a6edfa000211b1", + "sha256:f97a605d7c8bc2c6d1172c2f0d5a65b24142e11a58de689046e62c2d632ca8c1" + ], + "index": "pypi", + "version": "==0.761" + }, + "mypy-extensions": { + "hashes": [ + "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d", + "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8" + ], + "version": "==0.4.3" }, "packaging": { "hashes": [ - "sha256:28b924174df7a2fa32c1953825ff29c61e2f5e082343165438812f00d3a7fc47", - "sha256:d9551545c6d761f3def1677baf08ab2a3ca17c56879e70fecba2fc4dde4ed108" + "sha256:aec3fdbb8bc9e4bb65f0634b9f551ced63983a529d6a8931817d52fdd0816ddb", + "sha256:fe1d8331dfa7cc0a883b49d75fc76380b2ab2734b220fbb87d774e4fd4b851f8" ], - "version": "==19.2" + "version": "==20.0" }, "pluggy": { "hashes": [ - "sha256:0db4b7601aae1d35b4a033282da476845aa19185c1e6964b25cf324b5e4ec3e6", - "sha256:fa5fa1622fa6dd5c030e9cad086fa19ef6a0cf6d7a2d12318e10cb49d6d68f34" + "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0", + "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d" ], - "version": "==0.13.0" + "version": "==0.13.1" }, "py": { "hashes": [ - "sha256:64f65755aee5b381cea27766a3a147c3f15b9b6b9ac88676de66ba2ae36793fa", - "sha256:dc639b046a6e2cff5bbe40194ad65936d6ba360b52b3c3fe1d08a82dd50b5e53" + "sha256:5e27081401262157467ad6e7f851b7aa402c5852dbcb3dae06768434de5752aa", + "sha256:c20fdd83a5dbc0af9efd622bee9a5564e278f6380fffcacc43ba6f43db2813b0" ], - "version": "==1.8.0" + "version": "==1.8.1" }, "pyparsing": { "hashes": [ - "sha256:6f98a7b9397e206d78cc01df10131398f1c8b8510a2f4d97d9abd82e1aacdd80", - "sha256:d9338df12903bbf5d65a0e4e87c2161968b10d2e489652bb47001d82a9b028b4" + "sha256:4c830582a84fb022400b85429791bc551f1f4871c33f23e44f353119e92f969f", + "sha256:c342dccb5250c08d45fd6f8b4a559613ca603b57498511740e65cd11a2e7dcec" ], - "version": "==2.4.2" + "version": "==2.4.6" }, "pytest": { "hashes": [ - "sha256:27abc3fef618a01bebb1f0d6d303d2816a99aa87a5968ebc32fe971be91eb1e6", - "sha256:58cee9e09242937e136dbb3dab466116ba20d6b7828c7620f23947f37eb4dae4" + "sha256:6b571215b5a790f9b41f19f3531c53a45cf6bb8ef2988bc1ff9afb38270b25fa", + "sha256:e41d489ff43948babd0fad7ad5e49b8735d5d55e26628a58673c39ff61d95de4" ], "index": "pypi", - "version": "==5.2.2" + "version": "==5.3.2" }, "pytest-cov": { "hashes": [ @@ -381,22 +400,20 @@ }, "pyyaml": { "hashes": [ - "sha256:0113bc0ec2ad727182326b61326afa3d1d8280ae1122493553fd6f4397f33df9", - "sha256:01adf0b6c6f61bd11af6e10ca52b7d4057dd0be0343eb9283c878cf3af56aee4", - "sha256:5124373960b0b3f4aa7df1707e63e9f109b5263eca5976c66e08b1c552d4eaf8", - "sha256:5ca4f10adbddae56d824b2c09668e91219bb178a1eee1faa56af6f99f11bf696", - "sha256:7907be34ffa3c5a32b60b95f4d95ea25361c951383a894fec31be7252b2b6f34", - "sha256:7ec9b2a4ed5cad025c2278a1e6a19c011c80a3caaac804fd2d329e9cc2c287c9", - "sha256:87ae4c829bb25b9fe99cf71fbb2140c448f534e24c998cc60f39ae4f94396a73", - "sha256:9de9919becc9cc2ff03637872a440195ac4241c80536632fffeb6a1e25a74299", - "sha256:a5a85b10e450c66b49f98846937e8cfca1db3127a9d5d1e31ca45c3d0bef4c5b", - "sha256:b0997827b4f6a7c286c01c5f60384d218dca4ed7d9efa945c3e1aa623d5709ae", - "sha256:b631ef96d3222e62861443cc89d6563ba3eeb816eeb96b2629345ab795e53681", - "sha256:bf47c0607522fdbca6c9e817a6e81b08491de50f3766a7a0e6a5be7905961b41", - "sha256:f81025eddd0327c7d4cfe9b62cf33190e1e736cc6e97502b3ec425f574b3e7a8" + "sha256:059b2ee3194d718896c0ad077dd8c043e5e909d9180f387ce42012662a4946d6", + "sha256:1cf708e2ac57f3aabc87405f04b86354f66799c8e62c28c5fc5f88b5521b2dbf", + "sha256:24521fa2890642614558b492b473bee0ac1f8057a7263156b02e8b14c88ce6f5", + "sha256:4fee71aa5bc6ed9d5f116327c04273e25ae31a3020386916905767ec4fc5317e", + "sha256:70024e02197337533eef7b85b068212420f950319cc8c580261963aefc75f811", + "sha256:74782fbd4d4f87ff04159e986886931456a1894c61229be9eaf4de6f6e44b99e", + "sha256:940532b111b1952befd7db542c370887a8611660d2b9becff75d39355303d82d", + "sha256:cb1f2f5e426dc9f07a7681419fe39cee823bb74f723f36f70399123f439e9b20", + "sha256:dbbb2379c19ed6042e8f11f2a2c66d39cceb8aeace421bfc29d085d93eda3689", + "sha256:e3a057b7a64f1222b56e47bcff5e4b94c4f61faac04c7c4ecb1985e18caa3994", + "sha256:e9f45bd5b92c7974e59bcd2dcc8631a6b6cc380a904725fce7bc08872e691615" ], "index": "pypi", - "version": "==5.1.2" + "version": "==5.3" }, "requests": { "hashes": [ @@ -407,10 +424,10 @@ }, "six": { "hashes": [ - "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", - "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" + "sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd", + "sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66" ], - "version": "==1.12.0" + "version": "==1.13.0" }, "toml": { "hashes": [ @@ -421,32 +438,65 @@ }, "tox": { "hashes": [ - "sha256:0bc216b6a2e6afe764476b4a07edf2c1dab99ed82bb146a1130b2e828f5bff5e", - "sha256:c4f6b319c20ba4913dbfe71ebfd14ff95d1853c4231493608182f66e566ecfe1" + "sha256:06ba73b149bf838d5cd25dc30c2dd2671ae5b2757cf98e5c41a35fe449f131b3", + "sha256:806d0a9217584558cc93747a945a9d9bff10b141a5287f0c8429a08828a22192" ], "index": "pypi", - "version": "==3.14.0" + "version": "==3.14.3" + }, + "typed-ast": { + "hashes": [ + "sha256:1170afa46a3799e18b4c977777ce137bb53c7485379d9706af8a59f2ea1aa161", + "sha256:18511a0b3e7922276346bcb47e2ef9f38fb90fd31cb9223eed42c85d1312344e", + "sha256:262c247a82d005e43b5b7f69aff746370538e176131c32dda9cb0f324d27141e", + "sha256:2b907eb046d049bcd9892e3076c7a6456c93a25bebfe554e931620c90e6a25b0", + "sha256:354c16e5babd09f5cb0ee000d54cfa38401d8b8891eefa878ac772f827181a3c", + "sha256:48e5b1e71f25cfdef98b013263a88d7145879fbb2d5185f2a0c79fa7ebbeae47", + "sha256:4e0b70c6fc4d010f8107726af5fd37921b666f5b31d9331f0bd24ad9a088e631", + "sha256:630968c5cdee51a11c05a30453f8cd65e0cc1d2ad0d9192819df9978984529f4", + "sha256:66480f95b8167c9c5c5c87f32cf437d585937970f3fc24386f313a4c97b44e34", + "sha256:71211d26ffd12d63a83e079ff258ac9d56a1376a25bc80b1cdcdf601b855b90b", + "sha256:7954560051331d003b4e2b3eb822d9dd2e376fa4f6d98fee32f452f52dd6ebb2", + "sha256:838997f4310012cf2e1ad3803bce2f3402e9ffb71ded61b5ee22617b3a7f6b6e", + "sha256:95bd11af7eafc16e829af2d3df510cecfd4387f6453355188342c3e79a2ec87a", + "sha256:bc6c7d3fa1325a0c6613512a093bc2a2a15aeec350451cbdf9e1d4bffe3e3233", + "sha256:cc34a6f5b426748a507dd5d1de4c1978f2eb5626d51326e43280941206c209e1", + "sha256:d755f03c1e4a51e9b24d899561fec4ccaf51f210d52abdf8c07ee2849b212a36", + "sha256:d7c45933b1bdfaf9f36c579671fec15d25b06c8398f113dab64c18ed1adda01d", + "sha256:d896919306dd0aa22d0132f62a1b78d11aaf4c9fc5b3410d3c666b818191630a", + "sha256:fdc1c9bbf79510b76408840e009ed65958feba92a88833cdceecff93ae8fff66", + "sha256:ffde2fbfad571af120fcbfbbc61c72469e72f550d676c3342492a9dfdefb8f12" + ], + "version": "==1.4.0" + }, + "typing-extensions": { + "hashes": [ + "sha256:091ecc894d5e908ac75209f10d5b4f118fbdb2eb1ede6a63544054bb1edb41f2", + "sha256:910f4656f54de5993ad9304959ce9bb903f90aadc7c67a0bef07e678014e892d", + "sha256:cf8b63fedea4d89bab840ecbb93e75578af28f76f66c35889bd7065f5af88575" + ], + "version": "==3.7.4.1" }, "urllib3": { "hashes": [ - "sha256:3de946ffbed6e6746608990594d08faac602528ac7015ac28d33cee6a45b7398", - "sha256:9a107b99a5393caf59c7aa3c1249c16e6879447533d0887f4336dde834c7be86" + "sha256:a8a318824cc77d1fd4b2bec2ded92646630d7fe8619497b142c84a9e6f5a7293", + "sha256:f3c5fd51747d450d4dcf6f923c81f78f811aab8205fda64b0aba34a4e48b0745" ], - "version": "==1.25.6" + "version": "==1.25.7" }, "virtualenv": { "hashes": [ - "sha256:11cb4608930d5fd3afb545ecf8db83fa50e1f96fc4fca80c94b07d2c83146589", - "sha256:d257bb3773e48cac60e475a19b608996c73f4d333b3ba2e4e57d5ac6134e0136" + "sha256:0d62c70883c0342d59c11d0ddac0d954d0431321a41ab20851facf2b222598f3", + "sha256:55059a7a676e4e19498f1aad09b8313a38fcc0cdbe4fdddc0e9b06946d21b4bb" ], - "version": "==16.7.7" + "version": "==16.7.9" }, "wcwidth": { "hashes": [ - "sha256:3df37372226d6e63e1b1e1eda15c594bca98a22d33a23832a90998faa96bc65e", - "sha256:f4ebe71925af7b40a864553f761ed559b43544f8f71746c2d756c7fe788ade7c" + "sha256:8fd29383f539be45b20bd4df0dc29c20ba48654a41e661925e612311e9f3c603", + "sha256:f28b3e8a6483e5d49e7f8949ac1a78314e740333ae305b4ba5defd3e74fb37a8" ], - "version": "==0.1.7" + "version": "==0.1.8" }, "zipp": { "hashes": [ diff --git a/filecheck/filecheck.py b/filecheck/filecheck.py index 0fb0b72..a2b1425 100644 --- a/filecheck/filecheck.py +++ b/filecheck/filecheck.py @@ -10,14 +10,16 @@ import random import shutil import time import hashlib +from pathlib import Path +from typing import Dict, List, Tuple, Callable, Optional -import oletools.oleid -import olefile -import officedissector +import oletools.oleid # type: ignore +import olefile # type: ignore +import officedissector # type: ignore import warnings -import exifread -from PIL import Image -from pdfid import PDFiD, cPDFiD +import exifread # type: ignore +from PIL import Image # type: ignore +from pdfid import PDFiD, cPDFiD # type: ignore from kittengroomer import FileBase, KittenGroomerBase, Logging @@ -26,27 +28,27 @@ class Config: """Configuration information for filecheck.py.""" # MIMES # Application subtypes (mimetype: 'application/') - mimes_ooxml = ('vnd.openxmlformats-officedocument.',) - mimes_office = ('msword', 'vnd.ms-',) - mimes_libreoffice = ('vnd.oasis.opendocument',) - mimes_rtf = ('rtf', 'richtext',) - mimes_pdf = ('pdf', 'postscript',) - mimes_xml = ('xml',) - mimes_ms = ('dosexec',) - mimes_compressed = ('zip', 'rar', 'x-rar', 'bzip2', 'lzip', 'lzma', 'lzop', - 'xz', 'compress', 'gzip', 'tar',) - mimes_data = ('octet-stream',) - mimes_audio = ('ogg',) + mimes_ooxml: Tuple[str, ...] = ('vnd.openxmlformats-officedocument.',) + mimes_office: Tuple[str, ...] = ('msword', 'vnd.ms-',) + mimes_libreoffice: Tuple[str, ...] = ('vnd.oasis.opendocument',) + mimes_rtf: Tuple[str, ...] = ('rtf', 'richtext',) + mimes_pdf: Tuple[str, ...] = ('pdf', 'postscript',) + mimes_xml: Tuple[str, ...] = ('xml',) + mimes_ms: Tuple[str, ...] = ('dosexec',) + mimes_compressed: Tuple[str, ...] = ('zip', 'rar', 'x-rar', 'bzip2', 'lzip', 'lzma', 'lzop', + 'xz', 'compress', 'gzip', 'tar',) + mimes_data: Tuple[str, ...] = ('octet-stream',) + mimes_audio: Tuple[str, ...] = ('ogg',) # Image subtypes - mimes_exif = ('image/jpeg', 'image/tiff',) - mimes_png = ('image/png',) + mimes_exif: Tuple[str, ...] = ('image/jpeg', 'image/tiff',) + mimes_png: Tuple[str, ...] = ('image/png',) # Mimetypes with metadata - mimes_metadata = ('image/jpeg', 'image/tiff', 'image/png',) + mimes_metadata: Tuple[str, ...] = ('image/jpeg', 'image/tiff', 'image/png',) # Mimetype aliases - aliases = { + aliases: Dict[str, str] = { # Win executables 'application/x-msdos-program': 'application/x-dosexec', 'application/x-dosexec': 'application/x-msdos-program', @@ -61,7 +63,7 @@ class Config: # Commonly used malicious extensions # Sources: http://www.howtogeek.com/137270/50-file-extensions-that-are-potentially-dangerous-on-windows/ # https://github.com/wiregit/wirecode/blob/master/components/core-settings/src/main/java/org/limewire/core/settings/FilterSettings.java - malicious_exts = ( + malicious_exts: Tuple[str, ...] = ( # Applications ".exe", ".pif", ".application", ".gadget", ".msi", ".msp", ".com", ".scr", ".hta", ".cpl", ".msc", ".jar", @@ -116,7 +118,7 @@ class Config: # In [12]: mimetypes.guess_type('toot.tar.gz', strict=False) # Out[12]: ('application/x-tar', 'gzip') # It works as expected if you do mimetypes.guess_type('application/gzip', strict=False) - override_ext = {'.gz': 'application/gzip'} + override_ext: Dict[str, str] = {'.gz': 'application/gzip'} SEVENZ_PATH = '/usr/bin/7z' @@ -130,12 +132,12 @@ class File(FileBase): filetype-specific processing methods. """ - def __init__(self, src_path, dst_path): + def __init__(self, src_path: Path, dst_path: Path): super(File, self).__init__(src_path, dst_path) - self.is_archive = False - self.tempdir_path = self.dst_path + '_temp' + self.is_archive: bool = False + self.tempdir_path: Path = Path(str(self.dst_path) + '_temp') - subtypes_apps = ( + subtypes_apps: Tuple[Tuple[Tuple[str, ...], Callable], ...] = ( (Config.mimes_office, self._winoffice), (Config.mimes_ooxml, self._ooxml), (Config.mimes_rtf, self.text), @@ -147,15 +149,15 @@ class File(FileBase): (Config.mimes_data, self._binary_app), (Config.mimes_audio, self.audio) ) - self.app_subtype_methods = self._make_method_dict(subtypes_apps) + self.app_subtype_methods: Dict[str, Callable] = self._make_method_dict(subtypes_apps) - types_metadata = ( + types_metadata: Tuple[Tuple[Tuple[str, ...], Callable], ...] = ( (Config.mimes_exif, self._metadata_exif), (Config.mimes_png, self._metadata_png), ) - self.metadata_mimetype_methods = self._make_method_dict(types_metadata) + self.metadata_mimetype_methods: Dict[str, Callable] = self._make_method_dict(types_metadata) - self.mime_processing_options = { + self.mime_processing_options: Dict[str, Callable] = { 'text': self.text, 'audio': self.audio, 'image': self.image, @@ -199,7 +201,7 @@ class File(FileBase): is_known_extension = self.extension in mimetypes.types_map.keys() if is_known_extension and self.mimetype not in expected_mimetypes and not is_empty_file: - self.make_dangerous('Mimetype does not match expected mimetypes ({}) for this extension'.format(expected_mimetypes)) + self.make_dangerous('Mimetype does not match expected mimetypes ({expected_mimetypes}) for this extension') def _check_mimetype(self): """ @@ -273,7 +275,7 @@ class File(FileBase): self.random_hashes.append((start_pos, hashed)) time.sleep(random.uniform(0.1, 0.5)) # Add a random sleep length - def _validate_random_hashes(self): + def _validate_random_hashes(self) -> bool: """Validate hashes computed by _compute_random_hashes""" if not os.path.exists(self.src_path) or os.path.isdir(self.src_path) or self.maintype == 'image': # Images are converted, we don't have to fear TOCTOU @@ -306,7 +308,7 @@ class File(FileBase): self.mime_processing_options.get(self.maintype, self.unknown)() # ##### Helper functions ##### - def _make_method_dict(self, list_of_tuples): + def _make_method_dict(self, list_of_tuples: Tuple) -> Dict[str, Callable]: """Returns a dictionary with mimetype: method pairs.""" dict_to_return = {} for list_of_subtypes, method in list_of_tuples: @@ -315,16 +317,16 @@ class File(FileBase): return dict_to_return @property - def has_metadata(self): + def has_metadata(self) -> bool: """True if filetype typically contains metadata, else False.""" if self.mimetype in Config.mimes_metadata: return True return False - def make_tempdir(self): + def make_tempdir(self) -> Path: """Make a temporary directory at self.tempdir_path.""" - if not os.path.exists(self.tempdir_path): - os.makedirs(self.tempdir_path) + if not self.tempdir_path.exists(): + self.tempdir_path.mkdir() return self.tempdir_path ####################### @@ -498,7 +500,7 @@ class File(FileBase): ####################### # Metadata extractors - def _metadata_exif(self, metadata_file_path): + def _metadata_exif(self, metadata_file_path) -> bool: """Read exif metadata from a jpg or tiff file using exifread.""" # TODO: can we shorten this method somehow? with open(self.src_path, 'rb') as img: @@ -527,7 +529,7 @@ class File(FileBase): self.set_property('metadata', 'exif') return True - def _metadata_png(self, metadata_file_path): + def _metadata_png(self, metadata_file_path) -> bool: """Extract metadata from a png file using PIL/Pillow.""" warnings.simplefilter('error', Image.DecompressionBombWarning) try: @@ -539,6 +541,7 @@ class File(FileBase): metadata_file.write("Key: {}\tValue: {}\n".format(tag, img.info[tag])) # LOG: handle metadata self.set_property('metadata', 'png') + return True except Exception as e: # Catch decompression bombs # TODO: only catch DecompressionBombWarnings here? self.add_error(e, "Caught exception processing metadata for {}".format(self.src_path)) @@ -582,7 +585,7 @@ class File(FileBase): if self.has_metadata: self.extract_metadata() tempdir_path = self.make_tempdir() - tempfile_path = os.path.join(tempdir_path, self.filename) + tempfile_path = tempdir_path / self.filename warnings.simplefilter('error', Image.DecompressionBombWarning) try: # Do image conversions with Image.open(self.src_path) as img_in: @@ -600,37 +603,37 @@ class File(FileBase): class GroomerLogger(object): """Groomer logging interface.""" - def __init__(self, src_root_path, dst_root_path, debug=False): - self._src_root_path = src_root_path - self._dst_root_path = dst_root_path - self._log_dir_path = self._make_log_dir(dst_root_path) - self.log_path = os.path.join(self._log_dir_path, 'circlean_log.txt') + def __init__(self, src_root_path: Path, dst_root_path: Path, debug: bool=False): + self._src_root_path: Path = src_root_path + self._dst_root_path: Path = dst_root_path + self._log_dir_path: Path = self._make_log_dir(dst_root_path) + self.log_path: Path = self._log_dir_path / 'circlean_log.txt' self._add_root_dir(src_root_path) if debug: - self.log_debug_err = os.path.join(self._log_dir_path, 'debug_stderr.log') - self.log_debug_out = os.path.join(self._log_dir_path, 'debug_stdout.log') + self.log_debug_err: Path = self._log_dir_path / 'debug_stderr.log' + self.log_debug_out: Path = self._log_dir_path / 'debug_stdout.log' else: - self.log_debug_err = os.devnull - self.log_debug_out = os.devnull + self.log_debug_err = Path(os.devnull) + self.log_debug_out = Path(os.devnull) - def _make_log_dir(self, root_dir_path): + def _make_log_dir(self, root_dir_path: Path) -> Path: """Create the directory in the dest dir that will hold the logs""" - log_dir_path = os.path.join(root_dir_path, 'logs') + log_dir_path = root_dir_path / 'logs' if os.path.exists(log_dir_path): shutil.rmtree(log_dir_path) os.makedirs(log_dir_path) return log_dir_path - def _add_root_dir(self, root_path): + def _add_root_dir(self, root_path: Path): """Add the root directory to the log""" dirname = os.path.split(root_path)[1] + '/' with open(self.log_path, mode='ab') as lf: lf.write(bytes(dirname, 'utf-8')) lf.write(b'\n') - def add_file(self, file_path, file_props, in_tempdir=False): + def add_file(self, file_path: Path, file_props: dict, in_tempdir: bool=False): """Add a file to the log. Takes a path and a dict of file properties.""" - depth = self._get_path_depth(file_path) + depth = self._get_path_depth(str(file_path)) try: file_hash = Logging.computehash(file_path)[:6] except IsADirectoryError: @@ -671,34 +674,34 @@ class GroomerLogger(object): depth -= 1 self._write_line_to_log(log_string, depth) - def add_dir(self, dir_path): + def add_dir(self, dir_path: Path): """Add a directory to the log""" - path_depth = self._get_path_depth(dir_path) - dirname = os.path.split(dir_path)[1] + '/' + path_depth = self._get_path_depth(str(dir_path)) + dirname = os.path.split(str(dir_path))[1] + '/' log_line = '+- ' + dirname self._write_line_to_log(log_line, path_depth) - def _format_file_size(self, size): + def _format_file_size(self, size: int) -> str: """Returns a string with the file size and appropriate unit""" file_size = size for unit in ('B', 'KB', 'MB', 'GB'): if file_size < 1024: return str(int(file_size)) + unit else: - file_size = file_size / 1024 + file_size = int(file_size / 1024) return str(int(file_size)) + 'GB' - def _get_path_depth(self, path): + def _get_path_depth(self, path: str) -> int: """Returns the relative path depth compared to root directory""" - if self._dst_root_path in path: - base_path = self._dst_root_path - elif self._src_root_path in path: - base_path = self._src_root_path + if str(self._dst_root_path) in path: + base_path = str(self._dst_root_path) + elif str(self._src_root_path) in path: + base_path = str(self._src_root_path) relpath = os.path.relpath(path, base_path) path_depth = relpath.count(os.path.sep) return path_depth - def _write_line_to_log(self, line, indentation_depth): + def _write_line_to_log(self, line: str, indentation_depth: int): """ Write a line to the log @@ -715,28 +718,28 @@ class GroomerLogger(object): class KittenGroomerFileCheck(KittenGroomerBase): - def __init__(self, root_src, root_dst, max_recursive_depth=2, debug=False): + def __init__(self, root_src: str, root_dst: str, max_recursive_depth: int=2, debug: bool=False): super(KittenGroomerFileCheck, self).__init__(root_src, root_dst) self.recursive_archive_depth = 0 self.max_recursive_depth = max_recursive_depth - self.logger = GroomerLogger(root_src, root_dst, debug) + self.logger = GroomerLogger(self.src_root_path, self.dst_root_path, debug) def __repr__(self): return "filecheck.KittenGroomerFileCheck object: {{{}}}".format( os.path.basename(self.src_root_path) ) - def process_dir(self, src_dir, dst_dir): + def process_dir(self, src_dir: Path, dst_dir: Path): """Process a directory on the source key.""" for srcpath in self.list_files_dirs(src_dir): - if not os.path.islink(srcpath) and os.path.isdir(srcpath): + if not srcpath.is_symlink() and srcpath.is_dir(): self.logger.add_dir(srcpath) else: - dstpath = os.path.join(dst_dir, os.path.basename(srcpath)) + dstpath = dst_dir / srcpath.name cur_file = File(srcpath, dstpath) self.process_file(cur_file) - def process_file(self, file): + def process_file(self, file: File): """ Process an individual file. @@ -753,13 +756,13 @@ class KittenGroomerFileCheck(KittenGroomerBase): if not file._validate_random_hashes(): # Something's fucked up. file.make_dangerous('The copied file is different from the one checked, removing.') - os.remove(file.dst_path) + file.dst_path.unlink() self.write_file_to_log(file) # TODO: Can probably handle cleaning up the tempdir better if hasattr(file, 'tempdir_path'): self.safe_rmtree(file.tempdir_path) - def process_archive(self, file): + def process_archive(self, file: File): """ Unpack an archive using 7zip and process contents using process_dir. @@ -781,25 +784,25 @@ class KittenGroomerFileCheck(KittenGroomerBase): self.safe_rmtree(tempdir_path) self.recursive_archive_depth -= 1 - def _run_process(self, command_string, timeout=None): + def _run_process(self, command_string: str, timeout: Optional[int]=None) -> bool: """Run command_string in a subprocess, wait until it finishes.""" args = shlex.split(command_string) with open(self.logger.log_debug_err, 'ab') as stderr, open(self.logger.log_debug_out, 'ab') as stdout: try: subprocess.check_call(args, stdout=stdout, stderr=stderr, timeout=timeout) except (subprocess.TimeoutExpired, subprocess.CalledProcessError): - return + return False return True - def write_file_to_log(self, file): + def write_file_to_log(self, file: File): """Pass information about `file` to self.logger.""" props = file.get_all_props() if not file.is_archive: # FIXME: in_tempdir is a hack to make image files appear at the correct tree depth in log - in_tempdir = os.path.exists(file.tempdir_path) + in_tempdir = file.tempdir_path.exists() self.logger.add_file(file.src_path, props, in_tempdir) - def list_files_dirs(self, root_dir_path): + def list_files_dirs(self, root_dir_path: Path) -> List[Path]: """ Returns a list of all files and directories @@ -807,14 +810,14 @@ class KittenGroomerFileCheck(KittenGroomerBase): """ queue = [] for path in sorted(os.listdir(root_dir_path), key=lambda x: str.lower(x)): - full_path = os.path.join(root_dir_path, path) + full_path = root_dir_path / path # check for symlinks first to prevent getting trapped in infinite symlink recursion - if os.path.islink(full_path): + if full_path.is_symlink(): queue.append(full_path) - elif os.path.isdir(full_path): + elif full_path.is_dir(): queue.append(full_path) queue += self.list_files_dirs(full_path) - elif os.path.isfile(full_path): + elif full_path.is_file(): queue.append(full_path) return queue @@ -822,7 +825,7 @@ class KittenGroomerFileCheck(KittenGroomerBase): self.process_dir(self.src_root_path, self.dst_root_path) -def main(kg_implementation, description): +def main(kg_implementation, description: str): parser = argparse.ArgumentParser(prog='KittenGroomer', description=description) parser.add_argument('-s', '--source', type=str, help='Source directory') parser.add_argument('-d', '--destination', type=str, help='Destination directory') diff --git a/kittengroomer/helpers.py b/kittengroomer/helpers.py index 847d3bc..7d2e5b0 100644 --- a/kittengroomer/helpers.py +++ b/kittengroomer/helpers.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ @@ -13,8 +13,10 @@ import hashlib import shutil import argparse import stat +from pathlib import Path +from typing import Union, Optional, List, Dict, Any, Tuple, Iterator -import magic +import magic # type: ignore class FileBase(object): @@ -24,49 +26,49 @@ class FileBase(object): Contains file attributes and various helper methods. """ - def __init__(self, src_path, dst_path): + def __init__(self, src_path: Path, dst_path: Path): """ Initialized with the source path and expected destination path. Create various properties and determine the file's mimetype. """ - self.src_path = src_path - self.dst_dir = os.path.dirname(dst_path) - self.filename = os.path.basename(src_path) - self.size = self._get_size(src_path) - self.is_dangerous = False - self.copied = False + self.src_path: Path = src_path + self.dst_dir: Path = dst_path.parent + self.filename: str = src_path.name + self.size: int = self._get_size(src_path) + self.is_dangerous: bool = False + self.copied: bool = False self.symlink_path = None - self._description_string = [] # array of descriptions to be joined - self._errors = {} - self._user_defined = {} - self.should_copy = True - self.mimetype = self._determine_mimetype(src_path) + self._description_string: List[str] = [] # array of descriptions to be joined + self._errors: Dict[Exception, str] = {} + self._user_defined: Dict[str, str] = {} + self.should_copy: bool = True + self.mimetype = self._determine_mimetype(str(src_path)) @property - def dst_path(self): - return os.path.join(self.dst_dir, self.filename) + def dst_path(self) -> Path: + return self.dst_dir / self.filename @property - def extension(self): - _, ext = os.path.splitext(self.filename) + def extension(self) -> Union[None, str]: + ext = self.src_path.suffix if ext == '': return None else: return ext.lower() @property - def maintype(self): + def maintype(self) -> Optional[str]: main, _ = self._split_mimetype(self.mimetype) return main @property - def subtype(self): + def subtype(self) -> Optional[str]: _, sub = self._split_mimetype(self.mimetype) return sub @property - def has_mimetype(self): + def has_mimetype(self) -> bool: """True if file has a main and sub mimetype, else False.""" if not self.maintype or not self.subtype: return False @@ -74,7 +76,7 @@ class FileBase(object): return True @property - def has_extension(self): + def has_extension(self) -> bool: """True if self.extension is set, else False.""" if self.extension is None: return False @@ -82,7 +84,7 @@ class FileBase(object): return True @property - def is_symlink(self): + def is_symlink(self) -> bool: """True if file is a symlink, else False.""" if self.symlink_path is None: return False @@ -90,27 +92,23 @@ class FileBase(object): return True @property - def description_string(self): + def description_string(self) -> str: if len(self._description_string) == 0: return 'No description' elif len(self._description_string) == 1: return self._description_string[0] else: ret_string = ', '.join(self._description_string) - return ret_string.strip(', ') + return ret_string.strip(', ') # NOTE: why strip? @description_string.setter - def description_string(self, value): - if hasattr(self, 'description_string'): - if isinstance(value, str): - if value not in self._description_string: - self._description_string.append(value) - else: - raise TypeError("Description_string can only include strings") - else: - self._description_string = value + def description_string(self, value: str): + if not isinstance(value, str): + raise TypeError(f"value ({value}) must be a 'str' and not a {type(value)}") + if value not in self._description_string: + self._description_string.append(value) - def set_property(self, prop_string, value): + def set_property(self, prop_string: str, value: Any): """ Take a property and a value and add them to the file's stored props. @@ -123,7 +121,7 @@ class FileBase(object): else: self._user_defined[prop_string] = value - def get_property(self, prop_string): + def get_property(self, prop_string: str) -> Any: """ Get the value for a property stored on the file. @@ -134,7 +132,7 @@ class FileBase(object): except AttributeError: return self._user_defined.get(prop_string, None) - def get_all_props(self): + def get_all_props(self) -> dict: """Return a dict containing all stored properties of this file.""" # Maybe move this onto the logger? I think that makes more sense props_dict = { @@ -155,11 +153,11 @@ class FileBase(object): } return props_dict - def add_error(self, error, info_string): + def add_error(self, error: Exception, info_string: str): """Add an `error`: `info_string` pair to the file.""" self._errors.update({error: info_string}) - def add_description(self, description_string): + def add_description(self, description_string: str): """ Add a description string to the file. @@ -167,7 +165,7 @@ class FileBase(object): """ self.set_property('description_string', description_string) - def make_dangerous(self, reason_string=None): + def make_dangerous(self, reason_string: Optional[str]=None): """ Mark file as dangerous. @@ -180,37 +178,33 @@ class FileBase(object): if reason_string: self.add_description(reason_string) - def safe_copy(self, src=None, dst=None): + def safe_copy(self): """ Copy file and create destination directories if needed. Sets all exec bits to '0'. """ - if src is None: - src = self.src_path - if dst is None: - dst = self.dst_path + src = self.src_path + dst = self.dst_path try: - os.makedirs(self.dst_dir, exist_ok=True) - shutil.copy(src, dst) + self.dst_dir.mkdir(exist_ok=True) + shutil.copy(str(src), str(dst)) current_perms = self._get_file_permissions(dst) only_exec_bits = 0o0111 perms_no_exec = current_perms & (~only_exec_bits) - os.chmod(dst, perms_no_exec) + dst.chmod(perms_no_exec) except IOError as e: # Probably means we can't write in the dest dir self.add_error(e, '') - def force_ext(self, extension): + def force_ext(self, extension: str): """If dst_path does not end in `extension`, append .ext to it.""" new_ext = self._check_leading_dot(extension) if not self.filename.endswith(new_ext): # TODO: log that the extension was changed self.filename += new_ext - if not self.get_property('extension') == new_ext: - self.set_property('extension', new_ext) - def create_metadata_file(self, extension): + def create_metadata_file(self, extension) -> Union[Path, bool]: # TODO: this method name is confusing """ Create a separate file to hold extracted metadata. @@ -220,29 +214,26 @@ class FileBase(object): ext = self._check_leading_dot(extension) try: # Prevent using the same path as another file from src_path - if os.path.exists(self.src_path + ext): - raise KittenGroomerError( - "Could not create metadata file for \"" + - self.filename + - "\": a file with that path exists.") + if Path(f'{self.src_path}{ext}').exists(): + raise KittenGroomerError(f'Could not create metadata file for "{self.filename}": a file with that path exists.') else: - os.makedirs(self.dst_dir, exist_ok=True) + self.dst_dir.mkdir(exist_ok=True) # TODO: shouldn't mutate state and also return something - self.metadata_file_path = self.dst_path + ext + self.metadata_file_path = Path(f'{self.dst_path}{ext}') return self.metadata_file_path # TODO: can probably let this exception bubble up except KittenGroomerError as e: self.add_error(e, '') return False - def _check_leading_dot(self, ext): + def _check_leading_dot(self, ext: str) -> str: # TODO: this method name is confusing if len(ext) > 0: if not ext.startswith('.'): return '.' + ext return ext - def _determine_mimetype(self, file_path): + def _determine_mimetype(self, file_path: str) -> str: if os.path.islink(file_path): # libmagic will throw an IOError on a broken symlink mimetype = 'inode/symlink' @@ -256,19 +247,18 @@ class FileBase(object): mt = None try: mimetype = mt.decode("utf-8") - except: + except Exception: # FIXME: what should the exception be if mimetype isn't utf-8? mimetype = mt return mimetype - def _split_mimetype(self, mimetype): + def _split_mimetype(self, mimetype: str) -> Tuple[Union[str, None], Union[str, None]]: + main_type, sub_type = None, None if mimetype and '/' in mimetype: main_type, sub_type = mimetype.split('/') - else: - main_type, sub_type = None, None return main_type, sub_type - def _get_size(self, file_path): + def _get_size(self, file_path: Path) -> int: """Filesize in bytes as an int, 0 if file does not exist.""" try: size = os.path.getsize(file_path) @@ -276,23 +266,23 @@ class FileBase(object): size = 0 return size - def _remove_exec_bit(self, file_path): + def _remove_exec_bit(self, file_path: Path): current_perms = self._get_file_permissions(file_path) perms_no_exec = current_perms & (~stat.S_IEXEC) os.chmod(file_path, perms_no_exec) - def _get_file_permissions(self, file_path): - full_mode = os.stat(file_path, follow_symlinks=False).st_mode + def _get_file_permissions(self, file_path: Path): + full_mode = file_path.lstat().st_mode return stat.S_IMODE(full_mode) class Logging(object): @staticmethod - def computehash(path): + def computehash(path: Path) -> str: """Return the sha256 hash of a file at a given path.""" s = hashlib.sha256() - with open(path, 'rb') as f: + with path.open('rb') as f: while True: buf = f.read(0x100000) if not buf: @@ -304,36 +294,35 @@ class Logging(object): class KittenGroomerBase(object): """Base object responsible for copy/sanitization process.""" - def __init__(self, src_root_path, dst_root_path): + def __init__(self, src_root_path: str, dst_root_path: str): """Initialized with path to source and dest directories.""" - self.src_root_path = os.path.abspath(src_root_path) - self.dst_root_path = os.path.abspath(dst_root_path) + self.src_root_path: Path = Path(os.path.abspath(src_root_path)) + self.dst_root_path: Path = Path(os.path.abspath(dst_root_path)) - def safe_rmtree(self, directory_path): + def safe_rmtree(self, directory_path: Path): """Remove a directory tree if it exists.""" - if os.path.exists(directory_path): + if directory_path.is_dir(): shutil.rmtree(directory_path) - def safe_remove(self, file_path): + def safe_remove(self, file_path: Path): """Remove file at file_path if it exists.""" - if os.path.exists(file_path): + if file_path.is_file(): os.remove(file_path) - def safe_mkdir(self, directory_path): + def safe_mkdir(self, directory_path: Path): """Make a directory if it does not exist.""" - if not os.path.exists(directory_path): + if not directory_path.exists(): os.makedirs(directory_path) - def list_all_files(self, directory_path): + def list_all_files(self, directory_path: Path) -> Iterator[Path]: """Generator yielding path to all of the files in a directory tree.""" for root, dirs, files in os.walk(directory_path): for filename in files: - filepath = os.path.join(root, filename) - yield filepath + yield Path(root) / filename ####################### - def processdir(self, src_dir, dst_dir): + def processdir(self, src_dir: Path, dst_dir: Path): """Implement this function to define file processing behavior.""" raise ImplementationRequired('Please implement processdir.') @@ -341,7 +330,7 @@ class KittenGroomerBase(object): class KittenGroomerError(Exception): """Base KittenGroomer exception handler.""" - def __init__(self, message): + def __init__(self, message: str): super(KittenGroomerError, self).__init__(message) self.message = message diff --git a/setup.py b/setup.py index 1711e45..46ce454 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ #!/usr/bin/python # -*- coding: utf-8 -*- -from setuptools import setup +from setuptools import setup # type: ignore setup( name='kittengroomer', diff --git a/tests/test_filecheck.py b/tests/test_filecheck.py index 78c72ed..efaa9e8 100644 --- a/tests/test_filecheck.py +++ b/tests/test_filecheck.py @@ -2,8 +2,9 @@ # -*- coding: utf-8 -*- import os +from pathlib import Path -import pytest +import pytest # type: ignore import yaml try: @@ -19,16 +20,16 @@ skip = pytest.mark.skip parametrize = pytest.mark.parametrize -NORMAL_FILES_PATH = 'tests/normal/' -DANGEROUS_FILES_PATH = 'tests/dangerous/' -UNCATEGORIZED_FILES_PATH = 'tests/uncategorized' -CATALOG_PATH = 'tests/file_catalog.yaml' +NORMAL_FILES_PATH = Path('tests/normal/') +DANGEROUS_FILES_PATH = Path('tests/dangerous/') +UNCATEGORIZED_FILES_PATH = Path('tests/uncategorized') +CATALOG_PATH = Path('tests/file_catalog.yaml') class SampleFile(): def __init__(self, path, exp_dangerous): - self.path = path - self.filename = os.path.basename(path) + self.path = Path(path) + self.filename = self.path.name self.exp_dangerous = exp_dangerous @@ -91,13 +92,13 @@ def get_filename(sample_file): @fixture(scope='module') -def src_dir_path(tmpdir_factory): - return tmpdir_factory.mktemp('src').strpath +def src_dir_path(tmp_path_factory): + return tmp_path_factory.mktemp('src') @fixture(scope='module') -def dest_dir_path(tmpdir_factory): - return tmpdir_factory.mktemp('dest').strpath +def dest_dir_path(tmp_path_factory): + return tmp_path_factory.mktemp('dest') @fixture @@ -113,7 +114,7 @@ def groomer(dest_dir_path): def test_sample_files(sample_file, groomer, dest_dir_path): if sample_file.xfail: pytest.xfail(reason='Marked xfail in file catalog') - file_dest_path = os.path.join(dest_dir_path, sample_file.filename) + file_dest_path = dest_dir_path / sample_file.filename file = File(sample_file.path, file_dest_path) groomer.process_file(file) print(file.description_string) diff --git a/tests/test_filecheck_logging.py b/tests/test_filecheck_logging.py index 08c6c2c..0259bd0 100644 --- a/tests/test_filecheck_logging.py +++ b/tests/test_filecheck_logging.py @@ -4,7 +4,7 @@ import os from datetime import datetime -import pytest +import pytest # type: ignore try: from filecheck.filecheck import KittenGroomerFileCheck diff --git a/tests/test_kittengroomer.py b/tests/test_kittengroomer.py index 1b4dcaa..6ab4ee6 100644 --- a/tests/test_kittengroomer.py +++ b/tests/test_kittengroomer.py @@ -2,9 +2,10 @@ # -*- coding: utf-8 -*- import os +from pathlib import Path import unittest.mock as mock -import pytest +import pytest # type: ignore from kittengroomer import FileBase, KittenGroomerBase from kittengroomer.helpers import ImplementationRequired @@ -17,35 +18,34 @@ fixture = pytest.fixture class TestFileBase: @fixture(scope='class') - def src_dir_path(self, tmpdir_factory): - return tmpdir_factory.mktemp('src').strpath + def src_dir_path(self, tmp_path_factory): + return tmp_path_factory.mktemp('src') @fixture(scope='class') - def dest_dir_path(self, tmpdir_factory): - return tmpdir_factory.mktemp('dest').strpath + def dest_dir_path(self, tmp_path_factory): + return tmp_path_factory.mktemp('dest') - @fixture - def tmpfile_path(self, tmpdir): - file_path = tmpdir.join('test.txt') - file_path.write('testing') - return file_path.strpath + @fixture(scope='class') + def tmpfile_path(self, src_dir_path): + path_src_file = src_dir_path / 'test.txt' + with path_src_file.open('w') as f: + f.write('testing') + return path_src_file - @fixture - def symlink_file_path(self, tmpdir, tmpfile_path): - symlink_path = tmpdir.join('symlinked') - symlink_path = symlink_path.strpath - os.symlink(tmpfile_path, symlink_path) + @fixture(scope='class') + def symlink_file_path(self, src_dir_path, tmpfile_path): + symlink_path = src_dir_path / 'symlinked' + symlink_path.symlink_to(tmpfile_path) return symlink_path @fixture - def text_file(self): + def text_file(self, tmpfile_path, dest_dir_path): with mock.patch( 'kittengroomer.helpers.magic.from_file', return_value='text/plain' ): - src_path = 'src/test.txt' - dst_path = 'dst/test.txt' - file = FileBase(src_path, dst_path) + dst_path = dest_dir_path / 'test.txt' + file = FileBase(tmpfile_path, dst_path) return file # Constructor behavior @@ -53,63 +53,63 @@ class TestFileBase: @mock.patch('kittengroomer.helpers.magic') def test_init_identify_filename(self, mock_libmagic): """Init should identify the filename correctly for src_path.""" - src_path = 'src/test.txt' - dst_path = 'dst/test.txt' + src_path = Path('src/test.txt') + dst_path = Path('dst/test.txt') file = FileBase(src_path, dst_path) assert file.filename == 'test.txt' @mock.patch('kittengroomer.helpers.magic') def test_init_identify_extension(self, mock_libmagic): """Init should identify the extension for src_path.""" - src_path = 'src/test.txt' - dst_path = 'dst/test.txt' + src_path = Path('src/test.txt') + dst_path = Path('dst/test.txt') file = FileBase(src_path, dst_path) assert file.extension == '.txt' @mock.patch('kittengroomer.helpers.magic') def test_init_uppercase_extension(self, mock_libmagic): """Init should coerce uppercase extension to lowercase""" - src_path = 'src/TEST.TXT' - dst_path = 'dst/TEST.TXT' + src_path = Path('src/test.txt') + dst_path = Path('dst/test.txt') file = FileBase(src_path, dst_path) assert file.extension == '.txt' @mock.patch('kittengroomer.helpers.magic') def test_has_extension_true(self, mock_libmagic): """If the file has an extension, has_extension should == True.""" - src_path = 'src/test.txt' - dst_path = 'dst/test.txt' + src_path = Path('src/test.txt') + dst_path = Path('dst/test.txt') file = FileBase(src_path, dst_path) assert file.has_extension is True @mock.patch('kittengroomer.helpers.magic') def test_has_extension_false(self, mock_libmagic): """If the file has no extension, has_extensions should == False.""" - src_path = 'src/test' - dst_path = 'dst/test' + src_path = Path('src/test') + dst_path = Path('dst/test') file = FileBase(src_path, dst_path) assert file.has_extension is False def test_init_file_doesnt_exist(self): """Init should raise an exception if the file doesn't exist.""" with pytest.raises(FileNotFoundError): - FileBase('', '') + FileBase(Path('non_existent'), Path('non_existent')) def test_init_srcpath_is_directory(self, tmpdir): """Init should raise an exception if given a path to a directory.""" with pytest.raises(IsADirectoryError): - FileBase(tmpdir.strpath, tmpdir.strpath) + FileBase(Path(tmpdir.strpath), Path(tmpdir.strpath)) @mock.patch('kittengroomer.helpers.magic') - def test_init_symlink(self, mock_libmagic, symlink_file_path): + def test_init_symlink(self, mock_libmagic, symlink_file_path, tmpdir): """Init should properly identify symlinks.""" - file = FileBase(symlink_file_path, '') + file = FileBase(symlink_file_path, Path(tmpdir.strpath)) assert file.mimetype == 'inode/symlink' @mock.patch('kittengroomer.helpers.magic') - def test_is_symlink_attribute(self, mock_libmagic, symlink_file_path): + def test_is_symlink_attribute(self, mock_libmagic, symlink_file_path, tmpdir): """If a file is a symlink, is_symlink should return True.""" - file = FileBase(symlink_file_path, '') + file = FileBase(symlink_file_path, Path(tmpdir.strpath)) assert file.is_symlink is True def test_init_mimetype_attribute_assigned_correctly(self): @@ -117,7 +117,7 @@ class TestFileBase: assigned properly.""" with mock.patch('kittengroomer.helpers.magic.from_file', return_value='text/plain'): - file = FileBase('', '') + file = FileBase(Path('non_existent'), Path('non_existent')) assert file.mimetype == 'text/plain' def test_maintype_and_subtype_attributes(self): @@ -125,7 +125,7 @@ class TestFileBase: the appropriate values.""" with mock.patch('kittengroomer.helpers.magic.from_file', return_value='text/plain'): - file = FileBase('', '') + file = FileBase(Path('non_existent'), Path('non_existent')) assert file.maintype == 'text' assert file.subtype == 'plain' @@ -133,14 +133,14 @@ class TestFileBase: """If a file doesn't have a full mimetype has_mimetype should == False.""" with mock.patch('kittengroomer.helpers.magic.from_file', return_value='data'): - file = FileBase('', '') + file = FileBase(Path('non_existent'), Path('non_existent')) assert file.has_mimetype is False def test_has_mimetype_mimetype_is_none(self): """If a file doesn't have a full mimetype has_mimetype should == False.""" with mock.patch('kittengroomer.helpers.FileBase._determine_mimetype', return_value=None): - file = FileBase('', '') + file = FileBase(Path('non_existent'), Path('non_existent')) assert file.has_mimetype is False # File properties @@ -223,38 +223,38 @@ class TestFileBase: """Force_ext should modify the path of the file to end in the new extension.""" text_file.force_ext('.test') - assert text_file.dst_path.endswith('.test') + assert text_file.dst_path.name.endswith('.test') def test_force_ext_add_dot(self, text_file): """Force_ext should add a dot to an extension given without one.""" text_file.force_ext('test') - assert text_file.dst_path.endswith('.test') + assert text_file.dst_path.name.endswith('.test') def test_force_ext_change_extension_attr(self, text_file): - """Force_ext should modify the extension attribute""" + """Force_ext should only modify the extension of the destination file""" text_file.force_ext('.thing') - assert text_file.extension == '.thing' + assert text_file.extension == '.txt' def test_force_ext_no_change(self, text_file): """Force_ext should do nothing if the current extension is the same as the new extension.""" text_file.force_ext('.txt') assert text_file.extension == '.txt' - assert '.txt.txt' not in text_file.dst_path + assert '.txt.txt' not in text_file.dst_path.name def test_safe_copy_calls_copy(self, src_dir_path, dest_dir_path): """Calling safe_copy should copy the file from the correct path to the correct destination path.""" - file_path = os.path.join(src_dir_path, 'test.txt') + file_path = src_dir_path / 'test.txt' with open(file_path, 'w+') as file: file.write('') - dst_path = os.path.join(dest_dir_path, 'test.txt') + dst_path = dest_dir_path / 'test.txt' with mock.patch('kittengroomer.helpers.magic.from_file', return_value='text/plain'): file = FileBase(file_path, dst_path) with mock.patch('kittengroomer.helpers.shutil.copy') as mock_copy: file.safe_copy() - mock_copy.assert_called_once_with(file_path, dst_path) + mock_copy.assert_called_once_with(str(file_path), str(dst_path)) def test_safe_copy_removes_exec_perms(self): """`safe_copy` should create a file that doesn't have any of the @@ -289,22 +289,27 @@ class TestKittenGroomerBase: @fixture(scope='class') def src_dir_path(self, tmpdir_factory): - return tmpdir_factory.mktemp('src').strpath + return Path(tmpdir_factory.mktemp('src').strpath) @fixture(scope='class') def dest_dir_path(self, tmpdir_factory): - return tmpdir_factory.mktemp('dest').strpath + return Path(tmpdir_factory.mktemp('dest').strpath) + + @fixture(scope='class') + def tmpfile_path(self, src_dir_path): + path_src_file = src_dir_path / 'test.txt' + with path_src_file.open('w') as f: + f.write('testing') + return path_src_file @fixture def groomer(self, src_dir_path, dest_dir_path): return KittenGroomerBase(src_dir_path, dest_dir_path) - def test_list_all_files_includes_file(self, tmpdir, groomer): + def test_list_all_files_includes_file(self, src_dir_path, tmpfile_path, groomer): """Calling list_all_files should include files in the given path.""" - file = tmpdir.join('test.txt') - file.write('testing') - files = groomer.list_all_files(tmpdir.strpath) - assert file.strpath in files + files = groomer.list_all_files(src_dir_path) + assert tmpfile_path in files def test_list_all_files_excludes_dir(self, tmpdir, groomer): """Calling list_all_files shouldn't include directories in the given @@ -317,12 +322,12 @@ class TestKittenGroomerBase: def test_safe_remove(self, groomer, src_dir_path): """Calling safe_remove should not raise an Exception if trying to remove a file that doesn't exist.""" - groomer.safe_remove(os.path.join(src_dir_path, 'thing')) + groomer.safe_remove(src_dir_path / 'thing') def test_safe_mkdir_file_exists(self, groomer, dest_dir_path): """Calling safe_mkdir should not overwrite an existing directory.""" - filepath = os.path.join(dest_dir_path, 'thing') - os.mkdir(filepath) + filepath = dest_dir_path / 'thing' + filepath.mkdir() groomer.safe_mkdir(filepath) def test_processdir_not_implemented(self, groomer):