From e063c2a283443b02fd1a50fba6f4d3df7b63f1f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 17 Dec 2019 10:06:50 +0100 Subject: [PATCH 1/2] fix: Properly install pymisp with file object dependencies --- Pipfile | 2 +- Pipfile.lock | 165 +++++++++++++++++++++++++++++++-------------------- REQUIREMENTS | 11 ++-- 3 files changed, 108 insertions(+), 70 deletions(-) diff --git a/Pipfile b/Pipfile index 1cb0889..1a99c42 100644 --- a/Pipfile +++ b/Pipfile @@ -18,7 +18,7 @@ pypdns = "*" pypssl = "*" pyeupi = "*" uwhois = {editable = true,git = "https://github.com/Rafiot/uwhoisd.git",ref = "testing",subdirectory = "client"} -pymisp = {editable = true,git = "https://github.com/MISP/PyMISP.git"} +pymisp = {editable = true,extras = ["fileobjects,openioc,virustotal,pdfexport"],git = "https://github.com/MISP/PyMISP.git"} pyonyphe = {editable = true,git = "https://github.com/sebdraven/pyonyphe"} pydnstrails = {editable = true,git = "https://github.com/sebdraven/pydnstrails"} pytesseract = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 15a5d51..dab4860 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "2cd074bb42f3fbefc9eefdcd673817af96b25fdf8e7e7a149878b7ae8bbfcc66" + "sha256": "30e84f4986146c248e706f52f425649660225889bfcdf5075c99854442ae5f42" }, "pipfile-spec": 6, "requires": { @@ -296,11 +296,11 @@ }, "importlib-metadata": { "hashes": [ - "sha256:3a8b2dfd0a2c6a3636e7c016a7e54ae04b997d30e69d5eacdca7a6c2221a1402", - "sha256:41e688146d000891f32b1669e8573c57e39e5060e7f5f647aa617cd9a9568278" + "sha256:073a852570f92da5f744a3472af1b61e28e9f78ccf0c9117658dc32b15de7b45", + "sha256:d95141fbfa7ef2ec65cfd945e2af7e5a6ddbd7c8d9a25e66ff3be8e3daf9f60f" ], "markers": "python_version < '3.8'", - "version": "==1.2.0" + "version": "==1.3.0" }, "isodate": { "hashes": [ @@ -323,6 +323,25 @@ ], "version": "==3.2.0" }, + "lief": { + "hashes": [ + "sha256:276cc63ec12a21bdf01b8d30962692c17499788234f0765247ca7a35872097ec", + "sha256:3e6baaeb52bdc339b5f19688b58fd8d5778b92e50221f920cedfa2bec1f4d5c2", + "sha256:45e5c592b57168c447698381d927eb2386ffdd52afe0c48245f848d4cc7ee05a", + "sha256:6547752b5db105cd41c9fa65d0d7452a4d7541b77ffee716b46246c6d81e172f", + "sha256:83b51e01627b5982662f9550ac1230758aa56945ed86829e4291932d98417da3", + "sha256:895599194ea7495bf304e39317b04df20cccf799fc2751867cc1aa4997cfcdae", + "sha256:8a91cee2568306fe1d2bf84341b459c85368317d01d7105fa49e4f4ede837076", + "sha256:913b36a67707dc2afa72f117bab9856ea3f434f332b04a002a0f9723c8779320", + "sha256:9f604a361a3b1b3ed5fdafed0321c5956cb3b265b5efe2250d1bf8911a80c65b", + "sha256:a487fe7234c04bccd58223dbb79214421176e2629814c7a4a887764cceb5be7c", + "sha256:bc8488fb0661cb436fe4bb4fe947d0f9aa020e9acaed233ccf01ab04d888c68a", + "sha256:bddbf333af62310a10cb738a1df1dc2b140dd9c663b55ba3500c10c249d416d2", + "sha256:cce48d7c97cef85e01e6cfeff55f2068956b5c0257eb9c2d2c6d15e33dd1e4fc", + "sha256:f8b3f66956c56b582b3adc573bf2a938c25fb21c8894b373a113e24c494fc982" + ], + "version": "==0.10.1" + }, "lxml": { "hashes": [ "sha256:00ac0d64949fef6b3693813fe636a2d56d97a5a49b5bbb86e4cc4cc50ebc9ea2", @@ -382,25 +401,25 @@ }, "multidict": { "hashes": [ - "sha256:07f9a6bf75ad675d53956b2c6a2d4ef2fa63132f33ecc99e9c24cf93beb0d10b", - "sha256:0ffe4d4d28cbe9801952bfb52a8095dd9ffecebd93f84bdf973c76300de783c5", - "sha256:1b605272c558e4c659dbaf0fb32a53bfede44121bcf77b356e6e906867b958b7", - "sha256:205a011e636d885af6dd0029e41e3514a46e05bb2a43251a619a6e8348b96fc0", - "sha256:250632316295f2311e1ed43e6b26a63b0216b866b45c11441886ac1543ca96e1", - "sha256:2bc9c2579312c68a3552ee816311c8da76412e6f6a9cf33b15152e385a572d2a", - "sha256:318aadf1cfb6741c555c7dd83d94f746dc95989f4f106b25b8a83dfb547f2756", - "sha256:42cdd649741a14b0602bf15985cad0dd4696a380081a3319cd1ead46fd0f0fab", - "sha256:5159c4975931a1a78bf6602bbebaa366747fce0a56cb2111f44789d2c45e379f", - "sha256:87e26d8b89127c25659e962c61a4c655ec7445d19150daea0759516884ecb8b4", - "sha256:891b7e142885e17a894d9d22b0349b92bb2da4769b4e675665d0331c08719be5", - "sha256:8d919034420378132d074bf89df148d0193e9780c9fe7c0e495e895b8af4d8a2", - "sha256:9c890978e2b37dd0dc1bd952da9a5d9f245d4807bee33e3517e4119c48d66f8c", - "sha256:a37433ce8cdb35fc9e6e47e1606fa1bfd6d70440879038dca7d8dd023197eaa9", - "sha256:c626029841ada34c030b94a00c573a0c7575fe66489cde148785b6535397d675", - "sha256:cfec9d001a83dc73580143f3c77e898cf7ad78b27bb5e64dbe9652668fcafec7", - "sha256:efaf1b18ea6c1f577b1371c0159edbe4749558bfe983e13aa24d0a0c01e1ad7b" + "sha256:09c19f642e055550c9319d5123221b7e07fc79bda58122aa93910e52f2ab2f29", + "sha256:0c1a5d5f7aa7189f7b83c4411c2af8f1d38d69c4360d5de3eea129c65d8d7ce2", + "sha256:12f22980e7ed0972a969520fb1e55682c9fca89a68b21b49ec43132e680be812", + "sha256:258660e9d6b52de1a75097944e12718d3aa59adc611b703361e3577d69167aaf", + "sha256:3374a23e707848f27b3438500db0c69eca82929337656fce556bd70031fbda74", + "sha256:503b7fce0054c73aa631cc910a470052df33d599f3401f3b77e54d31182525d5", + "sha256:6ce55f2c45ffc90239aab625bb1b4864eef33f73ea88487ef968291fbf09fb3f", + "sha256:725496dde5730f4ad0a627e1a58e2620c1bde0ad1c8080aae15d583eb23344ce", + "sha256:a3721078beff247d0cd4fb19d915c2c25f90907cf8d6cd49d0413a24915577c6", + "sha256:ba566518550f81daca649eded8b5c7dd09210a854637c82351410aa15c49324a", + "sha256:c42362750a51a15dc905cb891658f822ee5021bfbea898c03aa1ed833e2248a5", + "sha256:cf14aaf2ab067ca10bca0b14d5cbd751dd249e65d371734bc0e47ddd8fafc175", + "sha256:cf24e15986762f0e75a622eb19cfe39a042e952b8afba3e7408835b9af2be4fb", + "sha256:d7b6da08538302c5245cd3103f333655ba7f274915f1f5121c4f4b5fbdb3febe", + "sha256:e27e13b9ff0a914a6b8fb7e4947d4ac6be8e4f61ede17edffabd088817df9e26", + "sha256:e53b205f8afd76fc6c942ef39e8ee7c519c775d336291d32874082a87802c67c", + "sha256:ec804fc5f68695d91c24d716020278fcffd50890492690a7e1fef2e741f7172c" ], - "version": "==4.6.1" + "version": "==4.7.1" }, "np": { "hashes": [ @@ -674,6 +693,12 @@ ], "version": "==3.9.4" }, + "pydeep": { + "hashes": [ + "sha256:22866eb422d1d5907f8076ee792da65caecb172425d27576274e2a8eacf6afc1" + ], + "version": "==0.4" + }, "pydnstrails": { "editable": true, "git": "https://github.com/sebdraven/pydnstrails", @@ -707,8 +732,11 @@ }, "pymisp": { "editable": true, + "extras": [ + "fileobjects,openioc,virustotal,pdfexport" + ], "git": "https://github.com/MISP/PyMISP.git", - "ref": "36cc79ffb686430e02b382dfef85c29a5e27c39d" + "ref": "a26a8e450b14d48bb0c8ef46b32bff2f1eadc514" }, "pyonyphe": { "editable": true, @@ -771,6 +799,13 @@ "index": "pypi", "version": "==0.8.10" }, + "python-magic": { + "hashes": [ + "sha256:f2674dcfad52ae6c49d4803fa027809540b130db1dec928cfbb9240316831375", + "sha256:f3765c0f582d2dfc72c15f3b5a82aecfae9498bd29ca840d72f37d7bd38bfcd5" + ], + "version": "==0.4.15" + }, "python-pptx": { "hashes": [ "sha256:a857d69e52d7e8a8fb32fca8182fdd4a3c68c689de8d4e4460e9b4a95efa7bc4" @@ -893,14 +928,15 @@ }, "shodan": { "hashes": [ - "sha256:2efe383eeb083eb67137a00cc6fc5ea1fd848ce8053dfdea6696bc6ec05f6e98" + "sha256:eab999bca9d3b30e6fc549e609194ff2d6fac3caea252414e1d8d735efab8342" ], "index": "pypi", - "version": "==1.20.0" + "version": "==1.21.0" }, "sigmatools": { "hashes": [ - "sha256:2331bc1c6bd8e69ff3e201e51552328794f6cfc3597004fa0865341748750737" + "sha256:2331bc1c6bd8e69ff3e201e51552328794f6cfc3597004fa0865341748750737", + "sha256:4361515fb8d6c6389cc0d1e5057b1f7d4cec11b8fb814e561253c01050efa634" ], "index": "pypi", "version": "==0.15.0" @@ -1116,40 +1152,39 @@ }, "coverage": { "hashes": [ - "sha256:08907593569fe59baca0bf152c43f3863201efb6113ecb38ce7e97ce339805a6", - "sha256:0be0f1ed45fc0c185cfd4ecc19a1d6532d72f86a2bac9de7e24541febad72650", - "sha256:141f08ed3c4b1847015e2cd62ec06d35e67a3ac185c26f7635f4406b90afa9c5", - "sha256:19e4df788a0581238e9390c85a7a09af39c7b539b29f25c89209e6c3e371270d", - "sha256:23cc09ed395b03424d1ae30dcc292615c1372bfba7141eb85e11e50efaa6b351", - "sha256:245388cda02af78276b479f299bbf3783ef0a6a6273037d7c60dc73b8d8d7755", - "sha256:331cb5115673a20fb131dadd22f5bcaf7677ef758741312bee4937d71a14b2ef", - "sha256:386e2e4090f0bc5df274e720105c342263423e77ee8826002dcffe0c9533dbca", - "sha256:3a794ce50daee01c74a494919d5ebdc23d58873747fa0e288318728533a3e1ca", - "sha256:60851187677b24c6085248f0a0b9b98d49cba7ecc7ec60ba6b9d2e5574ac1ee9", - "sha256:63a9a5fc43b58735f65ed63d2cf43508f462dc49857da70b8980ad78d41d52fc", - "sha256:6b62544bb68106e3f00b21c8930e83e584fdca005d4fffd29bb39fb3ffa03cb5", - "sha256:6ba744056423ef8d450cf627289166da65903885272055fb4b5e113137cfa14f", - "sha256:7494b0b0274c5072bddbfd5b4a6c6f18fbbe1ab1d22a41e99cd2d00c8f96ecfe", - "sha256:826f32b9547c8091679ff292a82aca9c7b9650f9fda3e2ca6bf2ac905b7ce888", - "sha256:93715dffbcd0678057f947f496484e906bf9509f5c1c38fc9ba3922893cda5f5", - "sha256:9a334d6c83dfeadae576b4d633a71620d40d1c379129d587faa42ee3e2a85cce", - "sha256:af7ed8a8aa6957aac47b4268631fa1df984643f07ef00acd374e456364b373f5", - "sha256:bf0a7aed7f5521c7ca67febd57db473af4762b9622254291fbcbb8cd0ba5e33e", - "sha256:bf1ef9eb901113a9805287e090452c05547578eaab1b62e4ad456fcc049a9b7e", - "sha256:c0afd27bc0e307a1ffc04ca5ec010a290e49e3afbe841c5cafc5c5a80ecd81c9", - "sha256:dd579709a87092c6dbee09d1b7cfa81831040705ffa12a1b248935274aee0437", - "sha256:df6712284b2e44a065097846488f66840445eb987eb81b3cc6e4149e7b6982e1", - "sha256:e07d9f1a23e9e93ab5c62902833bf3e4b1f65502927379148b6622686223125c", - "sha256:e2ede7c1d45e65e209d6093b762e98e8318ddeff95317d07a27a2140b80cfd24", - "sha256:e4ef9c164eb55123c62411f5936b5c2e521b12356037b6e1c2617cef45523d47", - "sha256:eca2b7343524e7ba246cab8ff00cab47a2d6d54ada3b02772e908a45675722e2", - "sha256:eee64c616adeff7db37cc37da4180a3a5b6177f5c46b187894e633f088fb5b28", - "sha256:ef824cad1f980d27f26166f86856efe11eff9912c4fed97d3804820d43fa550c", - "sha256:efc89291bd5a08855829a3c522df16d856455297cf35ae827a37edac45f466a7", - "sha256:fa964bae817babece5aa2e8c1af841bebb6d0b9add8e637548809d040443fee0", - "sha256:ff37757e068ae606659c28c3bd0d923f9d29a85de79bf25b2b34b148473b5025" + "sha256:0cd13a6e98c37b510a2d34c8281d5e1a226aaf9b65b7d770ef03c63169965351", + "sha256:1a4b6b6a2a3a6612e6361130c2cc3dc4378d8c221752b96167ccbad94b47f3cd", + "sha256:2ee55e6dba516ddf6f484aa83ccabbb0adf45a18892204c23486938d12258cde", + "sha256:3be5338a2eb4ef03c57f20917e1d12a1fd10e3853fed060b6d6b677cb3745898", + "sha256:44b783b02db03c4777d8cf71bae19eadc171a6f2a96777d916b2c30a1eb3d070", + "sha256:475bf7c4252af0a56e1abba9606f1e54127cdf122063095c75ab04f6f99cf45e", + "sha256:47c81ee687eafc2f1db7f03fbe99aab81330565ebc62fb3b61edfc2216a550c8", + "sha256:4a7f8e72b18f2aca288ff02255ce32cc830bc04d993efbc87abf6beddc9e56c0", + "sha256:50197163a22fd17f79086e087a787883b3ec9280a509807daf158dfc2a7ded02", + "sha256:56b13000acf891f700f5067512b804d1ec8c301d627486c678b903859d07f798", + "sha256:79388ae29c896299b3567965dbcd93255f175c17c6c7bca38614d12718c47466", + "sha256:79fd5d3d62238c4f583b75d48d53cdae759fe04d4fb18fe8b371d88ad2b6f8be", + "sha256:7fe3e2fde2bf1d7ce25ebcd2d3de3650b8d60d9a73ce6dcef36e20191291613d", + "sha256:81042a24f67b96e4287774014fa27220d8a4d91af1043389e4d73892efc89ac6", + "sha256:81326f1095c53111f8afc95da281e1414185f4a538609a77ca50bdfa39a6c207", + "sha256:8873dc0d8f42142ea9f20c27bbdc485190fff93823c6795be661703369e5877d", + "sha256:88d2cbcb0a112f47eef71eb95460b6995da18e6f8ca50c264585abc2c473154b", + "sha256:91f2491aeab9599956c45a77c5666d323efdec790bfe23fcceafcd91105d585a", + "sha256:979daa8655ae5a51e8e7a24e7d34e250ae8309fd9719490df92cbb2fe2b0422b", + "sha256:9c871b006c878a890c6e44a5b2f3c6291335324b298c904dc0402ee92ee1f0be", + "sha256:a6d092545e5af53e960465f652e00efbf5357adad177b2630d63978d85e46a72", + "sha256:b5ed7837b923d1d71c4f587ae1539ccd96bfd6be9788f507dbe94dab5febbb5d", + "sha256:ba259f68250f16d2444cbbfaddaa0bb20e1560a4fdaad50bece25c199e6af864", + "sha256:be1d89614c6b6c36d7578496dc8625123bda2ff44f224cf8b1c45b810ee7383f", + "sha256:c1b030a79749aa8d1f1486885040114ee56933b15ccfc90049ba266e4aa2139f", + "sha256:c95bb147fab76f2ecde332d972d8f4138b8f2daee6c466af4ff3b4f29bd4c19e", + "sha256:d52c1c2d7e856cecc05aa0526453cb14574f821b7f413cc279b9514750d795c1", + "sha256:d609a6d564ad3d327e9509846c2c47f170456344521462b469e5cb39e48ba31c", + "sha256:e1bad043c12fb58e8c7d92b3d7f2f49977dcb80a08a6d1e7a5114a11bf819fca", + "sha256:e5a675f6829c53c87d79117a8eb656cc4a5f8918185a32fc93ba09778e90f6db", + "sha256:fec32646b98baf4a22fdceb08703965bd16dea09051fbeb31a04b5b6e72b846c" ], - "version": "==4.5.4" + "version": "==5.0" }, "entrypoints": { "hashes": [ @@ -1175,11 +1210,11 @@ }, "importlib-metadata": { "hashes": [ - "sha256:3a8b2dfd0a2c6a3636e7c016a7e54ae04b997d30e69d5eacdca7a6c2221a1402", - "sha256:41e688146d000891f32b1669e8573c57e39e5060e7f5f647aa617cd9a9568278" + "sha256:073a852570f92da5f744a3472af1b61e28e9f78ccf0c9117658dc32b15de7b45", + "sha256:d95141fbfa7ef2ec65cfd945e2af7e5a6ddbd7c8d9a25e66ff3be8e3daf9f60f" ], "markers": "python_version < '3.8'", - "version": "==1.2.0" + "version": "==1.3.0" }, "mccabe": { "hashes": [ @@ -1248,11 +1283,11 @@ }, "pytest": { "hashes": [ - "sha256:63344a2e3bce2e4d522fd62b4fdebb647c019f1f9e4ca075debbd13219db4418", - "sha256:f67403f33b2b1d25a6756184077394167fe5e2f9d8bdaab30707d19ccec35427" + "sha256:6b571215b5a790f9b41f19f3531c53a45cf6bb8ef2988bc1ff9afb38270b25fa", + "sha256:e41d489ff43948babd0fad7ad5e49b8735d5d55e26628a58673c39ff61d95de4" ], "index": "pypi", - "version": "==5.3.1" + "version": "==5.3.2" }, "requests": { "extras": [ diff --git a/REQUIREMENTS b/REQUIREMENTS index b279b59..ee6c7c1 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -3,7 +3,7 @@ -e git+https://github.com/D4-project/BGP-Ranking.git/@fd9c0e03af9b61d4bf0b67ac73c7208a55178a54#egg=pybgpranking&subdirectory=client -e git+https://github.com/D4-project/IPASN-History.git/@fc5e48608afc113e101ca6421bf693b7b9753f9e#egg=pyipasnhistory&subdirectory=client -e git+https://github.com/MISP/PyIntel471.git@0df8d51f1c1425de66714b3a5a45edb69b8cc2fc#egg=pyintel471 --e git+https://github.com/MISP/PyMISP.git@36cc79ffb686430e02b382dfef85c29a5e27c39d#egg=pymisp +-e git+https://github.com/MISP/PyMISP.git@a26a8e450b14d48bb0c8ef46b32bff2f1eadc514#egg=pymisp[fileobjects,openioc,virustotal,pdfexport] -e git+https://github.com/Rafiot/uwhoisd.git@411572840eba4c72dc321c549b36a54ed5cea9de#egg=uwhois&subdirectory=client -e git+https://github.com/cartertemm/ODTReader.git/@49d6938693f6faa3ff09998f86dba551ae3a996b#egg=odtreader -e git+https://github.com/sebdraven/pydnstrails@48c1f740025c51289f43a24863d1845ff12fd21a#egg=pydnstrails @@ -37,15 +37,16 @@ geoip2==2.9.0 httplib2==0.14.0 idna-ssl==1.1.0 ; python_version < '3.7' idna==2.8 -importlib-metadata==1.2.0 ; python_version < '3.8' +importlib-metadata==1.3.0 ; python_version < '3.8' isodate==0.6.0 jbxapi==3.4.0 jsonschema==3.2.0 +lief==0.10.1 lxml==4.4.2 maclookup==1.0.3 maxminddb==1.5.1 more-itertools==8.0.2 -multidict==4.6.1 +multidict==4.7.1 np==1.0.2 numpy==1.17.4 oauth2==1.9.0.post1 @@ -60,6 +61,7 @@ psutil==5.6.7 pycparser==2.19 pycryptodome==3.9.4 pycryptodomex==3.9.4 +pydeep==0.4 pyeupi==1.0 pygeoip==0.3.2 pyopenssl==19.1.0 @@ -70,6 +72,7 @@ pyrsistent==0.15.6 pytesseract==0.3.0 python-dateutil==2.8.1 python-docx==0.8.10 +python-magic==0.4.15 python-pptx==0.6.18 python-utils==2.3.0 pytz==2019.3 @@ -81,7 +84,7 @@ redis==3.3.11 reportlab==3.5.32 requests-cache==0.5.2 requests[security]==2.22.0 -shodan==1.20.0 +shodan==1.21.0 sigmatools==0.15.0 six==1.13.0 socketio-client==0.5.6 From adda243c33567cf7aefb876671366d4a705501da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 17 Dec 2019 10:19:05 +0100 Subject: [PATCH 2/2] fix: Missing dependency in travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3fc08dc..0b87679 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,7 @@ before_install: - docker build -t misp-modules --build-arg BUILD_DATE=$(date -u +"%Y-%m-%d") docker/ install: - - sudo apt-get install libzbar0 libzbar-dev libpoppler-cpp-dev tesseract-ocr + - sudo apt-get install libzbar0 libzbar-dev libpoppler-cpp-dev tesseract-ocr libfuzzy-dev - pip install pipenv - pipenv install --dev # install pyfaup