diff --git a/.travis.yml b/.travis.yml index b574d4c..e8fea8e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,14 +19,14 @@ script: - pid=$! - sleep 5 - pipenv run nosetests --with-coverage --cover-package=misp_modules - - kill -s INT $pid + - kill -s KILL $pid - pushd ~/ - pipenv run coverage run -m --parallel-mode --source=misp_modules misp_modules.__init__ -s -l 127.0.0.1 & - pid=$! - popd - sleep 5 - pipenv run nosetests --with-coverage --cover-package=misp_modules - - kill -s INT $pid + - kill -s KILL $pid - pipenv run flake8 --ignore=E501,W503 misp_modules after_success: diff --git a/Pipfile b/Pipfile index 45c05f5..2f2d172 100644 --- a/Pipfile +++ b/Pipfile @@ -41,6 +41,7 @@ domaintools_api = "*" misp-modules = {editable = true,path = "."} pybgpranking = {editable = true,git = "https://github.com/D4-project/BGP-Ranking.git/",subdirectory = "client"} pyipasnhistory = {editable = true,git = "https://github.com/D4-project/IPASN-History.git/",subdirectory = "client"} +backscatter = "*" [requires] python_version = "3.6" diff --git a/Pipfile.lock b/Pipfile.lock index 9e6265d..b5faca1 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "d0cd64bfe7702365d3ea66d1f51a1ec8592df2490899e7e163fe38f97172561e" + "sha256": "23dec0fa6400c828e294ea9981b433903c17358ca61d7abdaec8df5a1c89f08c" }, "pipfile-spec": 6, "requires": { @@ -59,10 +59,18 @@ }, "attrs": { "hashes": [ - "sha256:10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69", - "sha256:ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb" + "sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79", + "sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399" ], - "version": "==18.2.0" + "version": "==19.1.0" + }, + "backscatter": { + "hashes": [ + "sha256:7a0d1aa3661635de81e2a09b15d53e35cbe399a111cc58a70925f80e6874abd3", + "sha256:afb0efcf5d2551dac953ec4c38fb710b274b8e811775650e02c1ef42cafb14c8" + ], + "index": "pypi", + "version": "==0.2.4" }, "beautifulsoup4": { "hashes": [ @@ -82,10 +90,10 @@ }, "certifi": { "hashes": [ - "sha256:47f9c83ef4c0c621eaef743f133f09fa8a74a9b75f037e8624f83bd1b6626cb7", - "sha256:993f830721089fef441cdfeb4b2c8c9df86f0c63239f06bd025a76a7daddb033" + "sha256:59b7658e26ca9c7339e00f8f4636cdfe59d34fa37b9b04f6f9e9926b3cece1a5", + "sha256:b26104d6835d1f5e49452a26eb2ff87fe7090b89dfcaee5ea2212697e1e1d7ae" ], - "version": "==2018.11.29" + "version": "==2019.3.9" }, "chardet": { "hashes": [ @@ -177,10 +185,10 @@ }, "jsonschema": { "hashes": [ - "sha256:acc8a90c31d11060516cfd0b414b9f8bcf4bc691b21f0f786ea57dd5255c79db", - "sha256:dd3f8ecb1b52d94d45eedb67cb86cac57b94ded562c5d98f63719e55ce58557b" + "sha256:0c0a81564f181de3212efa2d17de1910f8732fa1b71c42266d983cd74304e20d", + "sha256:a5f6559964a3851f59040d3b961de5e68e70971afb88ba519d27e6a039efff1a" ], - "version": "==3.0.0" + "version": "==3.0.1" }, "maclookup": { "hashes": [ @@ -245,58 +253,54 @@ }, "pillow": { "hashes": [ - "sha256:051de330a06c99d6f84bcf582960487835bcae3fc99365185dc2d4f65a390c0e", - "sha256:0ae5289948c5e0a16574750021bd8be921c27d4e3527800dc9c2c1d2abc81bf7", - "sha256:0b1efce03619cdbf8bcc61cfae81fcda59249a469f31c6735ea59badd4a6f58a", - "sha256:163136e09bd1d6c6c6026b0a662976e86c58b932b964f255ff384ecc8c3cefa3", - "sha256:18e912a6ccddf28defa196bd2021fe33600cbe5da1aa2f2e2c6df15f720b73d1", - "sha256:24ec3dea52339a610d34401d2d53d0fb3c7fd08e34b20c95d2ad3973193591f1", - "sha256:267f8e4c0a1d7e36e97c6a604f5b03ef58e2b81c1becb4fccecddcb37e063cc7", - "sha256:3273a28734175feebbe4d0a4cde04d4ed20f620b9b506d26f44379d3c72304e1", - "sha256:4c678e23006798fc8b6f4cef2eaad267d53ff4c1779bd1af8725cc11b72a63f3", - "sha256:4d4bc2e6bb6861103ea4655d6b6f67af8e5336e7216e20fff3e18ffa95d7a055", - "sha256:505738076350a337c1740a31646e1de09a164c62c07db3b996abdc0f9d2e50cf", - "sha256:5233664eadfa342c639b9b9977190d64ad7aca4edc51a966394d7e08e7f38a9f", - "sha256:5d95cb9f6cced2628f3e4de7e795e98b2659dfcc7176ab4a01a8b48c2c2f488f", - "sha256:7eda4c737637af74bac4b23aa82ea6fbb19002552be85f0b89bc27e3a762d239", - "sha256:801ddaa69659b36abf4694fed5aa9f61d1ecf2daaa6c92541bbbbb775d97b9fe", - "sha256:825aa6d222ce2c2b90d34a0ea31914e141a85edefc07e17342f1d2fdf121c07c", - "sha256:9c215442ff8249d41ff58700e91ef61d74f47dfd431a50253e1a1ca9436b0697", - "sha256:a3d90022f2202bbb14da991f26ca7a30b7e4c62bf0f8bf9825603b22d7e87494", - "sha256:a631fd36a9823638fe700d9225f9698fb59d049c942d322d4c09544dc2115356", - "sha256:a6523a23a205be0fe664b6b8747a5c86d55da960d9586db039eec9f5c269c0e6", - "sha256:a756ecf9f4b9b3ed49a680a649af45a8767ad038de39e6c030919c2f443eb000", - "sha256:b117287a5bdc81f1bac891187275ec7e829e961b8032c9e5ff38b70fd036c78f", - "sha256:ba04f57d1715ca5ff74bb7f8a818bf929a204b3b3c2c2826d1e1cc3b1c13398c", - "sha256:cd878195166723f30865e05d87cbaf9421614501a4bd48792c5ed28f90fd36ca", - "sha256:cee815cc62d136e96cf76771b9d3eb58e0777ec18ea50de5cfcede8a7c429aa8", - "sha256:d1722b7aa4b40cf93ac3c80d3edd48bf93b9208241d166a14ad8e7a20ee1d4f3", - "sha256:d7c1c06246b05529f9984435fc4fa5a545ea26606e7f450bdbe00c153f5aeaad", - "sha256:e9c8066249c040efdda84793a2a669076f92a301ceabe69202446abb4c5c5ef9", - "sha256:f227d7e574d050ff3996049e086e1f18c7bd2d067ef24131e50a1d3fe5831fbc", - "sha256:fc9a12aad714af36cf3ad0275a96a733526571e52710319855628f476dcb144e" + "sha256:15c056bfa284c30a7f265a41ac4cbbc93bdbfc0dfe0613b9cb8a8581b51a9e55", + "sha256:1a4e06ba4f74494ea0c58c24de2bb752818e9d504474ec95b0aa94f6b0a7e479", + "sha256:1c3c707c76be43c9e99cb7e3d5f1bee1c8e5be8b8a2a5eeee665efbf8ddde91a", + "sha256:1fd0b290203e3b0882d9605d807b03c0f47e3440f97824586c173eca0aadd99d", + "sha256:24114e4a6e1870c5a24b1da8f60d0ba77a0b4027907860188ea82bd3508c80eb", + "sha256:258d886a49b6b058cd7abb0ab4b2b85ce78669a857398e83e8b8e28b317b5abb", + "sha256:33c79b6dd6bc7f65079ab9ca5bebffb5f5d1141c689c9c6a7855776d1b09b7e8", + "sha256:367385fc797b2c31564c427430c7a8630db1a00bd040555dfc1d5c52e39fcd72", + "sha256:3c1884ff078fb8bf5f63d7d86921838b82ed4a7d0c027add773c2f38b3168754", + "sha256:44e5240e8f4f8861d748f2a58b3f04daadab5e22bfec896bf5434745f788f33f", + "sha256:46aa988e15f3ea72dddd81afe3839437b755fffddb5e173886f11460be909dce", + "sha256:74d90d499c9c736d52dd6d9b7221af5665b9c04f1767e35f5dd8694324bd4601", + "sha256:809c0a2ce9032cbcd7b5313f71af4bdc5c8c771cb86eb7559afd954cab82ebb5", + "sha256:85d1ef2cdafd5507c4221d201aaf62fc9276f8b0f71bd3933363e62a33abc734", + "sha256:8c3889c7681af77ecfa4431cd42a2885d093ecb811e81fbe5e203abc07e0995b", + "sha256:9218d81b9fca98d2c47d35d688a0cea0c42fd473159dfd5612dcb0483c63e40b", + "sha256:9aa4f3827992288edd37c9df345783a69ef58bd20cc02e64b36e44bcd157bbf1", + "sha256:9d80f44137a70b6f84c750d11019a3419f409c944526a95219bea0ac31f4dd91", + "sha256:b7ebd36128a2fe93991293f997e44be9286503c7530ace6a55b938b20be288d8", + "sha256:c4c78e2c71c257c136cdd43869fd3d5e34fc2162dc22e4a5406b0ebe86958239", + "sha256:c6a842537f887be1fe115d8abb5daa9bc8cc124e455ff995830cc785624a97af", + "sha256:cf0a2e040fdf5a6d95f4c286c6ef1df6b36c218b528c8a9158ec2452a804b9b8", + "sha256:cfd28aad6fc61f7a5d4ee556a997dc6e5555d9381d1390c00ecaf984d57e4232", + "sha256:dca5660e25932771460d4688ccbb515677caaf8595f3f3240ec16c117deff89a", + "sha256:de7aedc85918c2f887886442e50f52c1b93545606317956d65f342bd81cb4fc3", + "sha256:e6c0bbf8e277b74196e3140c35f9a1ae3eafd818f7f2d3a15819c49135d6c062" ], "index": "pypi", - "version": "==5.4.1" + "version": "==6.0.0" }, "psutil": { "hashes": [ - "sha256:5ce6b5eb0267233459f4d3980c205828482f450999b8f5b684d9629fea98782a", - "sha256:72cebfaa422b7978a1d3632b65ff734a34c6b34f4578b68a5c204d633756b810", - "sha256:77c231b4dff8c1c329a4cd1c22b96c8976c597017ff5b09993cd148d6a94500c", - "sha256:8846ab0be0cdccd6cc92ecd1246a16e2f2e49f53bd73e522c3a75ac291e1b51d", - "sha256:a013b4250ccbddc9d22feca0f986a1afc71717ad026c0f2109bbffd007351191", - "sha256:ad43b83119eeea6d5751023298cd331637e542cbd332196464799e25a5519f8f", - "sha256:c177777c787d247d02dae6c855330f9ed3e1abf8ca1744c26dd5ff968949999a", - "sha256:ec1ef313530a9457e48d25e3fdb1723dfa636008bf1b970027462d46f2555d59", - "sha256:ef3e5e02b3c5d1df366abe7b4820400d5c427579668ad4465ff189d28ded5ebd" + "sha256:23e9cd90db94fbced5151eaaf9033ae9667c033dffe9e709da761c20138d25b6", + "sha256:27858d688a58cbfdd4434e1c40f6c79eb5014b709e725c180488ccdf2f721729", + "sha256:354601a1d1a1322ae5920ba397c58d06c29728a15113598d1a8158647aaa5385", + "sha256:9c3a768486194b4592c7ae9374faa55b37b9877fd9746fb4028cb0ac38fd4c60", + "sha256:c1fd45931889dc1812ba61a517630d126f6185f688eac1693171c6524901b7de", + "sha256:d463a142298112426ebd57351b45c39adb41341b91f033aa903fa4c6f76abecc", + "sha256:e1494d20ffe7891d07d8cb9a8b306c1a38d48b13575265d090fc08910c56d474", + "sha256:ec4b4b638b84d42fc48139f9352f6c6587ee1018d55253542ee28db7480cc653", + "sha256:fa0a570e0a30b9dd618bffbece590ae15726b47f9f1eaf7518dfb35f4d7dcd21" ], - "version": "==5.5.1" + "version": "==5.6.1" }, "pybgpranking": { "editable": true, "git": "https://github.com/D4-project/BGP-Ranking.git/", - "ref": "37c97ae252ec4bf1d67733a49d4895c8cb009cf9", + "ref": "019ef1c40aad1e5bb5c5072c9a998c6a8f0271f3", "subdirectory": "client" }, "pydnstrails": { @@ -327,13 +331,13 @@ "pyipasnhistory": { "editable": true, "git": "https://github.com/D4-project/IPASN-History.git/", - "ref": "e846cd36fe1ed6b22f60890bba89f84e61b62e59", + "ref": "0c4f11792061417b77ca6e22d2ece18109d74c75", "subdirectory": "client" }, "pymisp": { "editable": true, "git": "https://github.com/MISP/PyMISP.git", - "ref": "634ecc3ac308d01ebf5f5fbb9aace7746a2b8707" + "ref": "64bcaad0e578129543cdffad532a232722615f6c" }, "pyonyphe": { "editable": true, @@ -383,19 +387,19 @@ }, "pyyaml": { "hashes": [ - "sha256:3d7da3009c0f3e783b2c873687652d83b1bbfd5c88e9813fb7e5b03c0dd3108b", - "sha256:3ef3092145e9b70e3ddd2c7ad59bdd0252a94dfe3949721633e41344de00a6bf", - "sha256:40c71b8e076d0550b2e6380bada1f1cd1017b882f7e16f09a65be98e017f211a", - "sha256:558dd60b890ba8fd982e05941927a3911dc409a63dcb8b634feaa0cda69330d3", - "sha256:a7c28b45d9f99102fa092bb213aa12e0aaf9a6a1f5e395d36166639c1f96c3a1", - "sha256:aa7dd4a6a427aed7df6fb7f08a580d68d9b118d90310374716ae90b710280af1", - "sha256:bc558586e6045763782014934bfaf39d48b8ae85a2713117d16c39864085c613", - "sha256:d46d7982b62e0729ad0175a9bc7e10a566fc07b224d2c79fafb5e032727eaa04", - "sha256:d5eef459e30b09f5a098b9cea68bebfeb268697f78d647bd255a085371ac7f3f", - "sha256:e01d3203230e1786cd91ccfdc8f8454c8069c91bee3962ad93b87a4b2860f537", - "sha256:e170a9e6fcfd19021dd29845af83bb79236068bf5fd4df3327c1be18182b2531" + "sha256:1adecc22f88d38052fb787d959f003811ca858b799590a5eaa70e63dca50308c", + "sha256:436bc774ecf7c103814098159fbb84c2715d25980175292c648f2da143909f95", + "sha256:460a5a4248763f6f37ea225d19d5c205677d8d525f6a83357ca622ed541830c2", + "sha256:5a22a9c84653debfbf198d02fe592c176ea548cccce47553f35f466e15cf2fd4", + "sha256:7a5d3f26b89d688db27822343dfa25c599627bc92093e788956372285c6298ad", + "sha256:9372b04a02080752d9e6f990179a4ab840227c6e2ce15b95e1278456664cf2ba", + "sha256:a5dcbebee834eaddf3fa7366316b880ff4062e4bcc9787b78c7fbb4a26ff2dd1", + "sha256:aee5bab92a176e7cd034e57f46e9df9a9862a71f8f37cad167c6fc74c65f5b4e", + "sha256:c51f642898c0bacd335fc119da60baae0824f2cde95b0330b56c0553439f0673", + "sha256:c68ea4d3ba1705da1e0d85da6684ac657912679a649e8868bd850d2c299cce13", + "sha256:e23d0cc5299223dcc37885dae624f382297717e459ea24053709675a976a3e19" ], - "version": "==3.13" + "version": "==5.1" }, "rdflib": { "hashes": [ @@ -406,44 +410,44 @@ }, "redis": { "hashes": [ - "sha256:724932360d48e5407e8f82e405ab3650a36ed02c7e460d1e6fddf0f038422b54", - "sha256:9b19425a38fd074eb5795ff2b0d9a55b46a44f91f5347995f27e3ad257a7d775" + "sha256:6946b5dca72e86103edc8033019cc3814c031232d339d5f4533b02ea85685175", + "sha256:8ca418d2ddca1b1a850afa1680a7d2fd1f3322739271de4b704e0d4668449273" ], - "version": "==3.2.0" + "version": "==3.2.1" }, "reportlab": { "hashes": [ - "sha256:069f684cd0aaa518a27dc9124aed29cee8998e21ddf19604e53214ec8462bdd7", - "sha256:09b68ec01d86b4b120456b3f3202570ec96f57624e3a4fc36f3829323391daa4", - "sha256:0c32be9a406172c29ea20ff55a709ccac1e7fb09f15aba67cb7b455fd1d3dbe0", - "sha256:233196cf25e97cfe7c452524ea29d9a4909f1cb66599299233be1efaaaa7a7a3", - "sha256:2b5e4533f3e5b962835a5ce44467e66d1ecc822761d1b508077b5087a06be338", - "sha256:2e860bcdace5a558356802a92ae8658d7e5fdaa00ded82e83a3f2987c562cb66", - "sha256:3546029e63a9a9dc24ee38959eb417678c2425b96cd27b31e09e216dafc94666", - "sha256:4452b93f9c73b6b70311e7d69082d64da81b38e91bfb4766397630092e6da6fd", - "sha256:528c74a1c6527d1859c2c7a64a94a1cba485b00175162ea23699ae58a1e94939", - "sha256:6116e750f98018febc08dfee6df20446cf954adbcfa378d2c703d56c8864aff3", - "sha256:6b2b3580c647d75ef129172cb3da648cdb24566987b0b59c5ebb80ab770748d6", - "sha256:727b5f2bed08552d143fc99649b1863c773729f580a416844f9d9967bb0a1ae8", - "sha256:74c24a3ec0a3d4f8acb13a07192f45bdb54a1cc3c2286241677e7e8bcd5011fa", - "sha256:98ccd2f8b4f8636db05f3f14db0b471ad6bb4b66ae0dc9052c4822b3bd5d6a7d", - "sha256:a5905aa567946bc938b489a7249c7890c3fd3c9b7b5680dece5bc551c2ddbe0d", - "sha256:acbb7f676b8586b770719e9683eda951fdb38eb7970d46fcbf3cdda88d912a64", - "sha256:b5e30f865add48cf880f1c363eb505b97f2f7baaa88c155f87a335a76515a3e5", - "sha256:be2a7c33a2c28bbd3f453ffe4f0e5200b88c803a097f4cf52d69c6b53fad7a8f", - "sha256:c356bb600f59ac64955813d6497a08bfd5d0c451cb5829b61e3913d0ac084e26", - "sha256:c7ec4ae2393beab584921b1287a04e94fd98c28315e348362d89b85f4b464546", - "sha256:d476edc831bb3e9ebd04d1403abaf3ea57b3e4c2276c91a54fdfb6efbd3f9d97", - "sha256:db059e1a0691c872784062421ec51848539eb4f5210142682e61059a5ca7cc55", - "sha256:dd423a6753509ab14a0ac1b5be39d219c8f8d3781cce3deb4f45eda31969b5e8", - "sha256:ed9b7c0d71ce6fe2b31c6cde530ad8238632b876a5d599218739bda142a77f7c", - "sha256:f0a2465af4006f97b05e1f1546d67d3a3213d414894bf28be7f87f550a7f4a55", - "sha256:f20bfe26e57e8e1f575a9e0325be04dd3562db9f247ffdd73b5d4df6dec53bc2", - "sha256:f3463f2cb40a1b515ac0133ba859eca58f53b56760da9abb27ed684c565f853c", - "sha256:facc3c9748ab1525fb8401a1223bce4f24f0d6aa1a9db86c55db75777ccf40f9" + "sha256:0135bc54a463db5315c93bba4182fb83dc088fefaa7da18784ecd2a0c4a9c068", + "sha256:09e167e01458ea1e0cf3acff634ae9ecc1f1757e7585060d039c90b762859cfd", + "sha256:0dfcea18ba3ca1fac55cb273d056a8a43a48bd04d419299b3267e1994c72455a", + "sha256:1a61e56593ea1a8a38135eedfb40f79dcad13164fff034313ebf2a30e200ca79", + "sha256:1bdd871c2087d3853a0e9a3a573b1a7535500f3341944b1e34e68f3213cd28b8", + "sha256:26878a4b9c45f046c635b5695681188c19806f08b04129ea01c9ed51c7754039", + "sha256:27c62264c758aa30113df105da816223d149e4e87ee778ad49469725b79be2eb", + "sha256:29a9dd3954465b9e4efb129ffda9ab3e6a4f06488e8aa2efd5aff8ad332f13c2", + "sha256:5740e3218ca98c1bc86bd2d2e2a8c1d23e7c97d949d6377ac30aaf449f01c363", + "sha256:605892bb3f822a1e7342ce2b461d645ab8e4d13875127c0ae5377f76853db422", + "sha256:6dacc72552bc0dd50286e856f09a5e646a007d9345598bf6f75b117a200bfd9d", + "sha256:7021b7c8ba6d8e69e4c68c9473067482aaa40b9094270b45dbf798fcb0e09bd4", + "sha256:8acd950dad5b20a417579d1253c1065222dde48f9412e71533b052ab3dd98632", + "sha256:8b8fb3b0dd1e2124aba24544a02c95bff1fffa966b0581f30abf4fb28e414005", + "sha256:920c61c942eb1cc446e1647a04978f4afe31993ed403b74576a018c3ca526394", + "sha256:928e8d99befe064e28e9a29a4fd9afcf2066dcd758b0903280e67e221527422a", + "sha256:a04787eee401a74c80b65e539b5fe9226fdeabe25caa3d216c21dc990b2f8a01", + "sha256:a5bb6bd7753cba854425fcf7ecf04627a17de78d47ef9e8fac615887c5658da3", + "sha256:a70d970619014dc83b4406bcfed7e2f9d5aaf5f521aad808f5560d90ea896fb4", + "sha256:ae468fe82c8af3d1987113f03c1f87d01daa5b4c85c1f10da126be84423a744d", + "sha256:b278d83a7f76410bd310b368309e6e4b19664ffa686abfa9f0696130b09c17d3", + "sha256:b6623e9a96db3edc4b384e036e67c7bc87bbd7e5dc2d72ce66efa0043f9383b0", + "sha256:dc15cfa577bb25f0a598d483cf6dcc5ecad576ba723fe9bec63b6ec720dab2a3", + "sha256:dffdb4f6b34ce791e67365f3f96ab3c45b4cdd2c70d212fac98fb146dc75ac80", + "sha256:e84020e3482856da733e1359cb7b84e6bac09179bd3af860e70468a9c3cb43e3", + "sha256:edda09668e8474d5acb1a37fb64599557b43a714f1469bd49a058e95b5b410ff", + "sha256:f77e9835873931d25f836a3c107e53e0f7d3c0b4906b13063815308cf5ca1fac", + "sha256:f91d16ff07d5d3c92303f64c6864d74d3b6a491dde186bfef90c58088f932998" ], "index": "pypi", - "version": "==3.5.13" + "version": "==3.5.17" }, "requests": { "hashes": [ @@ -469,10 +473,10 @@ }, "sigmatools": { "hashes": [ - "sha256:98c9897f27e7c99f398bff537bb6b0259599177d955f8b60a22db1b246f9cb0b" + "sha256:ae980b6d6fd466294911efa493934d24e3c5df406da4a190b9fff0943a81cc5f" ], "index": "pypi", - "version": "==0.7.1" + "version": "==0.10" }, "six": { "hashes": [ @@ -483,10 +487,10 @@ }, "soupsieve": { "hashes": [ - "sha256:afa56bf14907bb09403e5d15fbed6275caa4174d36b975226e3b67a3bb6e2c4b", - "sha256:eaed742b48b1f3e2d45ba6f79401b2ed5dc33b2123dfe216adb90d4bfa0ade26" + "sha256:3aef141566afd07201b525c17bfaadd07580a8066f82b57f7c9417f26adbd0a3", + "sha256:e41a65e99bd125972d84221022beb1e4b5cfc68fa12c170c39834ce32d1b294c" ], - "version": "==1.8" + "version": "==1.9" }, "sparqlwrapper": { "hashes": [ @@ -504,17 +508,23 @@ "index": "pypi", "version": "==1.1.0" }, + "tabulate": { + "hashes": [ + "sha256:8af07a39377cee1103a5c8b3330a421c2d99b9141e9cc5ddd2e3263fea416943" + ], + "version": "==0.8.3" + }, "tornado": { "hashes": [ - "sha256:0662d28b1ca9f67108c7e3b77afabfb9c7e87bde174fbda78186ecedc2499a9d", - "sha256:4e5158d97583502a7e2739951553cbd88a72076f152b4b11b64b9a10c4c49409", - "sha256:732e836008c708de2e89a31cb2fa6c0e5a70cb60492bee6f1ea1047500feaf7f", - "sha256:8154ec22c450df4e06b35f131adc4f2f3a12ec85981a203301d310abf580500f", - "sha256:8e9d728c4579682e837c92fdd98036bd5cdefa1da2aaf6acf26947e6dd0c01c5", - "sha256:d4b3e5329f572f055b587efc57d29bd051589fb5a43ec8898c77a47ec2fa2bbb", - "sha256:e5f2585afccbff22390cddac29849df463b252b711aa2ce7c5f3f342a5b3b444" + "sha256:1174dcb84d08887b55defb2cda1986faeeea715fff189ef3dc44cce99f5fca6b", + "sha256:2613fab506bd2aedb3722c8c64c17f8f74f4070afed6eea17f20b2115e445aec", + "sha256:44b82bc1146a24e5b9853d04c142576b4e8fa7a92f2e30bc364a85d1f75c4de2", + "sha256:457fcbee4df737d2defc181b9073758d73f54a6cfc1f280533ff48831b39f4a8", + "sha256:49603e1a6e24104961497ad0c07c799aec1caac7400a6762b687e74c8206677d", + "sha256:8c2f40b99a8153893793559919a355d7b74649a11e59f411b0b0a1793e160bc0", + "sha256:e1d897889c3b5a829426b7d52828fb37b28bc181cd598624e65c8be40ee3f7fa" ], - "version": "==5.1.1" + "version": "==6.0.2" }, "url-normalize": { "hashes": [ @@ -545,20 +555,20 @@ }, "vulners": { "hashes": [ - "sha256:40041bcf893fa1bfaf29c650369d9a249991911f28b4d8795f7bc06508013e14", - "sha256:6d00709300dcc7e2727499d8a60f51eaced1dc6b63cc19cb8a4b065b658c51aa", - "sha256:de8cef247c9852c39bd54434e63026b46bdb2bd4ca22813bf66626b7d359b0f3" + "sha256:6617d5904b5369507bc34105071d312e9e1c38d73654505e7b15b9a3f1325915", + "sha256:8b05d12a9dd7cbc07198a13281299a6e014ec348522e214b1efd097e194b7568", + "sha256:a19b02e0a112d70951e10c5abc1993f7f029234212828e1b617ab35f4e460a24" ], "index": "pypi", - "version": "==1.4.4" + "version": "==1.4.7" }, "wand": { "hashes": [ - "sha256:7d6b8dc9d4eaccc430b9c86e6b749013220c994970a3f39e902b397e2fa732c3", - "sha256:cc0b5c9cd50fecd10dc8888b739dd5984c6f8085d2954f34903b83ca39a91236" + "sha256:91810d241ab0851d40e67c946beb960b869c4f4160c397eac291ec6283ee3e3f", + "sha256:ae7c0958509a22f531b7b97e93adfd3f1208f0ac1c593af9e5f0cffa4ac06d5b" ], "index": "pypi", - "version": "==0.5.1" + "version": "==0.5.2" }, "xlsxwriter": { "hashes": [ @@ -611,17 +621,17 @@ }, "attrs": { "hashes": [ - "sha256:10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69", - "sha256:ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb" + "sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79", + "sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399" ], - "version": "==18.2.0" + "version": "==19.1.0" }, "certifi": { "hashes": [ - "sha256:47f9c83ef4c0c621eaef743f133f09fa8a74a9b75f037e8624f83bd1b6626cb7", - "sha256:993f830721089fef441cdfeb4b2c8c9df86f0c63239f06bd025a76a7daddb033" + "sha256:59b7658e26ca9c7339e00f8f4636cdfe59d34fa37b9b04f6f9e9926b3cece1a5", + "sha256:b26104d6835d1f5e49452a26eb2ff87fe7090b89dfcaee5ea2212697e1e1d7ae" ], - "version": "==2018.11.29" + "version": "==2019.3.9" }, "chardet": { "hashes": [ @@ -640,39 +650,39 @@ }, "coverage": { "hashes": [ - "sha256:09e47c529ff77bf042ecfe858fb55c3e3eb97aac2c87f0349ab5a7efd6b3939f", - "sha256:0a1f9b0eb3aa15c990c328535655847b3420231af299386cfe5efc98f9c250fe", - "sha256:0cc941b37b8c2ececfed341444a456912e740ecf515d560de58b9a76562d966d", - "sha256:10e8af18d1315de936d67775d3a814cc81d0747a1a0312d84e27ae5610e313b0", - "sha256:1b4276550b86caa60606bd3572b52769860a81a70754a54acc8ba789ce74d607", - "sha256:1e8a2627c48266c7b813975335cfdea58c706fe36f607c97d9392e61502dc79d", - "sha256:2b224052bfd801beb7478b03e8a66f3f25ea56ea488922e98903914ac9ac930b", - "sha256:447c450a093766744ab53bf1e7063ec82866f27bcb4f4c907da25ad293bba7e3", - "sha256:46101fc20c6f6568561cdd15a54018bb42980954b79aa46da8ae6f008066a30e", - "sha256:4710dc676bb4b779c4361b54eb308bc84d64a2fa3d78e5f7228921eccce5d815", - "sha256:510986f9a280cd05189b42eee2b69fecdf5bf9651d4cd315ea21d24a964a3c36", - "sha256:5535dda5739257effef56e49a1c51c71f1d37a6e5607bb25a5eee507c59580d1", - "sha256:5a7524042014642b39b1fcae85fb37556c200e64ec90824ae9ecf7b667ccfc14", - "sha256:5f55028169ef85e1fa8e4b8b1b91c0b3b0fa3297c4fb22990d46ff01d22c2d6c", - "sha256:6694d5573e7790a0e8d3d177d7a416ca5f5c150742ee703f3c18df76260de794", - "sha256:6831e1ac20ac52634da606b658b0b2712d26984999c9d93f0c6e59fe62ca741b", - "sha256:77f0d9fa5e10d03aa4528436e33423bfa3718b86c646615f04616294c935f840", - "sha256:828ad813c7cdc2e71dcf141912c685bfe4b548c0e6d9540db6418b807c345ddd", - "sha256:85a06c61598b14b015d4df233d249cd5abfa61084ef5b9f64a48e997fd829a82", - "sha256:8cb4febad0f0b26c6f62e1628f2053954ad2c555d67660f28dfb1b0496711952", - "sha256:a5c58664b23b248b16b96253880b2868fb34358911400a7ba39d7f6399935389", - "sha256:aaa0f296e503cda4bc07566f592cd7a28779d433f3a23c48082af425d6d5a78f", - "sha256:ab235d9fe64833f12d1334d29b558aacedfbca2356dfb9691f2d0d38a8a7bfb4", - "sha256:b3b0c8f660fae65eac74fbf003f3103769b90012ae7a460863010539bb7a80da", - "sha256:bab8e6d510d2ea0f1d14f12642e3f35cefa47a9b2e4c7cea1852b52bc9c49647", - "sha256:c45297bbdbc8bb79b02cf41417d63352b70bcb76f1bbb1ee7d47b3e89e42f95d", - "sha256:d19bca47c8a01b92640c614a9147b081a1974f69168ecd494687c827109e8f42", - "sha256:d64b4340a0c488a9e79b66ec9f9d77d02b99b772c8b8afd46c1294c1d39ca478", - "sha256:da969da069a82bbb5300b59161d8d7c8d423bc4ccd3b410a9b4d8932aeefc14b", - "sha256:ed02c7539705696ecb7dc9d476d861f3904a8d2b7e894bd418994920935d36bb", - "sha256:ee5b8abc35b549012e03a7b1e86c09491457dba6c94112a2482b18589cc2bdb9" + "sha256:3684fabf6b87a369017756b551cef29e505cb155ddb892a7a29277b978da88b9", + "sha256:39e088da9b284f1bd17c750ac672103779f7954ce6125fd4382134ac8d152d74", + "sha256:3c205bc11cc4fcc57b761c2da73b9b72a59f8d5ca89979afb0c1c6f9e53c7390", + "sha256:465ce53a8c0f3a7950dfb836438442f833cf6663d407f37d8c52fe7b6e56d7e8", + "sha256:48020e343fc40f72a442c8a1334284620f81295256a6b6ca6d8aa1350c763bbe", + "sha256:5296fc86ab612ec12394565c500b412a43b328b3907c0d14358950d06fd83baf", + "sha256:5f61bed2f7d9b6a9ab935150a6b23d7f84b8055524e7be7715b6513f3328138e", + "sha256:68a43a9f9f83693ce0414d17e019daee7ab3f7113a70c79a3dd4c2f704e4d741", + "sha256:6b8033d47fe22506856fe450470ccb1d8ba1ffb8463494a15cfc96392a288c09", + "sha256:7ad7536066b28863e5835e8cfeaa794b7fe352d99a8cded9f43d1161be8e9fbd", + "sha256:7bacb89ccf4bedb30b277e96e4cc68cd1369ca6841bde7b005191b54d3dd1034", + "sha256:839dc7c36501254e14331bcb98b27002aa415e4af7ea039d9009409b9d2d5420", + "sha256:8f9a95b66969cdea53ec992ecea5406c5bd99c9221f539bca1e8406b200ae98c", + "sha256:932c03d2d565f75961ba1d3cec41ddde00e162c5b46d03f7423edcb807734eab", + "sha256:988529edadc49039d205e0aa6ce049c5ccda4acb2d6c3c5c550c17e8c02c05ba", + "sha256:998d7e73548fe395eeb294495a04d38942edb66d1fa61eb70418871bc621227e", + "sha256:9de60893fb447d1e797f6bf08fdf0dbcda0c1e34c1b06c92bd3a363c0ea8c609", + "sha256:9e80d45d0c7fcee54e22771db7f1b0b126fb4a6c0a2e5afa72f66827207ff2f2", + "sha256:a545a3dfe5082dc8e8c3eb7f8a2cf4f2870902ff1860bd99b6198cfd1f9d1f49", + "sha256:a5d8f29e5ec661143621a8f4de51adfb300d7a476224156a39a392254f70687b", + "sha256:aca06bfba4759bbdb09bf52ebb15ae20268ee1f6747417837926fae990ebc41d", + "sha256:bb23b7a6fd666e551a3094ab896a57809e010059540ad20acbeec03a154224ce", + "sha256:bfd1d0ae7e292105f29d7deaa9d8f2916ed8553ab9d5f39ec65bcf5deadff3f9", + "sha256:c62ca0a38958f541a73cf86acdab020c2091631c137bd359c4f5bddde7b75fd4", + "sha256:c709d8bda72cf4cd348ccec2a4881f2c5848fd72903c185f363d361b2737f773", + "sha256:c968a6aa7e0b56ecbd28531ddf439c2ec103610d3e2bf3b75b813304f8cb7723", + "sha256:df785d8cb80539d0b55fd47183264b7002077859028dfe3070cf6359bf8b2d9c", + "sha256:f406628ca51e0ae90ae76ea8398677a921b36f0bd71aab2099dfed08abd0322f", + "sha256:f46087bbd95ebae244a0eda01a618aff11ec7a069b15a3ef8f6b520db523dcf1", + "sha256:f8019c5279eb32360ca03e9fac40a12667715546eed5c5eb59eb381f2f501260", + "sha256:fc5f4d209733750afd2714e9109816a29500718b32dd9a5db01c0cb3a019b96a" ], - "version": "==4.5.2" + "version": "==4.5.3" }, "entrypoints": { "hashes": [ @@ -705,11 +715,11 @@ }, "more-itertools": { "hashes": [ - "sha256:0125e8f60e9e031347105eb1682cef932f5e97d7b9a1a28d9bf00c22a5daef40", - "sha256:590044e3942351a1bdb1de960b739ff4ce277960f2425ad4509446dbace8d9d1" + "sha256:2112d2ca570bb7c3e53ea1a35cd5df42bb0fd10c45f0fb97178679c3c03d64c7", + "sha256:c3e4748ba1aad8dba30a4886b0b1a2004f9a863837b8654e7059eebf727afa5a" ], "markers": "python_version > '2.7'", - "version": "==6.0.0" + "version": "==7.0.0" }, "nose": { "hashes": [ @@ -743,18 +753,18 @@ }, "pyflakes": { "hashes": [ - "sha256:5e8c00e30c464c99e0b501dc160b13a14af7f27d4dffb529c556e30a159e231d", - "sha256:f277f9ca3e55de669fba45b7393a1449009cff5a37d1af10ebb76c52765269cd" + "sha256:17dbeb2e3f4d772725c777fabc446d5634d1038f234e77343108ce445ea69ce0", + "sha256:d976835886f8c5b31d47970ed689944a0262b5f3afa00a5a7b4dc81e5449f8a2" ], - "version": "==2.1.0" + "version": "==2.1.1" }, "pytest": { "hashes": [ - "sha256:067a1d4bf827ffdd56ad21bd46674703fce77c5957f6c1eef731f6146bfcef1c", - "sha256:9687049d53695ad45cf5fdc7bbd51f0c49f1ea3ecfc4b7f3fde7501b541f17f4" + "sha256:13c5e9fb5ec5179995e9357111ab089af350d788cbc944c628f3cde72285809b", + "sha256:f21d2f1fb8200830dcbb5d8ec466a9c9120e20d8b53c7585d180125cce1d297a" ], "index": "pypi", - "version": "==4.3.0" + "version": "==4.4.0" }, "requests": { "hashes": [ diff --git a/README.md b/README.md index 501e54f..5669b19 100644 --- a/README.md +++ b/README.md @@ -32,11 +32,14 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/ * [EUPI](misp_modules/modules/expansion/eupi.py) - a hover and expansion module to get information about an URL from the [Phishing Initiative project](https://phishing-initiative.eu/?lang=en). * [Farsight DNSDB Passive DNS](misp_modules/modules/expansion/farsight_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information. * [GeoIP](misp_modules/modules/expansion/geoip_country.py) - a hover and expansion module to get GeoIP information from geolite/maxmind. +* [Greynoise](misp_modules/modules/expansion/greynoise.py) - a hover to get information from greynoise. * [hashdd](misp_modules/modules/expansion/hashdd.py) - a hover module to check file hashes against [hashdd.com](http://www.hashdd.com) including NSLR dataset. +* [hibp](misp_modules/modules/expansion/hibp.py) - a hover module to lookup against Have I Been Pwned? * [intel471](misp_modules/modules/expansion/intel471.py) - an expansion module to get info from [Intel471](https://intel471.com). * [IPASN](misp_modules/modules/expansion/ipasn.py) - a hover and expansion to get the BGP ASN of an IP address. * [iprep](misp_modules/modules/expansion/iprep.py) - an expansion module to get IP reputation from packetmail.net. * [macaddress.io](misp_modules/modules/expansion/macaddress_io.py) - a hover module to retrieve vendor details and other information regarding a given MAC address or an OUI from [MAC address Vendor Lookup](https://macaddress.io). See [integration tutorial here](https://macaddress.io/integrations/MISP-module). +* [macvendors](misp_modules/modules/expansion/macvendors.py) - a hover module to retrieve mac vendor information. * [onyphe](misp_modules/modules/expansion/onyphe.py) - a modules to process queries on Onyphe. * [onyphe_full](misp_modules/modules/expansion/onyphe_full.py) - a modules to process full queries on Onyphe. * [OTX](misp_modules/modules/expansion/otx.py) - an expansion module for [OTX](https://otx.alienvault.com/). @@ -87,15 +90,13 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/ ## How to install and start MISP modules in a Python virtualenv? ~~~~bash -sudo apt-get install python3-dev python3-pip libpq5 libjpeg-dev tesseract-ocr imagemagick +sudo apt-get install python3-dev python3-pip libpq5 libjpeg-dev tesseract-ocr imagemagick virtualenv sudo -u www-data virtualenv -p python3 /var/www/MISP/venv cd /usr/local/src/ sudo git clone https://github.com/MISP/misp-modules.git cd misp-modules sudo -u www-data /var/www/MISP/venv/bin/pip install -I -r REQUIREMENTS sudo -u www-data /var/www/MISP/venv/bin/pip install . -sudo apt install ruby-pygments.rb -y -sudo gem install asciidoctor-pdf --pre sudo sed -i -e '$i \sudo -u www-data /var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s > /tmp/misp-modules_rc.local.log &\n' /etc/rc.local /var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s & #to start the modules ~~~~ @@ -109,8 +110,6 @@ sudo git clone https://github.com/MISP/misp-modules.git cd misp-modules sudo pip3 install -I -r REQUIREMENTS sudo pip3 install -I . -sudo apt install ruby-pygments.rb -y -sudo gem install asciidoctor-pdf --pre sudo sed -i -e '$i \sudo -u www-data /var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s > /tmp/misp-modules_rc.local.log &\n' /etc/rc.local /var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s & #to start the modules ~~~~ @@ -125,7 +124,6 @@ cd misp-modules scl enable rh-python36 ‘python3 –m pip install cryptography’ scl enable rh-python36 ‘python3 –m pip install -I -r REQUIREMENTS’ scl enable rh-python36 ‘python3 –m pip install –I .’ -scl enable rh-ruby22 ‘gem install asciidoctor-pdf –pre’ ~~~~ Create the service file /etc/systemd/system/misp-workers.service : ~~~~ @@ -510,14 +508,14 @@ sudo git checkout MyModBranch Remove the contents of the build directory and re-install misp-modules. -~~~python +~~~bash sudo rm -fr build/* sudo pip3 install --upgrade . ~~~ SSH in with a different terminal and run `misp-modules` with debugging enabled. -~~~python +~~~bash sudo killall misp-modules misp-modules -d ~~~ diff --git a/REQUIREMENTS b/REQUIREMENTS index 9ea9a4b..99e1c02 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -1,20 +1,20 @@ -i https://pypi.org/simple -e . --e git+https://github.com/D4-project/BGP-Ranking.git/@7e698f87366e6f99b4d0d11852737db28e3ddc62#egg=pybgpranking&subdirectory=client +-e git+https://github.com/D4-project/BGP-Ranking.git/@37c97ae252ec4bf1d67733a49d4895c8cb009cf9#egg=pybgpranking&subdirectory=client -e git+https://github.com/D4-project/IPASN-History.git/@e846cd36fe1ed6b22f60890bba89f84e61b62e59#egg=pyipasnhistory&subdirectory=client -e git+https://github.com/MISP/PyIntel471.git@0df8d51f1c1425de66714b3a5a45edb69b8cc2fc#egg=pyintel471 --e git+https://github.com/MISP/PyMISP.git@634ecc3ac308d01ebf5f5fbb9aace7746a2b8707#egg=pymisp --e git+https://github.com/Rafiot/uwhoisd.git@f6f035e52213c8abc20f2084d28cfffb399457cb#egg=uwhois&subdirectory=client +-e git+https://github.com/MISP/PyMISP.git@b8759673b91e733c307698abdc0d5ed82fd7e0de#egg=pymisp +-e git+https://github.com/Rafiot/uwhoisd.git@411572840eba4c72dc321c549b36a54ed5cea9de#egg=uwhois&subdirectory=client -e git+https://github.com/sebdraven/pydnstrails@48c1f740025c51289f43a24863d1845ff12fd21a#egg=pydnstrails --e git+https://github.com/sebdraven/pyonyphe@66329baeee7cab844f2203c047c2551828eaf14d#egg=pyonyphe +-e git+https://github.com/sebdraven/pyonyphe@cbb0168d5cb28a9f71f7ab3773164a7039ccdb12#egg=pyonyphe aiohttp==3.4.4 antlr4-python3-runtime==4.7.2 ; python_version >= '3' async-timeout==3.0.1 -attrs==18.2.0 -backscatter==0.2.3 +attrs==19.1.0 +backscatter==0.2.4 beautifulsoup4==4.7.1 blockchain==1.4.4 -certifi==2018.11.29 +certifi==2019.3.9 chardet==3.0.4 click-plugins==1.0.4 click==7.0 @@ -24,43 +24,44 @@ domaintools-api==0.3.3 enum-compat==0.0.2 ez-setup==0.9 future==0.17.1 -httplib2==0.12.0 +httplib2==0.12.1 idna-ssl==1.1.0 ; python_version < '3.7' idna==2.8 isodate==0.6.0 -jsonschema==2.6.0 +jsonschema==3.0.1 maclookup==1.0.3 multidict==4.5.2 oauth2==1.9.0.post1 passivetotal==1.0.30 pillow==5.4.1 -psutil==5.5.0 +psutil==5.6.0 pyeupi==1.0 pygeoip==0.3.2 pyparsing==2.3.1 pypdns==1.3 pypssl==2.1 +pyrsistent==0.14.11 pytesseract==0.2.6 -python-dateutil==2.7.5 +python-dateutil==2.8.0 pyyaml==3.13 rdflib==4.2.2 -redis==3.1.0 +redis==3.2.0 +reportlab==3.5.13 requests-cache==0.4.13 requests==2.21.0 -reportlab -shodan==1.10.4 -sigmatools==0.7.1 +shodan==1.11.1 +sigmatools==0.9 six==1.12.0 -soupsieve==1.7.3 +soupsieve==1.8 sparqlwrapper==1.8.2 stix2-patterns==1.1.0 -tornado==5.1.1 -# Python 3.5 only provides url-normalize v1.3.3 -url-normalize>=1.3.3 +tabulate==0.8.3 +tornado==6.0.1 +url-normalize==1.4.1 urlarchiver==0.2 urllib3==1.24.1 -vulners==1.4.0 -wand==0.5.0 -xlsxwriter==1.1.2 +vulners==1.4.5 +wand==0.5.1 +xlsxwriter==1.1.5 yara-python==3.8.1 yarl==1.3.0 diff --git a/doc/export_mod/pdfexport.json b/doc/export_mod/pdfexport.json index 9803c77..f1654dc 100644 --- a/doc/export_mod/pdfexport.json +++ b/doc/export_mod/pdfexport.json @@ -1,7 +1,7 @@ { "description": "Simple export of a MISP event to PDF.", - "requirements": ["PyMISP", "asciidoctor"], - "features": "The module takes care of the PDF file building, and work with any MISP Event. Except the requirement of asciidoctor, used to create the file, there is no special feature concerning the Event.", + "requirements": ["PyMISP", "reportlab"], + "features": "The module takes care of the PDF file building, and work with any MISP Event. Except the requirement of reportlab, used to create the file, there is no special feature concerning the Event. Some parameters can be given through the config dict. 'MISP_base_url_for_dynamic_link' is your MISP URL, to attach an hyperlink to your event on your MISP instance from the PDF. Keep it clear to avoid hyperlinks in the generated pdf.\n 'MISP_name_for_metadata' is your CERT or MISP instance name. Used as text in the PDF' metadata\n 'Activate_textual_description' is a boolean (True or void) to activate the textual description/header abstract of an event\n 'Activate_galaxy_description' is a boolean (True or void) to activate the description of event related galaxies.\n 'Activate_related_events' is a boolean (True or void) to activate the description of related event. Be aware this might leak information on confidential events linked to the current event !\n 'Activate_internationalization_fonts' is a boolean (True or void) to activate Noto fonts instead of default fonts (Helvetica). This allows the support of CJK alphabet. Be sure to have followed the procedure to download Noto fonts (~70Mo) in the right place (/tools/pdf_fonts/Noto_TTF), to allow PyMisp to find and use them during PDF generation.\n 'Custom_fonts_path' is a text (path or void) to the TTF file of your choice, to create the PDF with it. Be aware the PDF won't support bold/italic/special style anymore with this option ", "references": ["https://acrobat.adobe.com/us/en/acrobat/about-adobe-pdf.html"], "input": "MISP Event", "output": "MISP Event in a PDF file." diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py index d8fb153..42f74a3 100644 --- a/misp_modules/modules/expansion/__init__.py +++ b/misp_modules/modules/expansion/__init__.py @@ -8,4 +8,4 @@ __all__ = ['vmray_submit', 'bgpranking', 'circl_passivedns', 'circl_passivessl', 'yara_syntax_validator', 'hashdd', 'onyphe', 'onyphe_full', 'rbl', 'xforceexchange', 'sigma_syntax_validator', 'stix2_pattern_syntax_validator', 'sigma_queries', 'dbl_spamhaus', 'vulners', 'yara_query', 'macaddress_io', - 'intel471', 'backscatter_io', 'btc_scam_check'] + 'intel471', 'backscatter_io', 'btc_scam_check', 'hibp', 'greynoise', 'macvendors'] diff --git a/misp_modules/modules/expansion/btc_steroids.py b/misp_modules/modules/expansion/btc_steroids.py index 430c67d..04b7138 100755 --- a/misp_modules/modules/expansion/btc_steroids.py +++ b/misp_modules/modules/expansion/btc_steroids.py @@ -1,4 +1,3 @@ -import sys import json import requests import time @@ -91,6 +90,7 @@ def mprint(input): def handler(q=False): global result_text global conversion_rates + result_text = "" # start_time = time.time() # now = time.time() if q is False: @@ -105,7 +105,6 @@ def handler(q=False): btc = request['btc'] else: return False - mprint("\nAddress:\t" + btc) try: req = requests.get(blockchain_all.format(btc, "&limit=50")) @@ -113,8 +112,18 @@ def handler(q=False): except Exception: # print(e) print(req.text) - result_text = "" - sys.exit(1) + result_text = "Not a valid BTC address" + r = { + 'results': [ + { + 'types': ['text'], + 'values':[ + str(result_text) + ] + } + ] + } + return r n_tx = jreq['n_tx'] balance = float(jreq['final_balance'] / 100000000) diff --git a/misp_modules/modules/expansion/greynoise.py b/misp_modules/modules/expansion/greynoise.py new file mode 100644 index 0000000..d26736d --- /dev/null +++ b/misp_modules/modules/expansion/greynoise.py @@ -0,0 +1,43 @@ +import requests +import json + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['ip-dst', 'ip-src'], 'output': ['text']} +moduleinfo = {'version': '0.1', 'author': 'Aurélien Schwab ', 'description': 'Module to access GreyNoise.io API.', 'module-type': ['hover']} +moduleconfig = ['user-agent'] # TODO take this into account in the code + +greynoise_api_url = 'http://api.greynoise.io:8888/v1/query/ip' +default_user_agent = 'MISP-Module' + + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + for input_type in mispattributes['input']: + if input_type in request: + ip = request[input_type] + break + else: + misperrors['error'] = "Unsupported attributes type" + return misperrors + data = {'ip': ip} + r = requests.post(greynoise_api_url, data=data, headers={'user-agent': default_user_agent}) # Real request + if r.status_code == 200: # OK (record found) + response = json.loads(r.text) + if response: + return {'results': [{'types': mispattributes['output'], 'values': response}]} + elif r.status_code == 404: # Not found (not an error) + return {'results': [{'types': mispattributes['output'], 'values': 'No data'}]} + else: # Real error + misperrors['error'] = 'GreyNoise API not accessible (HTTP ' + str(r.status_code) + ')' + return misperrors['error'] + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/expansion/hibp.py b/misp_modules/modules/expansion/hibp.py new file mode 100644 index 0000000..8db3fa7 --- /dev/null +++ b/misp_modules/modules/expansion/hibp.py @@ -0,0 +1,43 @@ +import requests +import json + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['email-dst', 'email-src'], 'output': ['text']} # All mails as input +moduleinfo = {'version': '0.1', 'author': 'Aurélien Schwab', 'description': 'Module to access haveibeenpwned.com API.', 'module-type': ['hover']} +moduleconfig = ['user-agent'] # TODO take this into account in the code + +haveibeenpwned_api_url = 'https://api.haveibeenpwned.com/api/v2/breachedaccount/' +default_user_agent = 'MISP-Module' # User agent (must be set, requiered by API)) + + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + for input_type in mispattributes['input']: + if input_type in request: + email = request[input_type] + break + else: + misperrors['error'] = "Unsupported attributes type" + return misperrors + + r = requests.get(haveibeenpwned_api_url + email, headers={'user-agent': default_user_agent}) # Real request + if r.status_code == 200: # OK (record found) + breaches = json.loads(r.text) + if breaches: + return {'results': [{'types': mispattributes['output'], 'values': breaches}]} + elif r.status_code == 404: # Not found (not an error) + return {'results': [{'types': mispattributes['output'], 'values': 'OK (Not Found)'}]} + else: # Real error + misperrors['error'] = 'haveibeenpwned.com API not accessible (HTTP ' + str(r.status_code) + ')' + return misperrors['error'] + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/expansion/macvendors.py b/misp_modules/modules/expansion/macvendors.py new file mode 100644 index 0000000..55d0ef3 --- /dev/null +++ b/misp_modules/modules/expansion/macvendors.py @@ -0,0 +1,42 @@ +import requests +import json + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['mac-address'], 'output': ['text']} +moduleinfo = {'version': '0.1', 'author': 'Aurélien Schwab', 'description': 'Module to access Macvendors API.', 'module-type': ['hover']} +moduleconfig = ['user-agent'] # TODO take this into account in the code + +macvendors_api_url = 'https://api.macvendors.com/' +default_user_agent = 'MISP-Module' + + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + for input_type in mispattributes['input']: + if input_type in request: + mac = request[input_type] + break + else: + misperrors['error'] = "Unsupported attributes type" + return misperrors + r = requests.get(macvendors_api_url + mac, headers={'user-agent': default_user_agent}) # Real request + if r.status_code == 200: # OK (record found) + response = r.text + if response: + return {'results': [{'types': mispattributes['output'], 'values': response}]} + elif r.status_code == 404: # Not found (not an error) + return {'results': [{'types': mispattributes['output'], 'values': 'Not found'}]} + else: # Real error + misperrors['error'] = 'MacVendors API not accessible (HTTP ' + str(r.status_code) + ')' + return misperrors['error'] + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/export_mod/pdfexport.py b/misp_modules/modules/export_mod/pdfexport.py index 6b0c12f..44b3bc9 100755 --- a/misp_modules/modules/export_mod/pdfexport.py +++ b/misp_modules/modules/export_mod/pdfexport.py @@ -15,8 +15,7 @@ moduleinfo = {'version': '2', 'require_standard_format': True} # config fields that your code expects from the site admin -moduleconfig = ["MISP_base_url_for_dynamic_link", "MISP_name_for_metadata"] - +moduleconfig = ["MISP_base_url_for_dynamic_link", "MISP_name_for_metadata", "Activate_textual_description", "Activate_galaxy_description", "Activate_related_events", "Activate_internationalization_fonts", "Custom_fonts_path"] mispattributes = {} outputFileExtension = "pdf" diff --git a/setup.py b/setup.py index fc78750..55ed8b7 100644 --- a/setup.py +++ b/setup.py @@ -12,6 +12,7 @@ setup( description='MISP modules are autonomous modules that can be used for expansion and other services in MISP', packages=find_packages(), entry_points={'console_scripts': ['misp-modules = misp_modules:main']}, + scripts=['tools/update_misp_modules.sh'], test_suite="tests", classifiers=[ 'License :: OSI Approved :: GNU Affero General Public License v3', diff --git a/tests/test_expansions.py b/tests/test_expansions.py new file mode 100644 index 0000000..d581a31 --- /dev/null +++ b/tests/test_expansions.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import unittest +import requests +from urllib.parse import urljoin + + +class TestExpansions(unittest.TestCase): + + def setUp(self): + self.maxDiff = None + self.headers = {'Content-Type': 'application/json'} + self.url = "http://127.0.0.1:6666/" + + def misp_modules_post(self, query): + return requests.post(urljoin(self.url, "query"), json=query) + + def get_values(self, response): + return response.json()['results'][0]['values'] + + def test_cve(self): + query = {"module": "cve", "vulnerability": "CVE-2010-3333"} + response = self.misp_modules_post(query) + self.assertTrue(self.get_values(response).startswith("Stack-based buffer overflow in Microsoft Office XP SP3, Office 2003 SP3")) + + def test_dns(self): + query = {"module": "dns", "hostname": "www.circl.lu", "config": {"nameserver": "8.8.8.8"}} + response = self.misp_modules_post(query) + self.assertEqual(self.get_values(response), ['149.13.33.14']) + + def test_macvendors(self): + query = {"module": "macvendors", "mac-address": "FC-A1-3E-2A-1C-33"} + response = self.misp_modules_post(query) + self.assertEqual(self.get_values(response), 'Samsung Electronics Co.,Ltd') + + def test_haveibeenpwned(self): + query = {"module": "hibp", "email-src": "info@circl.lu"} + response = self.misp_modules_post(query) + self.assertEqual(self.get_values(response), 'OK (Not Found)') + + def test_greynoise(self): + query = {"module": "greynoise", "ip-dst": "1.1.1.1"} + response = self.misp_modules_post(query) + self.assertEqual(self.get_values(response)['status'], 'ok') + + def test_ipasn(self): + query = {"module": "ipasn", "ip-dst": "1.1.1.1"} + response = self.misp_modules_post(query) + key = list(self.get_values(response)['response'].keys())[0] + entry = self.get_values(response)['response'][key]['asn'] + self.assertEqual(entry, '13335') + + def test_bgpranking(self): + query = {"module": "bgpranking", "AS": "13335"} + response = self.misp_modules_post(query) + self.assertEqual(self.get_values(response)['response']['asn_description'], 'CLOUDFLARENET - Cloudflare, Inc., US') diff --git a/tools/update_misp_modules.sh b/tools/update_misp_modules.sh new file mode 100755 index 0000000..372d146 --- /dev/null +++ b/tools/update_misp_modules.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +set -e +set -x + +# Updates the MISP Modules while respecting the current permissions +# It aims to support the two following installation methods: +# * Everything is runinng on the same machine following the MISP installation guide. +# * The modules are installed using pipenv on a different machine from the one where MISP is running. + +if [ -d "/var/www/MISP" ] && [ -d "/usr/local/src/misp-modules" ] +then + echo "MISP is installed on the same machine, following the recommanded install script. Using MISP virtualenv." + PATH_TO_MISP="/var/www/MISP" + PATH_TO_MISP_MODULES="/usr/local/src/misp-modules" + + pushd ${PATH_TO_MISP_MODULES} + USER=`stat -c "%U" .` + sudo -H -u ${USER} git pull + sudo -H -u ${USER} ${PATH_TO_MISP}/venv/bin/pip install -U -r REQUIREMENTS + sudo -H -u ${USER} ${PATH_TO_MISP}/venv/bin/pip install -U -e . + + service misp-modules restart + + popd +else + if ! [ -x "$(command -v pipenv)" ]; then + echo 'Error: pipenv not available, unable to automatically update.' >&2 + exit 1 + fi + + echo "Standalone mode, use pipenv from the current directory." + git pull + pipenv install +fi + +