From 5d824a24f9f1348f78da8e6a5e7c61329db26bb8 Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Thu, 21 Feb 2019 10:06:28 +0530 Subject: [PATCH 01/59] chg: [pypi] Made sure url-normalize installs less stric. --- REQUIREMENTS | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/REQUIREMENTS b/REQUIREMENTS index 0720e90..7c8dde3 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -54,7 +54,8 @@ soupsieve==1.7.3 sparqlwrapper==1.8.2 stix2-patterns==1.1.0 tornado==5.1.1 -url-normalize==1.4.1 +# Python 3.5 only provides url-normalize v1.3.3 +url-normalize>=1.3.3 urlarchiver==0.2 urllib3==1.24.1 vulners==1.4.0 From 9611c7f2a9a1bd84081ca3bc625f731d8185c842 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Sat, 9 Mar 2019 06:15:16 +0100 Subject: [PATCH 02/59] chg: Bump Requirements --- Pipfile.lock | 74 ++++++++++++++++++++++++++-------------------------- REQUIREMENTS | 14 +++++----- 2 files changed, 44 insertions(+), 44 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index 85cd9db..36d7c3c 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -59,10 +59,10 @@ }, "attrs": { "hashes": [ - "sha256:10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69", - "sha256:ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb" + "sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79", + "sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399" ], - "version": "==18.2.0" + "version": "==19.1.0" }, "beautifulsoup4": { "hashes": [ @@ -177,10 +177,10 @@ }, "jsonschema": { "hashes": [ - "sha256:acc8a90c31d11060516cfd0b414b9f8bcf4bc691b21f0f786ea57dd5255c79db", - "sha256:dd3f8ecb1b52d94d45eedb67cb86cac57b94ded562c5d98f63719e55ce58557b" + "sha256:0c0a81564f181de3212efa2d17de1910f8732fa1b71c42266d983cd74304e20d", + "sha256:a5f6559964a3851f59040d3b961de5e68e70971afb88ba519d27e6a039efff1a" ], - "version": "==3.0.0" + "version": "==3.0.1" }, "maclookup": { "hashes": [ @@ -281,17 +281,17 @@ }, "psutil": { "hashes": [ - "sha256:5ce6b5eb0267233459f4d3980c205828482f450999b8f5b684d9629fea98782a", - "sha256:72cebfaa422b7978a1d3632b65ff734a34c6b34f4578b68a5c204d633756b810", - "sha256:77c231b4dff8c1c329a4cd1c22b96c8976c597017ff5b09993cd148d6a94500c", - "sha256:8846ab0be0cdccd6cc92ecd1246a16e2f2e49f53bd73e522c3a75ac291e1b51d", - "sha256:a013b4250ccbddc9d22feca0f986a1afc71717ad026c0f2109bbffd007351191", - "sha256:ad43b83119eeea6d5751023298cd331637e542cbd332196464799e25a5519f8f", - "sha256:c177777c787d247d02dae6c855330f9ed3e1abf8ca1744c26dd5ff968949999a", - "sha256:ec1ef313530a9457e48d25e3fdb1723dfa636008bf1b970027462d46f2555d59", - "sha256:ef3e5e02b3c5d1df366abe7b4820400d5c427579668ad4465ff189d28ded5ebd" + "sha256:1020a37214c4138e34962881372b40f390582b5c8245680c04349c2afb785a25", + "sha256:151c9858c268a1523e16fab33e3bc3bae8a0e57b57cf7fcad85fb409cbac6baf", + "sha256:1c8e6444ca1cee9a60a1a35913b8409722f7474616e0e21004e4ffadba59964b", + "sha256:722dc0dcce5272f3c5c41609fdc2c8f0ee3f976550c2d2f2057e26ba760be9c0", + "sha256:86f61a1438c026c980a4c3e2dd88a5774a3a0f00d6d0954d6c5cf8d1921b804e", + "sha256:c4a2f42abee709ed97b4498c21aa608ac31fc1f7cc8aa60ebdcd3c80757a038d", + "sha256:d9cdc2e82aeb82200fff3640f375fac39d88b1bed27ce08377cd7fb0e3621cb7", + "sha256:da6676a484adec2fdd3e1ce1b70799881ffcb958e40208dd4c5beba0011f3589", + "sha256:dca71c08335fbfc6929438fe3a502f169ba96dd20e50b3544053d6be5cb19d82" ], - "version": "==5.5.1" + "version": "==5.6.0" }, "pybgpranking": { "editable": true, @@ -333,7 +333,7 @@ "pymisp": { "editable": true, "git": "https://github.com/MISP/PyMISP.git", - "ref": "62e047f3c1972e21aa36a8882bebf4488cdc1f84" + "ref": "b8759673b91e733c307698abdc0d5ed82fd7e0de" }, "pyonyphe": { "editable": true, @@ -469,10 +469,10 @@ }, "sigmatools": { "hashes": [ - "sha256:98c9897f27e7c99f398bff537bb6b0259599177d955f8b60a22db1b246f9cb0b" + "sha256:3bdbd2ee99c32f245e948d6b882219729ab379685dd7366e4d6149c390e08170" ], "index": "pypi", - "version": "==0.7.1" + "version": "==0.9" }, "six": { "hashes": [ @@ -506,15 +506,15 @@ }, "tornado": { "hashes": [ - "sha256:0662d28b1ca9f67108c7e3b77afabfb9c7e87bde174fbda78186ecedc2499a9d", - "sha256:4e5158d97583502a7e2739951553cbd88a72076f152b4b11b64b9a10c4c49409", - "sha256:732e836008c708de2e89a31cb2fa6c0e5a70cb60492bee6f1ea1047500feaf7f", - "sha256:8154ec22c450df4e06b35f131adc4f2f3a12ec85981a203301d310abf580500f", - "sha256:8e9d728c4579682e837c92fdd98036bd5cdefa1da2aaf6acf26947e6dd0c01c5", - "sha256:d4b3e5329f572f055b587efc57d29bd051589fb5a43ec8898c77a47ec2fa2bbb", - "sha256:e5f2585afccbff22390cddac29849df463b252b711aa2ce7c5f3f342a5b3b444" + "sha256:1a58f2d603476d5e462f7c28ca1dbb5ac7e51348b27a9cac849cdec3471101f8", + "sha256:33f93243cd46dd398e5d2bbdd75539564d1f13f25d704cfc7541db74066d6695", + "sha256:34e59401afcecf0381a28228daad8ed3275bcb726810654612d5e9c001f421b7", + "sha256:35817031611d2c296c69e5023ea1f9b5720be803e3bb119464bb2a0405d5cd70", + "sha256:666b335cef5cc2759c21b7394cff881f71559aaf7cb8c4458af5bb6cb7275b47", + "sha256:81203efb26debaaef7158187af45bc440796de9fb1df12a75b65fae11600a255", + "sha256:de274c65f45f6656c375cdf1759dbf0bc52902a1e999d12a35eb13020a641a53" ], - "version": "==5.1.1" + "version": "==6.0.1" }, "url-normalize": { "hashes": [ @@ -545,12 +545,12 @@ }, "vulners": { "hashes": [ - "sha256:40041bcf893fa1bfaf29c650369d9a249991911f28b4d8795f7bc06508013e14", - "sha256:6d00709300dcc7e2727499d8a60f51eaced1dc6b63cc19cb8a4b065b658c51aa", - "sha256:de8cef247c9852c39bd54434e63026b46bdb2bd4ca22813bf66626b7d359b0f3" + "sha256:08a7ccb2b210d45143354c6161c73fe209dc14fae8692e8b793b36b79330ad11", + "sha256:bfe2478cc11c69ba7e436d7a5df925e227565782c0bd603929fb3d612c73d78d", + "sha256:d035f6a883625878a1dc377830d17d9702ef138ca31569ac01cb8686874f89cd" ], "index": "pypi", - "version": "==1.4.4" + "version": "==1.4.5" }, "wand": { "hashes": [ @@ -611,10 +611,10 @@ }, "attrs": { "hashes": [ - "sha256:10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69", - "sha256:ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb" + "sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79", + "sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399" ], - "version": "==18.2.0" + "version": "==19.1.0" }, "certifi": { "hashes": [ @@ -743,10 +743,10 @@ }, "pyflakes": { "hashes": [ - "sha256:5e8c00e30c464c99e0b501dc160b13a14af7f27d4dffb529c556e30a159e231d", - "sha256:f277f9ca3e55de669fba45b7393a1449009cff5a37d1af10ebb76c52765269cd" + "sha256:17dbeb2e3f4d772725c777fabc446d5634d1038f234e77343108ce445ea69ce0", + "sha256:d976835886f8c5b31d47970ed689944a0262b5f3afa00a5a7b4dc81e5449f8a2" ], - "version": "==2.1.0" + "version": "==2.1.1" }, "pytest": { "hashes": [ diff --git a/REQUIREMENTS b/REQUIREMENTS index 4709747..d672411 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -3,14 +3,14 @@ -e git+https://github.com/D4-project/BGP-Ranking.git/@37c97ae252ec4bf1d67733a49d4895c8cb009cf9#egg=pybgpranking&subdirectory=client -e git+https://github.com/D4-project/IPASN-History.git/@e846cd36fe1ed6b22f60890bba89f84e61b62e59#egg=pyipasnhistory&subdirectory=client -e git+https://github.com/MISP/PyIntel471.git@0df8d51f1c1425de66714b3a5a45edb69b8cc2fc#egg=pyintel471 --e git+https://github.com/MISP/PyMISP.git@62e047f3c1972e21aa36a8882bebf4488cdc1f84#egg=pymisp +-e git+https://github.com/MISP/PyMISP.git@b8759673b91e733c307698abdc0d5ed82fd7e0de#egg=pymisp -e git+https://github.com/Rafiot/uwhoisd.git@411572840eba4c72dc321c549b36a54ed5cea9de#egg=uwhois&subdirectory=client -e git+https://github.com/sebdraven/pydnstrails@48c1f740025c51289f43a24863d1845ff12fd21a#egg=pydnstrails -e git+https://github.com/sebdraven/pyonyphe@cbb0168d5cb28a9f71f7ab3773164a7039ccdb12#egg=pyonyphe aiohttp==3.4.4 antlr4-python3-runtime==4.7.2 ; python_version >= '3' async-timeout==3.0.1 -attrs==18.2.0 +attrs==19.1.0 beautifulsoup4==4.7.1 blockchain==1.4.4 certifi==2018.11.29 @@ -27,13 +27,13 @@ httplib2==0.12.1 idna-ssl==1.1.0 ; python_version < '3.7' idna==2.8 isodate==0.6.0 -jsonschema==3.0.0 +jsonschema==3.0.1 maclookup==1.0.3 multidict==4.5.2 oauth2==1.9.0.post1 passivetotal==1.0.30 pillow==5.4.1 -psutil==5.5.1 +psutil==5.6.0 pyeupi==1.0 pygeoip==0.3.2 pyparsing==2.3.1 @@ -49,16 +49,16 @@ reportlab==3.5.13 requests-cache==0.4.13 requests==2.21.0 shodan==1.11.1 -sigmatools==0.7.1 +sigmatools==0.9 six==1.12.0 soupsieve==1.8 sparqlwrapper==1.8.2 stix2-patterns==1.1.0 -tornado==5.1.1 +tornado==6.0.1 url-normalize==1.4.1 urlarchiver==0.2 urllib3==1.24.1 -vulners==1.4.4 +vulners==1.4.5 wand==0.5.1 xlsxwriter==1.1.5 yara-python==3.8.1 From c4ced9dfbf49ec605f230b35564e17a94bcea6d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Sat, 9 Mar 2019 06:40:23 +0100 Subject: [PATCH 03/59] fix: Tornado expects a KILL now. --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index b574d4c..e8fea8e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,14 +19,14 @@ script: - pid=$! - sleep 5 - pipenv run nosetests --with-coverage --cover-package=misp_modules - - kill -s INT $pid + - kill -s KILL $pid - pushd ~/ - pipenv run coverage run -m --parallel-mode --source=misp_modules misp_modules.__init__ -s -l 127.0.0.1 & - pid=$! - popd - sleep 5 - pipenv run nosetests --with-coverage --cover-package=misp_modules - - kill -s INT $pid + - kill -s KILL $pid - pipenv run flake8 --ignore=E501,W503 misp_modules after_success: From 4b77cb5055e0a114c5bf361cadefc4aacbe0ae39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Sun, 10 Mar 2019 21:17:30 +0100 Subject: [PATCH 04/59] new: Add missing dependency (backscatter) --- Pipfile | 1 + Pipfile.lock | 92 ++++++++++++++++++++++++++++++---------------------- REQUIREMENTS | 4 ++- 3 files changed, 57 insertions(+), 40 deletions(-) diff --git a/Pipfile b/Pipfile index 45c05f5..2f2d172 100644 --- a/Pipfile +++ b/Pipfile @@ -41,6 +41,7 @@ domaintools_api = "*" misp-modules = {editable = true,path = "."} pybgpranking = {editable = true,git = "https://github.com/D4-project/BGP-Ranking.git/",subdirectory = "client"} pyipasnhistory = {editable = true,git = "https://github.com/D4-project/IPASN-History.git/",subdirectory = "client"} +backscatter = "*" [requires] python_version = "3.6" diff --git a/Pipfile.lock b/Pipfile.lock index 36d7c3c..3c902e7 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "d0cd64bfe7702365d3ea66d1f51a1ec8592df2490899e7e163fe38f97172561e" + "sha256": "23dec0fa6400c828e294ea9981b433903c17358ca61d7abdaec8df5a1c89f08c" }, "pipfile-spec": 6, "requires": { @@ -64,6 +64,14 @@ ], "version": "==19.1.0" }, + "backscatter": { + "hashes": [ + "sha256:7a0d1aa3661635de81e2a09b15d53e35cbe399a111cc58a70925f80e6874abd3", + "sha256:afb0efcf5d2551dac953ec4c38fb710b274b8e811775650e02c1ef42cafb14c8" + ], + "index": "pypi", + "version": "==0.2.4" + }, "beautifulsoup4": { "hashes": [ "sha256:034740f6cb549b4e932ae1ab975581e6103ac8f942200a0e9759065984391858", @@ -82,10 +90,10 @@ }, "certifi": { "hashes": [ - "sha256:47f9c83ef4c0c621eaef743f133f09fa8a74a9b75f037e8624f83bd1b6626cb7", - "sha256:993f830721089fef441cdfeb4b2c8c9df86f0c63239f06bd025a76a7daddb033" + "sha256:59b7658e26ca9c7339e00f8f4636cdfe59d34fa37b9b04f6f9e9926b3cece1a5", + "sha256:b26104d6835d1f5e49452a26eb2ff87fe7090b89dfcaee5ea2212697e1e1d7ae" ], - "version": "==2018.11.29" + "version": "==2019.3.9" }, "chardet": { "hashes": [ @@ -504,6 +512,12 @@ "index": "pypi", "version": "==1.1.0" }, + "tabulate": { + "hashes": [ + "sha256:8af07a39377cee1103a5c8b3330a421c2d99b9141e9cc5ddd2e3263fea416943" + ], + "version": "==0.8.3" + }, "tornado": { "hashes": [ "sha256:1a58f2d603476d5e462f7c28ca1dbb5ac7e51348b27a9cac849cdec3471101f8", @@ -618,10 +632,10 @@ }, "certifi": { "hashes": [ - "sha256:47f9c83ef4c0c621eaef743f133f09fa8a74a9b75f037e8624f83bd1b6626cb7", - "sha256:993f830721089fef441cdfeb4b2c8c9df86f0c63239f06bd025a76a7daddb033" + "sha256:59b7658e26ca9c7339e00f8f4636cdfe59d34fa37b9b04f6f9e9926b3cece1a5", + "sha256:b26104d6835d1f5e49452a26eb2ff87fe7090b89dfcaee5ea2212697e1e1d7ae" ], - "version": "==2018.11.29" + "version": "==2019.3.9" }, "chardet": { "hashes": [ @@ -640,39 +654,39 @@ }, "coverage": { "hashes": [ - "sha256:09e47c529ff77bf042ecfe858fb55c3e3eb97aac2c87f0349ab5a7efd6b3939f", - "sha256:0a1f9b0eb3aa15c990c328535655847b3420231af299386cfe5efc98f9c250fe", - "sha256:0cc941b37b8c2ececfed341444a456912e740ecf515d560de58b9a76562d966d", - "sha256:10e8af18d1315de936d67775d3a814cc81d0747a1a0312d84e27ae5610e313b0", - "sha256:1b4276550b86caa60606bd3572b52769860a81a70754a54acc8ba789ce74d607", - "sha256:1e8a2627c48266c7b813975335cfdea58c706fe36f607c97d9392e61502dc79d", - "sha256:2b224052bfd801beb7478b03e8a66f3f25ea56ea488922e98903914ac9ac930b", - "sha256:447c450a093766744ab53bf1e7063ec82866f27bcb4f4c907da25ad293bba7e3", - "sha256:46101fc20c6f6568561cdd15a54018bb42980954b79aa46da8ae6f008066a30e", - "sha256:4710dc676bb4b779c4361b54eb308bc84d64a2fa3d78e5f7228921eccce5d815", - "sha256:510986f9a280cd05189b42eee2b69fecdf5bf9651d4cd315ea21d24a964a3c36", - "sha256:5535dda5739257effef56e49a1c51c71f1d37a6e5607bb25a5eee507c59580d1", - "sha256:5a7524042014642b39b1fcae85fb37556c200e64ec90824ae9ecf7b667ccfc14", - "sha256:5f55028169ef85e1fa8e4b8b1b91c0b3b0fa3297c4fb22990d46ff01d22c2d6c", - "sha256:6694d5573e7790a0e8d3d177d7a416ca5f5c150742ee703f3c18df76260de794", - "sha256:6831e1ac20ac52634da606b658b0b2712d26984999c9d93f0c6e59fe62ca741b", - "sha256:77f0d9fa5e10d03aa4528436e33423bfa3718b86c646615f04616294c935f840", - "sha256:828ad813c7cdc2e71dcf141912c685bfe4b548c0e6d9540db6418b807c345ddd", - "sha256:85a06c61598b14b015d4df233d249cd5abfa61084ef5b9f64a48e997fd829a82", - "sha256:8cb4febad0f0b26c6f62e1628f2053954ad2c555d67660f28dfb1b0496711952", - "sha256:a5c58664b23b248b16b96253880b2868fb34358911400a7ba39d7f6399935389", - "sha256:aaa0f296e503cda4bc07566f592cd7a28779d433f3a23c48082af425d6d5a78f", - "sha256:ab235d9fe64833f12d1334d29b558aacedfbca2356dfb9691f2d0d38a8a7bfb4", - "sha256:b3b0c8f660fae65eac74fbf003f3103769b90012ae7a460863010539bb7a80da", - "sha256:bab8e6d510d2ea0f1d14f12642e3f35cefa47a9b2e4c7cea1852b52bc9c49647", - "sha256:c45297bbdbc8bb79b02cf41417d63352b70bcb76f1bbb1ee7d47b3e89e42f95d", - "sha256:d19bca47c8a01b92640c614a9147b081a1974f69168ecd494687c827109e8f42", - "sha256:d64b4340a0c488a9e79b66ec9f9d77d02b99b772c8b8afd46c1294c1d39ca478", - "sha256:da969da069a82bbb5300b59161d8d7c8d423bc4ccd3b410a9b4d8932aeefc14b", - "sha256:ed02c7539705696ecb7dc9d476d861f3904a8d2b7e894bd418994920935d36bb", - "sha256:ee5b8abc35b549012e03a7b1e86c09491457dba6c94112a2482b18589cc2bdb9" + "sha256:3684fabf6b87a369017756b551cef29e505cb155ddb892a7a29277b978da88b9", + "sha256:39e088da9b284f1bd17c750ac672103779f7954ce6125fd4382134ac8d152d74", + "sha256:3c205bc11cc4fcc57b761c2da73b9b72a59f8d5ca89979afb0c1c6f9e53c7390", + "sha256:465ce53a8c0f3a7950dfb836438442f833cf6663d407f37d8c52fe7b6e56d7e8", + "sha256:48020e343fc40f72a442c8a1334284620f81295256a6b6ca6d8aa1350c763bbe", + "sha256:5296fc86ab612ec12394565c500b412a43b328b3907c0d14358950d06fd83baf", + "sha256:5f61bed2f7d9b6a9ab935150a6b23d7f84b8055524e7be7715b6513f3328138e", + "sha256:68a43a9f9f83693ce0414d17e019daee7ab3f7113a70c79a3dd4c2f704e4d741", + "sha256:6b8033d47fe22506856fe450470ccb1d8ba1ffb8463494a15cfc96392a288c09", + "sha256:7ad7536066b28863e5835e8cfeaa794b7fe352d99a8cded9f43d1161be8e9fbd", + "sha256:7bacb89ccf4bedb30b277e96e4cc68cd1369ca6841bde7b005191b54d3dd1034", + "sha256:839dc7c36501254e14331bcb98b27002aa415e4af7ea039d9009409b9d2d5420", + "sha256:8f9a95b66969cdea53ec992ecea5406c5bd99c9221f539bca1e8406b200ae98c", + "sha256:932c03d2d565f75961ba1d3cec41ddde00e162c5b46d03f7423edcb807734eab", + "sha256:988529edadc49039d205e0aa6ce049c5ccda4acb2d6c3c5c550c17e8c02c05ba", + "sha256:998d7e73548fe395eeb294495a04d38942edb66d1fa61eb70418871bc621227e", + "sha256:9de60893fb447d1e797f6bf08fdf0dbcda0c1e34c1b06c92bd3a363c0ea8c609", + "sha256:9e80d45d0c7fcee54e22771db7f1b0b126fb4a6c0a2e5afa72f66827207ff2f2", + "sha256:a545a3dfe5082dc8e8c3eb7f8a2cf4f2870902ff1860bd99b6198cfd1f9d1f49", + "sha256:a5d8f29e5ec661143621a8f4de51adfb300d7a476224156a39a392254f70687b", + "sha256:aca06bfba4759bbdb09bf52ebb15ae20268ee1f6747417837926fae990ebc41d", + "sha256:bb23b7a6fd666e551a3094ab896a57809e010059540ad20acbeec03a154224ce", + "sha256:bfd1d0ae7e292105f29d7deaa9d8f2916ed8553ab9d5f39ec65bcf5deadff3f9", + "sha256:c62ca0a38958f541a73cf86acdab020c2091631c137bd359c4f5bddde7b75fd4", + "sha256:c709d8bda72cf4cd348ccec2a4881f2c5848fd72903c185f363d361b2737f773", + "sha256:c968a6aa7e0b56ecbd28531ddf439c2ec103610d3e2bf3b75b813304f8cb7723", + "sha256:df785d8cb80539d0b55fd47183264b7002077859028dfe3070cf6359bf8b2d9c", + "sha256:f406628ca51e0ae90ae76ea8398677a921b36f0bd71aab2099dfed08abd0322f", + "sha256:f46087bbd95ebae244a0eda01a618aff11ec7a069b15a3ef8f6b520db523dcf1", + "sha256:f8019c5279eb32360ca03e9fac40a12667715546eed5c5eb59eb381f2f501260", + "sha256:fc5f4d209733750afd2714e9109816a29500718b32dd9a5db01c0cb3a019b96a" ], - "version": "==4.5.2" + "version": "==4.5.3" }, "entrypoints": { "hashes": [ diff --git a/REQUIREMENTS b/REQUIREMENTS index d672411..99e1c02 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -11,9 +11,10 @@ aiohttp==3.4.4 antlr4-python3-runtime==4.7.2 ; python_version >= '3' async-timeout==3.0.1 attrs==19.1.0 +backscatter==0.2.4 beautifulsoup4==4.7.1 blockchain==1.4.4 -certifi==2018.11.29 +certifi==2019.3.9 chardet==3.0.4 click-plugins==1.0.4 click==7.0 @@ -54,6 +55,7 @@ six==1.12.0 soupsieve==1.8 sparqlwrapper==1.8.2 stix2-patterns==1.1.0 +tabulate==0.8.3 tornado==6.0.1 url-normalize==1.4.1 urlarchiver==0.2 From eb2dcca12b853ead10856b7de15b4d0fff4209f2 Mon Sep 17 00:00:00 2001 From: Sascha Rommelfangen Date: Thu, 14 Mar 2019 14:39:58 +0100 Subject: [PATCH 05/59] fixed a bug when checking malformed BTC addresses --- misp_modules/modules/expansion/btc_steroids.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/misp_modules/modules/expansion/btc_steroids.py b/misp_modules/modules/expansion/btc_steroids.py index 7011eda..5958502 100755 --- a/misp_modules/modules/expansion/btc_steroids.py +++ b/misp_modules/modules/expansion/btc_steroids.py @@ -91,6 +91,7 @@ def mprint(input): def handler(q=False): global result_text global conversion_rates + result_text = "" # start_time = time.time() # now = time.time() if q is False: @@ -105,7 +106,6 @@ def handler(q=False): btc = request['btc'] else: return False - mprint("\nAddress:\t" + btc) try: req = requests.get(blockchain_all.format(btc, "&limit=50")) @@ -113,8 +113,18 @@ def handler(q=False): except Exception: # print(e) print(req.text) - result_text = "" - sys.exit(1) + result_text = "Not a valid BTC address" + r = { + 'results': [ + { + 'types': ['text'], + 'values':[ + str(result_text) + ] + } + ] + } + return r n_tx = jreq['n_tx'] balance = float(jreq['final_balance'] / 100000000) From 97818e17d009f1c20806476aff8d10becb36f30e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D4=9C=D0=B5=D1=95?= <5124946+wesinator@users.noreply.github.com> Date: Thu, 14 Mar 2019 13:28:22 -0400 Subject: [PATCH 06/59] Fix command highlighting --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 951de64..e890433 100644 --- a/README.md +++ b/README.md @@ -505,14 +505,14 @@ sudo git checkout MyModBranch Remove the contents of the build directory and re-install misp-modules. -~~~python +~~~bash sudo rm -fr build/* sudo pip3 install --upgrade . ~~~ SSH in with a different terminal and run `misp-modules` with debugging enabled. -~~~python +~~~bash sudo killall misp-modules misp-modules -d ~~~ From 1c0984eaec5145643933779601390e811fb9084d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 15 Mar 2019 11:06:11 +0100 Subject: [PATCH 07/59] fix: Remove unused import --- misp_modules/modules/expansion/btc_steroids.py | 1 - 1 file changed, 1 deletion(-) diff --git a/misp_modules/modules/expansion/btc_steroids.py b/misp_modules/modules/expansion/btc_steroids.py index 2f3bebf..04b7138 100755 --- a/misp_modules/modules/expansion/btc_steroids.py +++ b/misp_modules/modules/expansion/btc_steroids.py @@ -1,4 +1,3 @@ -import sys import json import requests import time From 3e34f38cac78d6e495be8dcf4eee4d4b7dcae181 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 15 Mar 2019 11:12:19 +0100 Subject: [PATCH 08/59] chg: Bump dependencies. --- Pipfile.lock | 56 ++++++++++++++++++++++++++-------------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index 3c902e7..60f1d03 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -289,22 +289,22 @@ }, "psutil": { "hashes": [ - "sha256:1020a37214c4138e34962881372b40f390582b5c8245680c04349c2afb785a25", - "sha256:151c9858c268a1523e16fab33e3bc3bae8a0e57b57cf7fcad85fb409cbac6baf", - "sha256:1c8e6444ca1cee9a60a1a35913b8409722f7474616e0e21004e4ffadba59964b", - "sha256:722dc0dcce5272f3c5c41609fdc2c8f0ee3f976550c2d2f2057e26ba760be9c0", - "sha256:86f61a1438c026c980a4c3e2dd88a5774a3a0f00d6d0954d6c5cf8d1921b804e", - "sha256:c4a2f42abee709ed97b4498c21aa608ac31fc1f7cc8aa60ebdcd3c80757a038d", - "sha256:d9cdc2e82aeb82200fff3640f375fac39d88b1bed27ce08377cd7fb0e3621cb7", - "sha256:da6676a484adec2fdd3e1ce1b70799881ffcb958e40208dd4c5beba0011f3589", - "sha256:dca71c08335fbfc6929438fe3a502f169ba96dd20e50b3544053d6be5cb19d82" + "sha256:23e9cd90db94fbced5151eaaf9033ae9667c033dffe9e709da761c20138d25b6", + "sha256:27858d688a58cbfdd4434e1c40f6c79eb5014b709e725c180488ccdf2f721729", + "sha256:354601a1d1a1322ae5920ba397c58d06c29728a15113598d1a8158647aaa5385", + "sha256:9c3a768486194b4592c7ae9374faa55b37b9877fd9746fb4028cb0ac38fd4c60", + "sha256:c1fd45931889dc1812ba61a517630d126f6185f688eac1693171c6524901b7de", + "sha256:d463a142298112426ebd57351b45c39adb41341b91f033aa903fa4c6f76abecc", + "sha256:e1494d20ffe7891d07d8cb9a8b306c1a38d48b13575265d090fc08910c56d474", + "sha256:ec4b4b638b84d42fc48139f9352f6c6587ee1018d55253542ee28db7480cc653", + "sha256:fa0a570e0a30b9dd618bffbece590ae15726b47f9f1eaf7518dfb35f4d7dcd21" ], - "version": "==5.6.0" + "version": "==5.6.1" }, "pybgpranking": { "editable": true, "git": "https://github.com/D4-project/BGP-Ranking.git/", - "ref": "37c97ae252ec4bf1d67733a49d4895c8cb009cf9", + "ref": "4f2898af7c4e237b6497831d5acf3f4531ac14d8", "subdirectory": "client" }, "pydnstrails": { @@ -335,13 +335,13 @@ "pyipasnhistory": { "editable": true, "git": "https://github.com/D4-project/IPASN-History.git/", - "ref": "e846cd36fe1ed6b22f60890bba89f84e61b62e59", + "ref": "7ef09cf761fc58aa774ea305a33ba75959e39887", "subdirectory": "client" }, "pymisp": { "editable": true, "git": "https://github.com/MISP/PyMISP.git", - "ref": "b8759673b91e733c307698abdc0d5ed82fd7e0de" + "ref": "1dddfd72e08886673e57e23627064f9ea8303d4c" }, "pyonyphe": { "editable": true, @@ -391,19 +391,19 @@ }, "pyyaml": { "hashes": [ - "sha256:3d7da3009c0f3e783b2c873687652d83b1bbfd5c88e9813fb7e5b03c0dd3108b", - "sha256:3ef3092145e9b70e3ddd2c7ad59bdd0252a94dfe3949721633e41344de00a6bf", - "sha256:40c71b8e076d0550b2e6380bada1f1cd1017b882f7e16f09a65be98e017f211a", - "sha256:558dd60b890ba8fd982e05941927a3911dc409a63dcb8b634feaa0cda69330d3", - "sha256:a7c28b45d9f99102fa092bb213aa12e0aaf9a6a1f5e395d36166639c1f96c3a1", - "sha256:aa7dd4a6a427aed7df6fb7f08a580d68d9b118d90310374716ae90b710280af1", - "sha256:bc558586e6045763782014934bfaf39d48b8ae85a2713117d16c39864085c613", - "sha256:d46d7982b62e0729ad0175a9bc7e10a566fc07b224d2c79fafb5e032727eaa04", - "sha256:d5eef459e30b09f5a098b9cea68bebfeb268697f78d647bd255a085371ac7f3f", - "sha256:e01d3203230e1786cd91ccfdc8f8454c8069c91bee3962ad93b87a4b2860f537", - "sha256:e170a9e6fcfd19021dd29845af83bb79236068bf5fd4df3327c1be18182b2531" + "sha256:1adecc22f88d38052fb787d959f003811ca858b799590a5eaa70e63dca50308c", + "sha256:436bc774ecf7c103814098159fbb84c2715d25980175292c648f2da143909f95", + "sha256:460a5a4248763f6f37ea225d19d5c205677d8d525f6a83357ca622ed541830c2", + "sha256:5a22a9c84653debfbf198d02fe592c176ea548cccce47553f35f466e15cf2fd4", + "sha256:7a5d3f26b89d688db27822343dfa25c599627bc92093e788956372285c6298ad", + "sha256:9372b04a02080752d9e6f990179a4ab840227c6e2ce15b95e1278456664cf2ba", + "sha256:a5dcbebee834eaddf3fa7366316b880ff4062e4bcc9787b78c7fbb4a26ff2dd1", + "sha256:aee5bab92a176e7cd034e57f46e9df9a9862a71f8f37cad167c6fc74c65f5b4e", + "sha256:c51f642898c0bacd335fc119da60baae0824f2cde95b0330b56c0553439f0673", + "sha256:c68ea4d3ba1705da1e0d85da6684ac657912679a649e8868bd850d2c299cce13", + "sha256:e23d0cc5299223dcc37885dae624f382297717e459ea24053709675a976a3e19" ], - "version": "==3.13" + "version": "==5.1" }, "rdflib": { "hashes": [ @@ -764,11 +764,11 @@ }, "pytest": { "hashes": [ - "sha256:067a1d4bf827ffdd56ad21bd46674703fce77c5957f6c1eef731f6146bfcef1c", - "sha256:9687049d53695ad45cf5fdc7bbd51f0c49f1ea3ecfc4b7f3fde7501b541f17f4" + "sha256:592eaa2c33fae68c7d75aacf042efc9f77b27c08a6224a4f59beab8d9a420523", + "sha256:ad3ad5c450284819ecde191a654c09b0ec72257a2c711b9633d677c71c9850c4" ], "index": "pypi", - "version": "==4.3.0" + "version": "==4.3.1" }, "requests": { "hashes": [ From b89d068c041516f020877414bb18b0767fe43d97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 2 Apr 2019 15:30:11 +0200 Subject: [PATCH 09/59] new: Modules for greynoise, haveibeenpwned and macvendors Source: https://github.com/src7/misp-modules --- misp_modules/modules/expansion/greynoise.py | 40 ++++++++++++++ misp_modules/modules/expansion/hibp.py | 40 ++++++++++++++ misp_modules/modules/expansion/macvendors.py | 42 +++++++++++++++ tests/test_expansions.py | 57 ++++++++++++++++++++ 4 files changed, 179 insertions(+) create mode 100644 misp_modules/modules/expansion/greynoise.py create mode 100644 misp_modules/modules/expansion/hibp.py create mode 100644 misp_modules/modules/expansion/macvendors.py create mode 100644 tests/test_expansions.py diff --git a/misp_modules/modules/expansion/greynoise.py b/misp_modules/modules/expansion/greynoise.py new file mode 100644 index 0000000..324423d --- /dev/null +++ b/misp_modules/modules/expansion/greynoise.py @@ -0,0 +1,40 @@ +import requests +import json + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['ip-dst', 'ip-src'], 'output': ['text']} +moduleinfo = {'version': '0.1', 'author': 'Aurélien Schwab ', 'description': 'Module to access GreyNoise.io API.', 'module-type': ['hover']} +moduleconfig = ['user-agent']#TODO take this into account in the code + +greynoise_api_url = 'http://api.greynoise.io:8888/v1/query/ip' +default_user_agent = 'MISP-Module' + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + for input_type in mispattributes['input']: + if input_type in request: + ip = request[input_type] + break + else: + misperrors['error'] = "Unsupported attributes type" + return misperrors + data = {'ip':ip} + r = requests.post(greynoise_api_url, data=data, headers={'user-agent': default_user_agent})#Real request + if r.status_code == 200:#OK (record found) + response = json.loads(r.text) + if response: + return {'results': [{'types': mispattributes['output'], 'values': response}]} + elif r.status_code == 404:#Not found (not an error) + return {'results': [{'types': mispattributes['output'], 'values': 'No data'}]} + else:#Real error + misperrors['error'] = 'GreyNoise API not accessible (HTTP ' + str(r.status_code) + ')' + return misperrors['error'] + +def introspection(): + return mispattributes + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinf diff --git a/misp_modules/modules/expansion/hibp.py b/misp_modules/modules/expansion/hibp.py new file mode 100644 index 0000000..7010aee --- /dev/null +++ b/misp_modules/modules/expansion/hibp.py @@ -0,0 +1,40 @@ +import requests +import json + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['email-dst', 'email-src'], 'output': ['text']}#All mails as input +moduleinfo = {'version': '0.1', 'author': 'Aurélien Schwab', 'description': 'Module to access haveibeenpwned.com API.', 'module-type': ['hover']} +moduleconfig = ['user-agent']#TODO take this into account in the code + +haveibeenpwned_api_url = 'https://api.haveibeenpwned.com/api/v2/breachedaccount/' +default_user_agent = 'MISP-Module'#User agent (must be set, requiered by API)) + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + for input_type in mispattributes['input']: + if input_type in request: + email = request[input_type] + break + else: + misperrors['error'] = "Unsupported attributes type" + return misperrors + + r = requests.get(haveibeenpwned_api_url + email, headers={'user-agent': default_user_agent})#Real request + if r.status_code == 200:##OK (record found) + breaches = json.loads(r.text) + if breaches: + return {'results': [{'types': mispattributes['output'], 'values': breaches}]} + elif r.status_code == 404:#Not found (not an error) + return {'results': [{'types': mispattributes['output'], 'values': 'OK (Not Found)'}]} + else:#Real error + misperrors['error'] = 'haveibeenpwned.com API not accessible (HTTP ' + str(r.status_code) + ')' + return misperrors['error'] + +def introspection(): + return mispattributes + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinf diff --git a/misp_modules/modules/expansion/macvendors.py b/misp_modules/modules/expansion/macvendors.py new file mode 100644 index 0000000..55d0ef3 --- /dev/null +++ b/misp_modules/modules/expansion/macvendors.py @@ -0,0 +1,42 @@ +import requests +import json + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['mac-address'], 'output': ['text']} +moduleinfo = {'version': '0.1', 'author': 'Aurélien Schwab', 'description': 'Module to access Macvendors API.', 'module-type': ['hover']} +moduleconfig = ['user-agent'] # TODO take this into account in the code + +macvendors_api_url = 'https://api.macvendors.com/' +default_user_agent = 'MISP-Module' + + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + for input_type in mispattributes['input']: + if input_type in request: + mac = request[input_type] + break + else: + misperrors['error'] = "Unsupported attributes type" + return misperrors + r = requests.get(macvendors_api_url + mac, headers={'user-agent': default_user_agent}) # Real request + if r.status_code == 200: # OK (record found) + response = r.text + if response: + return {'results': [{'types': mispattributes['output'], 'values': response}]} + elif r.status_code == 404: # Not found (not an error) + return {'results': [{'types': mispattributes['output'], 'values': 'Not found'}]} + else: # Real error + misperrors['error'] = 'MacVendors API not accessible (HTTP ' + str(r.status_code) + ')' + return misperrors['error'] + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/tests/test_expansions.py b/tests/test_expansions.py new file mode 100644 index 0000000..d581a31 --- /dev/null +++ b/tests/test_expansions.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import unittest +import requests +from urllib.parse import urljoin + + +class TestExpansions(unittest.TestCase): + + def setUp(self): + self.maxDiff = None + self.headers = {'Content-Type': 'application/json'} + self.url = "http://127.0.0.1:6666/" + + def misp_modules_post(self, query): + return requests.post(urljoin(self.url, "query"), json=query) + + def get_values(self, response): + return response.json()['results'][0]['values'] + + def test_cve(self): + query = {"module": "cve", "vulnerability": "CVE-2010-3333"} + response = self.misp_modules_post(query) + self.assertTrue(self.get_values(response).startswith("Stack-based buffer overflow in Microsoft Office XP SP3, Office 2003 SP3")) + + def test_dns(self): + query = {"module": "dns", "hostname": "www.circl.lu", "config": {"nameserver": "8.8.8.8"}} + response = self.misp_modules_post(query) + self.assertEqual(self.get_values(response), ['149.13.33.14']) + + def test_macvendors(self): + query = {"module": "macvendors", "mac-address": "FC-A1-3E-2A-1C-33"} + response = self.misp_modules_post(query) + self.assertEqual(self.get_values(response), 'Samsung Electronics Co.,Ltd') + + def test_haveibeenpwned(self): + query = {"module": "hibp", "email-src": "info@circl.lu"} + response = self.misp_modules_post(query) + self.assertEqual(self.get_values(response), 'OK (Not Found)') + + def test_greynoise(self): + query = {"module": "greynoise", "ip-dst": "1.1.1.1"} + response = self.misp_modules_post(query) + self.assertEqual(self.get_values(response)['status'], 'ok') + + def test_ipasn(self): + query = {"module": "ipasn", "ip-dst": "1.1.1.1"} + response = self.misp_modules_post(query) + key = list(self.get_values(response)['response'].keys())[0] + entry = self.get_values(response)['response'][key]['asn'] + self.assertEqual(entry, '13335') + + def test_bgpranking(self): + query = {"module": "bgpranking", "AS": "13335"} + response = self.misp_modules_post(query) + self.assertEqual(self.get_values(response)['response']['asn_description'], 'CLOUDFLARENET - Cloudflare, Inc., US') From 9ea9816ad3911f0833d337f7b44cd580e940b35b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 2 Apr 2019 15:31:50 +0200 Subject: [PATCH 10/59] chg: Bump dependencies --- Pipfile.lock | 180 +++++++++++++++++++++++++-------------------------- 1 file changed, 88 insertions(+), 92 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index 60f1d03..b5faca1 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -253,39 +253,35 @@ }, "pillow": { "hashes": [ - "sha256:051de330a06c99d6f84bcf582960487835bcae3fc99365185dc2d4f65a390c0e", - "sha256:0ae5289948c5e0a16574750021bd8be921c27d4e3527800dc9c2c1d2abc81bf7", - "sha256:0b1efce03619cdbf8bcc61cfae81fcda59249a469f31c6735ea59badd4a6f58a", - "sha256:163136e09bd1d6c6c6026b0a662976e86c58b932b964f255ff384ecc8c3cefa3", - "sha256:18e912a6ccddf28defa196bd2021fe33600cbe5da1aa2f2e2c6df15f720b73d1", - "sha256:24ec3dea52339a610d34401d2d53d0fb3c7fd08e34b20c95d2ad3973193591f1", - "sha256:267f8e4c0a1d7e36e97c6a604f5b03ef58e2b81c1becb4fccecddcb37e063cc7", - "sha256:3273a28734175feebbe4d0a4cde04d4ed20f620b9b506d26f44379d3c72304e1", - "sha256:4c678e23006798fc8b6f4cef2eaad267d53ff4c1779bd1af8725cc11b72a63f3", - "sha256:4d4bc2e6bb6861103ea4655d6b6f67af8e5336e7216e20fff3e18ffa95d7a055", - "sha256:505738076350a337c1740a31646e1de09a164c62c07db3b996abdc0f9d2e50cf", - "sha256:5233664eadfa342c639b9b9977190d64ad7aca4edc51a966394d7e08e7f38a9f", - "sha256:5d95cb9f6cced2628f3e4de7e795e98b2659dfcc7176ab4a01a8b48c2c2f488f", - "sha256:7eda4c737637af74bac4b23aa82ea6fbb19002552be85f0b89bc27e3a762d239", - "sha256:801ddaa69659b36abf4694fed5aa9f61d1ecf2daaa6c92541bbbbb775d97b9fe", - "sha256:825aa6d222ce2c2b90d34a0ea31914e141a85edefc07e17342f1d2fdf121c07c", - "sha256:9c215442ff8249d41ff58700e91ef61d74f47dfd431a50253e1a1ca9436b0697", - "sha256:a3d90022f2202bbb14da991f26ca7a30b7e4c62bf0f8bf9825603b22d7e87494", - "sha256:a631fd36a9823638fe700d9225f9698fb59d049c942d322d4c09544dc2115356", - "sha256:a6523a23a205be0fe664b6b8747a5c86d55da960d9586db039eec9f5c269c0e6", - "sha256:a756ecf9f4b9b3ed49a680a649af45a8767ad038de39e6c030919c2f443eb000", - "sha256:b117287a5bdc81f1bac891187275ec7e829e961b8032c9e5ff38b70fd036c78f", - "sha256:ba04f57d1715ca5ff74bb7f8a818bf929a204b3b3c2c2826d1e1cc3b1c13398c", - "sha256:cd878195166723f30865e05d87cbaf9421614501a4bd48792c5ed28f90fd36ca", - "sha256:cee815cc62d136e96cf76771b9d3eb58e0777ec18ea50de5cfcede8a7c429aa8", - "sha256:d1722b7aa4b40cf93ac3c80d3edd48bf93b9208241d166a14ad8e7a20ee1d4f3", - "sha256:d7c1c06246b05529f9984435fc4fa5a545ea26606e7f450bdbe00c153f5aeaad", - "sha256:e9c8066249c040efdda84793a2a669076f92a301ceabe69202446abb4c5c5ef9", - "sha256:f227d7e574d050ff3996049e086e1f18c7bd2d067ef24131e50a1d3fe5831fbc", - "sha256:fc9a12aad714af36cf3ad0275a96a733526571e52710319855628f476dcb144e" + "sha256:15c056bfa284c30a7f265a41ac4cbbc93bdbfc0dfe0613b9cb8a8581b51a9e55", + "sha256:1a4e06ba4f74494ea0c58c24de2bb752818e9d504474ec95b0aa94f6b0a7e479", + "sha256:1c3c707c76be43c9e99cb7e3d5f1bee1c8e5be8b8a2a5eeee665efbf8ddde91a", + "sha256:1fd0b290203e3b0882d9605d807b03c0f47e3440f97824586c173eca0aadd99d", + "sha256:24114e4a6e1870c5a24b1da8f60d0ba77a0b4027907860188ea82bd3508c80eb", + "sha256:258d886a49b6b058cd7abb0ab4b2b85ce78669a857398e83e8b8e28b317b5abb", + "sha256:33c79b6dd6bc7f65079ab9ca5bebffb5f5d1141c689c9c6a7855776d1b09b7e8", + "sha256:367385fc797b2c31564c427430c7a8630db1a00bd040555dfc1d5c52e39fcd72", + "sha256:3c1884ff078fb8bf5f63d7d86921838b82ed4a7d0c027add773c2f38b3168754", + "sha256:44e5240e8f4f8861d748f2a58b3f04daadab5e22bfec896bf5434745f788f33f", + "sha256:46aa988e15f3ea72dddd81afe3839437b755fffddb5e173886f11460be909dce", + "sha256:74d90d499c9c736d52dd6d9b7221af5665b9c04f1767e35f5dd8694324bd4601", + "sha256:809c0a2ce9032cbcd7b5313f71af4bdc5c8c771cb86eb7559afd954cab82ebb5", + "sha256:85d1ef2cdafd5507c4221d201aaf62fc9276f8b0f71bd3933363e62a33abc734", + "sha256:8c3889c7681af77ecfa4431cd42a2885d093ecb811e81fbe5e203abc07e0995b", + "sha256:9218d81b9fca98d2c47d35d688a0cea0c42fd473159dfd5612dcb0483c63e40b", + "sha256:9aa4f3827992288edd37c9df345783a69ef58bd20cc02e64b36e44bcd157bbf1", + "sha256:9d80f44137a70b6f84c750d11019a3419f409c944526a95219bea0ac31f4dd91", + "sha256:b7ebd36128a2fe93991293f997e44be9286503c7530ace6a55b938b20be288d8", + "sha256:c4c78e2c71c257c136cdd43869fd3d5e34fc2162dc22e4a5406b0ebe86958239", + "sha256:c6a842537f887be1fe115d8abb5daa9bc8cc124e455ff995830cc785624a97af", + "sha256:cf0a2e040fdf5a6d95f4c286c6ef1df6b36c218b528c8a9158ec2452a804b9b8", + "sha256:cfd28aad6fc61f7a5d4ee556a997dc6e5555d9381d1390c00ecaf984d57e4232", + "sha256:dca5660e25932771460d4688ccbb515677caaf8595f3f3240ec16c117deff89a", + "sha256:de7aedc85918c2f887886442e50f52c1b93545606317956d65f342bd81cb4fc3", + "sha256:e6c0bbf8e277b74196e3140c35f9a1ae3eafd818f7f2d3a15819c49135d6c062" ], "index": "pypi", - "version": "==5.4.1" + "version": "==6.0.0" }, "psutil": { "hashes": [ @@ -304,7 +300,7 @@ "pybgpranking": { "editable": true, "git": "https://github.com/D4-project/BGP-Ranking.git/", - "ref": "4f2898af7c4e237b6497831d5acf3f4531ac14d8", + "ref": "019ef1c40aad1e5bb5c5072c9a998c6a8f0271f3", "subdirectory": "client" }, "pydnstrails": { @@ -335,13 +331,13 @@ "pyipasnhistory": { "editable": true, "git": "https://github.com/D4-project/IPASN-History.git/", - "ref": "7ef09cf761fc58aa774ea305a33ba75959e39887", + "ref": "0c4f11792061417b77ca6e22d2ece18109d74c75", "subdirectory": "client" }, "pymisp": { "editable": true, "git": "https://github.com/MISP/PyMISP.git", - "ref": "1dddfd72e08886673e57e23627064f9ea8303d4c" + "ref": "64bcaad0e578129543cdffad532a232722615f6c" }, "pyonyphe": { "editable": true, @@ -414,44 +410,44 @@ }, "redis": { "hashes": [ - "sha256:724932360d48e5407e8f82e405ab3650a36ed02c7e460d1e6fddf0f038422b54", - "sha256:9b19425a38fd074eb5795ff2b0d9a55b46a44f91f5347995f27e3ad257a7d775" + "sha256:6946b5dca72e86103edc8033019cc3814c031232d339d5f4533b02ea85685175", + "sha256:8ca418d2ddca1b1a850afa1680a7d2fd1f3322739271de4b704e0d4668449273" ], - "version": "==3.2.0" + "version": "==3.2.1" }, "reportlab": { "hashes": [ - "sha256:069f684cd0aaa518a27dc9124aed29cee8998e21ddf19604e53214ec8462bdd7", - "sha256:09b68ec01d86b4b120456b3f3202570ec96f57624e3a4fc36f3829323391daa4", - "sha256:0c32be9a406172c29ea20ff55a709ccac1e7fb09f15aba67cb7b455fd1d3dbe0", - "sha256:233196cf25e97cfe7c452524ea29d9a4909f1cb66599299233be1efaaaa7a7a3", - "sha256:2b5e4533f3e5b962835a5ce44467e66d1ecc822761d1b508077b5087a06be338", - "sha256:2e860bcdace5a558356802a92ae8658d7e5fdaa00ded82e83a3f2987c562cb66", - "sha256:3546029e63a9a9dc24ee38959eb417678c2425b96cd27b31e09e216dafc94666", - "sha256:4452b93f9c73b6b70311e7d69082d64da81b38e91bfb4766397630092e6da6fd", - "sha256:528c74a1c6527d1859c2c7a64a94a1cba485b00175162ea23699ae58a1e94939", - "sha256:6116e750f98018febc08dfee6df20446cf954adbcfa378d2c703d56c8864aff3", - "sha256:6b2b3580c647d75ef129172cb3da648cdb24566987b0b59c5ebb80ab770748d6", - "sha256:727b5f2bed08552d143fc99649b1863c773729f580a416844f9d9967bb0a1ae8", - "sha256:74c24a3ec0a3d4f8acb13a07192f45bdb54a1cc3c2286241677e7e8bcd5011fa", - "sha256:98ccd2f8b4f8636db05f3f14db0b471ad6bb4b66ae0dc9052c4822b3bd5d6a7d", - "sha256:a5905aa567946bc938b489a7249c7890c3fd3c9b7b5680dece5bc551c2ddbe0d", - "sha256:acbb7f676b8586b770719e9683eda951fdb38eb7970d46fcbf3cdda88d912a64", - "sha256:b5e30f865add48cf880f1c363eb505b97f2f7baaa88c155f87a335a76515a3e5", - "sha256:be2a7c33a2c28bbd3f453ffe4f0e5200b88c803a097f4cf52d69c6b53fad7a8f", - "sha256:c356bb600f59ac64955813d6497a08bfd5d0c451cb5829b61e3913d0ac084e26", - "sha256:c7ec4ae2393beab584921b1287a04e94fd98c28315e348362d89b85f4b464546", - "sha256:d476edc831bb3e9ebd04d1403abaf3ea57b3e4c2276c91a54fdfb6efbd3f9d97", - "sha256:db059e1a0691c872784062421ec51848539eb4f5210142682e61059a5ca7cc55", - "sha256:dd423a6753509ab14a0ac1b5be39d219c8f8d3781cce3deb4f45eda31969b5e8", - "sha256:ed9b7c0d71ce6fe2b31c6cde530ad8238632b876a5d599218739bda142a77f7c", - "sha256:f0a2465af4006f97b05e1f1546d67d3a3213d414894bf28be7f87f550a7f4a55", - "sha256:f20bfe26e57e8e1f575a9e0325be04dd3562db9f247ffdd73b5d4df6dec53bc2", - "sha256:f3463f2cb40a1b515ac0133ba859eca58f53b56760da9abb27ed684c565f853c", - "sha256:facc3c9748ab1525fb8401a1223bce4f24f0d6aa1a9db86c55db75777ccf40f9" + "sha256:0135bc54a463db5315c93bba4182fb83dc088fefaa7da18784ecd2a0c4a9c068", + "sha256:09e167e01458ea1e0cf3acff634ae9ecc1f1757e7585060d039c90b762859cfd", + "sha256:0dfcea18ba3ca1fac55cb273d056a8a43a48bd04d419299b3267e1994c72455a", + "sha256:1a61e56593ea1a8a38135eedfb40f79dcad13164fff034313ebf2a30e200ca79", + "sha256:1bdd871c2087d3853a0e9a3a573b1a7535500f3341944b1e34e68f3213cd28b8", + "sha256:26878a4b9c45f046c635b5695681188c19806f08b04129ea01c9ed51c7754039", + "sha256:27c62264c758aa30113df105da816223d149e4e87ee778ad49469725b79be2eb", + "sha256:29a9dd3954465b9e4efb129ffda9ab3e6a4f06488e8aa2efd5aff8ad332f13c2", + "sha256:5740e3218ca98c1bc86bd2d2e2a8c1d23e7c97d949d6377ac30aaf449f01c363", + "sha256:605892bb3f822a1e7342ce2b461d645ab8e4d13875127c0ae5377f76853db422", + "sha256:6dacc72552bc0dd50286e856f09a5e646a007d9345598bf6f75b117a200bfd9d", + "sha256:7021b7c8ba6d8e69e4c68c9473067482aaa40b9094270b45dbf798fcb0e09bd4", + "sha256:8acd950dad5b20a417579d1253c1065222dde48f9412e71533b052ab3dd98632", + "sha256:8b8fb3b0dd1e2124aba24544a02c95bff1fffa966b0581f30abf4fb28e414005", + "sha256:920c61c942eb1cc446e1647a04978f4afe31993ed403b74576a018c3ca526394", + "sha256:928e8d99befe064e28e9a29a4fd9afcf2066dcd758b0903280e67e221527422a", + "sha256:a04787eee401a74c80b65e539b5fe9226fdeabe25caa3d216c21dc990b2f8a01", + "sha256:a5bb6bd7753cba854425fcf7ecf04627a17de78d47ef9e8fac615887c5658da3", + "sha256:a70d970619014dc83b4406bcfed7e2f9d5aaf5f521aad808f5560d90ea896fb4", + "sha256:ae468fe82c8af3d1987113f03c1f87d01daa5b4c85c1f10da126be84423a744d", + "sha256:b278d83a7f76410bd310b368309e6e4b19664ffa686abfa9f0696130b09c17d3", + "sha256:b6623e9a96db3edc4b384e036e67c7bc87bbd7e5dc2d72ce66efa0043f9383b0", + "sha256:dc15cfa577bb25f0a598d483cf6dcc5ecad576ba723fe9bec63b6ec720dab2a3", + "sha256:dffdb4f6b34ce791e67365f3f96ab3c45b4cdd2c70d212fac98fb146dc75ac80", + "sha256:e84020e3482856da733e1359cb7b84e6bac09179bd3af860e70468a9c3cb43e3", + "sha256:edda09668e8474d5acb1a37fb64599557b43a714f1469bd49a058e95b5b410ff", + "sha256:f77e9835873931d25f836a3c107e53e0f7d3c0b4906b13063815308cf5ca1fac", + "sha256:f91d16ff07d5d3c92303f64c6864d74d3b6a491dde186bfef90c58088f932998" ], "index": "pypi", - "version": "==3.5.13" + "version": "==3.5.17" }, "requests": { "hashes": [ @@ -477,10 +473,10 @@ }, "sigmatools": { "hashes": [ - "sha256:3bdbd2ee99c32f245e948d6b882219729ab379685dd7366e4d6149c390e08170" + "sha256:ae980b6d6fd466294911efa493934d24e3c5df406da4a190b9fff0943a81cc5f" ], "index": "pypi", - "version": "==0.9" + "version": "==0.10" }, "six": { "hashes": [ @@ -491,10 +487,10 @@ }, "soupsieve": { "hashes": [ - "sha256:afa56bf14907bb09403e5d15fbed6275caa4174d36b975226e3b67a3bb6e2c4b", - "sha256:eaed742b48b1f3e2d45ba6f79401b2ed5dc33b2123dfe216adb90d4bfa0ade26" + "sha256:3aef141566afd07201b525c17bfaadd07580a8066f82b57f7c9417f26adbd0a3", + "sha256:e41a65e99bd125972d84221022beb1e4b5cfc68fa12c170c39834ce32d1b294c" ], - "version": "==1.8" + "version": "==1.9" }, "sparqlwrapper": { "hashes": [ @@ -520,15 +516,15 @@ }, "tornado": { "hashes": [ - "sha256:1a58f2d603476d5e462f7c28ca1dbb5ac7e51348b27a9cac849cdec3471101f8", - "sha256:33f93243cd46dd398e5d2bbdd75539564d1f13f25d704cfc7541db74066d6695", - "sha256:34e59401afcecf0381a28228daad8ed3275bcb726810654612d5e9c001f421b7", - "sha256:35817031611d2c296c69e5023ea1f9b5720be803e3bb119464bb2a0405d5cd70", - "sha256:666b335cef5cc2759c21b7394cff881f71559aaf7cb8c4458af5bb6cb7275b47", - "sha256:81203efb26debaaef7158187af45bc440796de9fb1df12a75b65fae11600a255", - "sha256:de274c65f45f6656c375cdf1759dbf0bc52902a1e999d12a35eb13020a641a53" + "sha256:1174dcb84d08887b55defb2cda1986faeeea715fff189ef3dc44cce99f5fca6b", + "sha256:2613fab506bd2aedb3722c8c64c17f8f74f4070afed6eea17f20b2115e445aec", + "sha256:44b82bc1146a24e5b9853d04c142576b4e8fa7a92f2e30bc364a85d1f75c4de2", + "sha256:457fcbee4df737d2defc181b9073758d73f54a6cfc1f280533ff48831b39f4a8", + "sha256:49603e1a6e24104961497ad0c07c799aec1caac7400a6762b687e74c8206677d", + "sha256:8c2f40b99a8153893793559919a355d7b74649a11e59f411b0b0a1793e160bc0", + "sha256:e1d897889c3b5a829426b7d52828fb37b28bc181cd598624e65c8be40ee3f7fa" ], - "version": "==6.0.1" + "version": "==6.0.2" }, "url-normalize": { "hashes": [ @@ -559,20 +555,20 @@ }, "vulners": { "hashes": [ - "sha256:08a7ccb2b210d45143354c6161c73fe209dc14fae8692e8b793b36b79330ad11", - "sha256:bfe2478cc11c69ba7e436d7a5df925e227565782c0bd603929fb3d612c73d78d", - "sha256:d035f6a883625878a1dc377830d17d9702ef138ca31569ac01cb8686874f89cd" + "sha256:6617d5904b5369507bc34105071d312e9e1c38d73654505e7b15b9a3f1325915", + "sha256:8b05d12a9dd7cbc07198a13281299a6e014ec348522e214b1efd097e194b7568", + "sha256:a19b02e0a112d70951e10c5abc1993f7f029234212828e1b617ab35f4e460a24" ], "index": "pypi", - "version": "==1.4.5" + "version": "==1.4.7" }, "wand": { "hashes": [ - "sha256:7d6b8dc9d4eaccc430b9c86e6b749013220c994970a3f39e902b397e2fa732c3", - "sha256:cc0b5c9cd50fecd10dc8888b739dd5984c6f8085d2954f34903b83ca39a91236" + "sha256:91810d241ab0851d40e67c946beb960b869c4f4160c397eac291ec6283ee3e3f", + "sha256:ae7c0958509a22f531b7b97e93adfd3f1208f0ac1c593af9e5f0cffa4ac06d5b" ], "index": "pypi", - "version": "==0.5.1" + "version": "==0.5.2" }, "xlsxwriter": { "hashes": [ @@ -719,11 +715,11 @@ }, "more-itertools": { "hashes": [ - "sha256:0125e8f60e9e031347105eb1682cef932f5e97d7b9a1a28d9bf00c22a5daef40", - "sha256:590044e3942351a1bdb1de960b739ff4ce277960f2425ad4509446dbace8d9d1" + "sha256:2112d2ca570bb7c3e53ea1a35cd5df42bb0fd10c45f0fb97178679c3c03d64c7", + "sha256:c3e4748ba1aad8dba30a4886b0b1a2004f9a863837b8654e7059eebf727afa5a" ], "markers": "python_version > '2.7'", - "version": "==6.0.0" + "version": "==7.0.0" }, "nose": { "hashes": [ @@ -764,11 +760,11 @@ }, "pytest": { "hashes": [ - "sha256:592eaa2c33fae68c7d75aacf042efc9f77b27c08a6224a4f59beab8d9a420523", - "sha256:ad3ad5c450284819ecde191a654c09b0ec72257a2c711b9633d677c71c9850c4" + "sha256:13c5e9fb5ec5179995e9357111ab089af350d788cbc944c628f3cde72285809b", + "sha256:f21d2f1fb8200830dcbb5d8ec466a9c9120e20d8b53c7585d180125cce1d297a" ], "index": "pypi", - "version": "==4.3.1" + "version": "==4.4.0" }, "requests": { "hashes": [ From c64f514a6fdfb21397dcddd8feddf7ee90bf5767 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 2 Apr 2019 15:39:27 +0200 Subject: [PATCH 11/59] fix: Typos in variable names --- misp_modules/modules/expansion/greynoise.py | 2 +- misp_modules/modules/expansion/hibp.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/greynoise.py b/misp_modules/modules/expansion/greynoise.py index 324423d..66f6985 100644 --- a/misp_modules/modules/expansion/greynoise.py +++ b/misp_modules/modules/expansion/greynoise.py @@ -37,4 +37,4 @@ def introspection(): def version(): moduleinfo['config'] = moduleconfig - return moduleinf + return moduleinfo diff --git a/misp_modules/modules/expansion/hibp.py b/misp_modules/modules/expansion/hibp.py index 7010aee..67af644 100644 --- a/misp_modules/modules/expansion/hibp.py +++ b/misp_modules/modules/expansion/hibp.py @@ -37,4 +37,4 @@ def introspection(): def version(): moduleinfo['config'] = moduleconfig - return moduleinf + return moduleinfo From 9cb21f98e1b6670d733940ea74d75a7a01a1b38e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 2 Apr 2019 15:46:17 +0200 Subject: [PATCH 12/59] fix: Add the new module sin the list of modules availables. --- misp_modules/modules/expansion/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py index d8fb153..42f74a3 100644 --- a/misp_modules/modules/expansion/__init__.py +++ b/misp_modules/modules/expansion/__init__.py @@ -8,4 +8,4 @@ __all__ = ['vmray_submit', 'bgpranking', 'circl_passivedns', 'circl_passivessl', 'yara_syntax_validator', 'hashdd', 'onyphe', 'onyphe_full', 'rbl', 'xforceexchange', 'sigma_syntax_validator', 'stix2_pattern_syntax_validator', 'sigma_queries', 'dbl_spamhaus', 'vulners', 'yara_query', 'macaddress_io', - 'intel471', 'backscatter_io', 'btc_scam_check'] + 'intel471', 'backscatter_io', 'btc_scam_check', 'hibp', 'greynoise', 'macvendors'] From f82933779f1567e1d4325f86a29b4459c9d624cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 2 Apr 2019 16:01:33 +0200 Subject: [PATCH 13/59] fix: pep8 foobar. --- misp_modules/modules/expansion/greynoise.py | 15 +++++++++------ misp_modules/modules/expansion/hibp.py | 17 ++++++++++------- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/misp_modules/modules/expansion/greynoise.py b/misp_modules/modules/expansion/greynoise.py index 66f6985..d26736d 100644 --- a/misp_modules/modules/expansion/greynoise.py +++ b/misp_modules/modules/expansion/greynoise.py @@ -4,11 +4,12 @@ import json misperrors = {'error': 'Error'} mispattributes = {'input': ['ip-dst', 'ip-src'], 'output': ['text']} moduleinfo = {'version': '0.1', 'author': 'Aurélien Schwab ', 'description': 'Module to access GreyNoise.io API.', 'module-type': ['hover']} -moduleconfig = ['user-agent']#TODO take this into account in the code +moduleconfig = ['user-agent'] # TODO take this into account in the code greynoise_api_url = 'http://api.greynoise.io:8888/v1/query/ip' default_user_agent = 'MISP-Module' + def handler(q=False): if q is False: return False @@ -20,21 +21,23 @@ def handler(q=False): else: misperrors['error'] = "Unsupported attributes type" return misperrors - data = {'ip':ip} - r = requests.post(greynoise_api_url, data=data, headers={'user-agent': default_user_agent})#Real request - if r.status_code == 200:#OK (record found) + data = {'ip': ip} + r = requests.post(greynoise_api_url, data=data, headers={'user-agent': default_user_agent}) # Real request + if r.status_code == 200: # OK (record found) response = json.loads(r.text) if response: return {'results': [{'types': mispattributes['output'], 'values': response}]} - elif r.status_code == 404:#Not found (not an error) + elif r.status_code == 404: # Not found (not an error) return {'results': [{'types': mispattributes['output'], 'values': 'No data'}]} - else:#Real error + else: # Real error misperrors['error'] = 'GreyNoise API not accessible (HTTP ' + str(r.status_code) + ')' return misperrors['error'] + def introspection(): return mispattributes + def version(): moduleinfo['config'] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/hibp.py b/misp_modules/modules/expansion/hibp.py index 67af644..8db3fa7 100644 --- a/misp_modules/modules/expansion/hibp.py +++ b/misp_modules/modules/expansion/hibp.py @@ -2,12 +2,13 @@ import requests import json misperrors = {'error': 'Error'} -mispattributes = {'input': ['email-dst', 'email-src'], 'output': ['text']}#All mails as input +mispattributes = {'input': ['email-dst', 'email-src'], 'output': ['text']} # All mails as input moduleinfo = {'version': '0.1', 'author': 'Aurélien Schwab', 'description': 'Module to access haveibeenpwned.com API.', 'module-type': ['hover']} -moduleconfig = ['user-agent']#TODO take this into account in the code +moduleconfig = ['user-agent'] # TODO take this into account in the code haveibeenpwned_api_url = 'https://api.haveibeenpwned.com/api/v2/breachedaccount/' -default_user_agent = 'MISP-Module'#User agent (must be set, requiered by API)) +default_user_agent = 'MISP-Module' # User agent (must be set, requiered by API)) + def handler(q=False): if q is False: @@ -21,20 +22,22 @@ def handler(q=False): misperrors['error'] = "Unsupported attributes type" return misperrors - r = requests.get(haveibeenpwned_api_url + email, headers={'user-agent': default_user_agent})#Real request - if r.status_code == 200:##OK (record found) + r = requests.get(haveibeenpwned_api_url + email, headers={'user-agent': default_user_agent}) # Real request + if r.status_code == 200: # OK (record found) breaches = json.loads(r.text) if breaches: return {'results': [{'types': mispattributes['output'], 'values': breaches}]} - elif r.status_code == 404:#Not found (not an error) + elif r.status_code == 404: # Not found (not an error) return {'results': [{'types': mispattributes['output'], 'values': 'OK (Not Found)'}]} - else:#Real error + else: # Real error misperrors['error'] = 'haveibeenpwned.com API not accessible (HTTP ' + str(r.status_code) + ')' return misperrors['error'] + def introspection(): return mispattributes + def version(): moduleinfo['config'] = moduleconfig return moduleinfo From 07a66d62b065bfe06225f5a0d1f6d95247e6d9b2 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Tue, 2 Apr 2019 20:03:11 +0200 Subject: [PATCH 14/59] chg: [doc] new modules added --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index e890433..5669b19 100644 --- a/README.md +++ b/README.md @@ -32,11 +32,14 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/ * [EUPI](misp_modules/modules/expansion/eupi.py) - a hover and expansion module to get information about an URL from the [Phishing Initiative project](https://phishing-initiative.eu/?lang=en). * [Farsight DNSDB Passive DNS](misp_modules/modules/expansion/farsight_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information. * [GeoIP](misp_modules/modules/expansion/geoip_country.py) - a hover and expansion module to get GeoIP information from geolite/maxmind. +* [Greynoise](misp_modules/modules/expansion/greynoise.py) - a hover to get information from greynoise. * [hashdd](misp_modules/modules/expansion/hashdd.py) - a hover module to check file hashes against [hashdd.com](http://www.hashdd.com) including NSLR dataset. +* [hibp](misp_modules/modules/expansion/hibp.py) - a hover module to lookup against Have I Been Pwned? * [intel471](misp_modules/modules/expansion/intel471.py) - an expansion module to get info from [Intel471](https://intel471.com). * [IPASN](misp_modules/modules/expansion/ipasn.py) - a hover and expansion to get the BGP ASN of an IP address. * [iprep](misp_modules/modules/expansion/iprep.py) - an expansion module to get IP reputation from packetmail.net. * [macaddress.io](misp_modules/modules/expansion/macaddress_io.py) - a hover module to retrieve vendor details and other information regarding a given MAC address or an OUI from [MAC address Vendor Lookup](https://macaddress.io). See [integration tutorial here](https://macaddress.io/integrations/MISP-module). +* [macvendors](misp_modules/modules/expansion/macvendors.py) - a hover module to retrieve mac vendor information. * [onyphe](misp_modules/modules/expansion/onyphe.py) - a modules to process queries on Onyphe. * [onyphe_full](misp_modules/modules/expansion/onyphe_full.py) - a modules to process full queries on Onyphe. * [OTX](misp_modules/modules/expansion/otx.py) - an expansion module for [OTX](https://otx.alienvault.com/). From 5ed91dcec2bf4509a5b3142f29c9ae417e6e1c02 Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Mon, 8 Apr 2019 16:03:41 +0900 Subject: [PATCH 15/59] fix: [doc] Small typo fix --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5669b19..18b5548 100644 --- a/README.md +++ b/README.md @@ -125,7 +125,7 @@ scl enable rh-python36 ‘python3 –m pip install cryptography’ scl enable rh-python36 ‘python3 –m pip install -I -r REQUIREMENTS’ scl enable rh-python36 ‘python3 –m pip install –I .’ ~~~~ -Create the service file /etc/systemd/system/misp-workers.service : +Create the service file /etc/systemd/system/misp-modules.service : ~~~~ [Unit] Description=MISP's modules From b5f2424f274813e044ba88210628575855263b5e Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Mon, 8 Apr 2019 16:17:22 +0900 Subject: [PATCH 16/59] chg: [doc] Updated README to reflect current virtualenv efforts. TODO: pipenv --- README.md | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 18b5548..400356e 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,7 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/ * [ThreatAnalyzer](misp_modules/modules/import_mod/threatanalyzer_import.py) - An import module to process ThreatAnalyzer archive.zip/analysis.json sandbox exports. * [VMRay](misp_modules/modules/import_mod/vmray_import.py) - An import module to process VMRay export. -## How to install and start MISP modules in a Python virtualenv? +## How to install and start MISP modules in a Python virtualenv? (recommended) ~~~~bash sudo apt-get install python3-dev python3-pip libpq5 libjpeg-dev tesseract-ocr imagemagick virtualenv @@ -97,7 +97,10 @@ sudo git clone https://github.com/MISP/misp-modules.git cd misp-modules sudo -u www-data /var/www/MISP/venv/bin/pip install -I -r REQUIREMENTS sudo -u www-data /var/www/MISP/venv/bin/pip install . -sudo sed -i -e '$i \sudo -u www-data /var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s > /tmp/misp-modules_rc.local.log &\n' /etc/rc.local +# Start misp-modules as a service +sudo cp etc/systemd/system/misp-modules.service /etc/systemd/system/ +sudo systemctl daemon-reload +sudo systemctl enable --now misp-modules /var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s & #to start the modules ~~~~ @@ -110,7 +113,10 @@ sudo git clone https://github.com/MISP/misp-modules.git cd misp-modules sudo pip3 install -I -r REQUIREMENTS sudo pip3 install -I . -sudo sed -i -e '$i \sudo -u www-data /var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s > /tmp/misp-modules_rc.local.log &\n' /etc/rc.local +# Start misp-modules as a service +sudo cp etc/systemd/system/misp-modules.service /etc/systemd/system/ +sudo systemctl daemon-reload +sudo systemctl enable --now misp-modules /var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s & #to start the modules ~~~~ @@ -490,7 +496,7 @@ Download a pre-built virtual image from the [MISP training materials](https://ww - Create a Host-Only adapter in VirtualBox - Set your Misp OVA to that Host-Only adapter - Start the virtual machine -- Get the IP address of the virutal machine +- Get the IP address of the virtual machine - SSH into the machine (Login info on training page) - Go into the misp-modules directory @@ -510,14 +516,16 @@ Remove the contents of the build directory and re-install misp-modules. ~~~bash sudo rm -fr build/* -sudo pip3 install --upgrade . +sudo -u www-data /var/www/MISP/venv/bin/pip install --upgrade . ~~~ SSH in with a different terminal and run `misp-modules` with debugging enabled. ~~~bash -sudo killall misp-modules -misp-modules -d +# In case misp-modules is not a service do: +# sudo killall misp-modules +sudo systemctl disable --now misp-modules +sudo -u www-data /var/www/MISP/venv/bin/misp-modules -d ~~~ From d24a6e2e249f2d36a92634197c4e4c21a1b3b9b6 Mon Sep 17 00:00:00 2001 From: iceone23 Date: Mon, 15 Apr 2019 06:17:27 -0700 Subject: [PATCH 17/59] Create cisco_firesight_manager_ACL_rule_export.py Cisco Firesight Manager ACL Rule Export module --- ...cisco_firesight_manager_ACL_rule_export.py | 135 ++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 misp_modules/modules/export_mod/cisco_firesight_manager_ACL_rule_export.py diff --git a/misp_modules/modules/export_mod/cisco_firesight_manager_ACL_rule_export.py b/misp_modules/modules/export_mod/cisco_firesight_manager_ACL_rule_export.py new file mode 100644 index 0000000..dcb6178 --- /dev/null +++ b/misp_modules/modules/export_mod/cisco_firesight_manager_ACL_rule_export.py @@ -0,0 +1,135 @@ +###################################################### +# # +# Author: Stanislav Klevtsov, Ukraine; Feb 2019. # +# # +# # +# Script was tested on the following configuration: # +# MISP v2.4.90 # +# Cisco Firesight Manager Console v6.2.3 (bld 84) # +# # +###################################################### + +import json +import base64 +import csv +from urllib.parse import quote + +misperrors = {'error': 'Error'} + +moduleinfo = {'version': '1', 'author': 'Stanislav Klevtsov', + 'description': 'Export malicious network activity attributes of the MISP event to Cisco firesight manager block rules', + 'module-type': ['export']} + + +moduleconfig = ['fmc_ip_addr', 'fmc_login', 'fmc_pass', 'domain_id', 'acpolicy_id'] + +fsmapping = {"ip-dst":"dst", "url":"request"} + +mispattributes = {'input':list(fsmapping.keys())} + +# options: event, attribute, event-collection, attribute-collection +inputSource = ['event'] + +outputFileExtension = 'sh' +responseType = 'application/txt' + +# .sh file templates +SH_FILE_HEADER = """#!/bin/sh\n\n""" + +BLOCK_JSON_TMPL = """ +BLOCK_RULE='{{ "action": "BLOCK", "enabled": true, "type": "AccessRule", "name": "{rule_name}", "destinationNetworks": {{ "literals": [ {dst_networks} ] }}, "urls": {{ "literals": [ {urls} ] }}, "newComments": [ "{event_info_comment}" ] }}'\n +""" + +BLOCK_DST_JSON_TMPL = """{{ "type": "Host", "value": "{ipdst}" }} """ +BLOCK_URL_JSON_TMPL = """{{ "type": "Url", "url": "{url}" }} """ + +CURL_ADD_RULE_TMPL = """ +curl -X POST -v -k -H 'Content-Type: application/json' -H \"Authorization: Basic $LOGINPASS_BASE64\" -H \"X-auth-access-token: $ACC_TOKEN\" -i \"https://$FIRESIGHT_IP_ADDR/api/fmc_config/v1/domain/$DOMAIN_ID/policy/accesspolicies/$ACPOLICY_ID/accessrules\" --data \"$BLOCK_RULE\" """ + + +def handler(q=False): + if q is False: + return False + + r = {'results': []} + request = json.loads(q) + + if "config" in request: + config = request["config"] + + #check if config is empty + if not config['fmc_ip_addr']: config['fmc_ip_addr'] = "0.0.0.0" + if not config['fmc_login']: config['fmc_login'] = "login" + if not config['fmc_pass']: config['fmc_pass'] = "password" + if not config['domain_id']: config['domain_id'] = "SET_FIRESIGHT_DOMAIN_ID" + if not config['acpolicy_id']: config['acpolicy_id'] = "SET_FIRESIGHT_ACPOLICY_ID" + + data = request["data"] + output = "" + ipdst = [] + urls = [] + + #populate the ACL rule with attributes + for ev in data: + + event = ev["Attribute"] + event_id = ev["Event"]["id"] + event_info = ev["Event"]["info"] + + for index, attr in enumerate(event): + if attr["to_ids"] is True: + + if attr["type"] in fsmapping: + if attr["type"] in "ip-dst": + ipdst.append(BLOCK_DST_JSON_TMPL.format(ipdst=attr["value"])) + else: + urls.append(BLOCK_URL_JSON_TMPL.format(url=quote(attr["value"], safe='@/:;?&=-_.,+!*'))) + + + #building the .sh file + output += SH_FILE_HEADER + output += "FIRESIGHT_IP_ADDR='" + config['fmc_ip_addr'] + "'\n" + + output += "LOGINPASS_BASE64=`echo -n '" + config['fmc_login'] + ":" + config['fmc_pass'] + "' | base64`\n" + output += "DOMAIN_ID='" + config['domain_id'] + "'\n" + output += "ACPOLICY_ID='" + config['acpolicy_id'] + "'\n\n" + + output += "ACC_TOKEN=`curl -X POST -v -k -sD - -o /dev/null -H \"Authorization: Basic $LOGINPASS_BASE64\" -i \"https://$FIRESIGHT_IP_ADDR/api/fmc_platform/v1/auth/generatetoken\" | grep -i x-auth-acc | sed 's/.*:\\ //g' | tr -d '[:space:]' | tr -d '\\n'`" + "\n" + + output += BLOCK_JSON_TMPL.format(rule_name="misp_event_"+event_id,dst_networks=', '.join(ipdst), urls=', '.join(urls), event_info_comment=event_info) + "\n" + + output += CURL_ADD_RULE_TMPL + # END building the .sh file + + r = {"data":base64.b64encode(output.encode('utf-8')).decode('utf-8')} + return r + + +def introspection(): + modulesetup = {} + try: + responseType + modulesetup['responseType'] = responseType + except NameError: + pass + try: + userConfig + modulesetup['userConfig'] = userConfig + except NameError: + pass + try: + outputFileExtension + modulesetup['outputFileExtension'] = outputFileExtension + except NameError: + pass + try: + inputSource + modulesetup['inputSource'] = inputSource + except NameError: + pass + return modulesetup + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo From f5167c2f230c7dea0725bdb585c6ce8d61496942 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 16 Apr 2019 11:25:39 +0200 Subject: [PATCH 18/59] fix: Make flake8 happy. --- ...cisco_firesight_manager_ACL_rule_export.py | 147 +++++++++--------- 1 file changed, 76 insertions(+), 71 deletions(-) diff --git a/misp_modules/modules/export_mod/cisco_firesight_manager_ACL_rule_export.py b/misp_modules/modules/export_mod/cisco_firesight_manager_ACL_rule_export.py index dcb6178..ab79692 100644 --- a/misp_modules/modules/export_mod/cisco_firesight_manager_ACL_rule_export.py +++ b/misp_modules/modules/export_mod/cisco_firesight_manager_ACL_rule_export.py @@ -11,21 +11,20 @@ import json import base64 -import csv from urllib.parse import quote misperrors = {'error': 'Error'} moduleinfo = {'version': '1', 'author': 'Stanislav Klevtsov', - 'description': 'Export malicious network activity attributes of the MISP event to Cisco firesight manager block rules', - 'module-type': ['export']} + 'description': 'Export malicious network activity attributes of the MISP event to Cisco firesight manager block rules', + 'module-type': ['export']} moduleconfig = ['fmc_ip_addr', 'fmc_login', 'fmc_pass', 'domain_id', 'acpolicy_id'] -fsmapping = {"ip-dst":"dst", "url":"request"} +fsmapping = {"ip-dst": "dst", "url": "request"} -mispattributes = {'input':list(fsmapping.keys())} +mispattributes = {'input': list(fsmapping.keys())} # options: event, attribute, event-collection, attribute-collection inputSource = ['event'] @@ -48,88 +47,94 @@ curl -X POST -v -k -H 'Content-Type: application/json' -H \"Authorization: Basic def handler(q=False): - if q is False: - return False - - r = {'results': []} - request = json.loads(q) + if q is False: + return False - if "config" in request: - config = request["config"] + r = {'results': []} + request = json.loads(q) - #check if config is empty - if not config['fmc_ip_addr']: config['fmc_ip_addr'] = "0.0.0.0" - if not config['fmc_login']: config['fmc_login'] = "login" - if not config['fmc_pass']: config['fmc_pass'] = "password" - if not config['domain_id']: config['domain_id'] = "SET_FIRESIGHT_DOMAIN_ID" - if not config['acpolicy_id']: config['acpolicy_id'] = "SET_FIRESIGHT_ACPOLICY_ID" + if "config" in request: + config = request["config"] - data = request["data"] - output = "" - ipdst = [] - urls = [] + # check if config is empty + if not config['fmc_ip_addr']: + config['fmc_ip_addr'] = "0.0.0.0" + if not config['fmc_login']: + config['fmc_login'] = "login" + if not config['fmc_pass']: + config['fmc_pass'] = "password" + if not config['domain_id']: + config['domain_id'] = "SET_FIRESIGHT_DOMAIN_ID" + if not config['acpolicy_id']: + config['acpolicy_id'] = "SET_FIRESIGHT_ACPOLICY_ID" - #populate the ACL rule with attributes - for ev in data: + data = request["data"] + output = "" + ipdst = [] + urls = [] - event = ev["Attribute"] - event_id = ev["Event"]["id"] - event_info = ev["Event"]["info"] + # populate the ACL rule with attributes + for ev in data: - for index, attr in enumerate(event): - if attr["to_ids"] is True: + event = ev["Attribute"] + event_id = ev["Event"]["id"] + event_info = ev["Event"]["info"] - if attr["type"] in fsmapping: - if attr["type"] in "ip-dst": - ipdst.append(BLOCK_DST_JSON_TMPL.format(ipdst=attr["value"])) - else: - urls.append(BLOCK_URL_JSON_TMPL.format(url=quote(attr["value"], safe='@/:;?&=-_.,+!*'))) + for index, attr in enumerate(event): + if attr["to_ids"] is True: + if attr["type"] in fsmapping: + if attr["type"] == "ip-dst": + ipdst.append(BLOCK_DST_JSON_TMPL.format(ipdst=attr["value"])) + else: + urls.append(BLOCK_URL_JSON_TMPL.format(url=quote(attr["value"], safe='@/:;?&=-_.,+!*'))) + # building the .sh file + output += SH_FILE_HEADER + output += "FIRESIGHT_IP_ADDR='{}'\n".format(config['fmc_ip_addr']) - #building the .sh file - output += SH_FILE_HEADER - output += "FIRESIGHT_IP_ADDR='" + config['fmc_ip_addr'] + "'\n" + output += "LOGINPASS_BASE64=`echo -n '{}:{}' | base64`\n".format(config['fmc_login'], config['fmc_pass']) + output += "DOMAIN_ID='{}'\n".format(config['domain_id']) + output += "ACPOLICY_ID='{}'\n\n".format(config['acpolicy_id']) - output += "LOGINPASS_BASE64=`echo -n '" + config['fmc_login'] + ":" + config['fmc_pass'] + "' | base64`\n" - output += "DOMAIN_ID='" + config['domain_id'] + "'\n" - output += "ACPOLICY_ID='" + config['acpolicy_id'] + "'\n\n" - - output += "ACC_TOKEN=`curl -X POST -v -k -sD - -o /dev/null -H \"Authorization: Basic $LOGINPASS_BASE64\" -i \"https://$FIRESIGHT_IP_ADDR/api/fmc_platform/v1/auth/generatetoken\" | grep -i x-auth-acc | sed 's/.*:\\ //g' | tr -d '[:space:]' | tr -d '\\n'`" + "\n" + output += "ACC_TOKEN=`curl -X POST -v -k -sD - -o /dev/null -H \"Authorization: Basic $LOGINPASS_BASE64\" -i \"https://$FIRESIGHT_IP_ADDR/api/fmc_platform/v1/auth/generatetoken\" | grep -i x-auth-acc | sed 's/.*:\\ //g' | tr -d '[:space:]' | tr -d '\\n'`\n" - output += BLOCK_JSON_TMPL.format(rule_name="misp_event_"+event_id,dst_networks=', '.join(ipdst), urls=', '.join(urls), event_info_comment=event_info) + "\n" + output += BLOCK_JSON_TMPL.format(rule_name="misp_event_{}".format(event_id), + dst_networks=', '.join(ipdst), + urls=', '.join(urls), + event_info_comment=event_info) + "\n" - output += CURL_ADD_RULE_TMPL - # END building the .sh file + output += CURL_ADD_RULE_TMPL + # END building the .sh file - r = {"data":base64.b64encode(output.encode('utf-8')).decode('utf-8')} - return r + r = {"data": base64.b64encode(output.encode('utf-8')).decode('utf-8')} + return r def introspection(): - modulesetup = {} - try: - responseType - modulesetup['responseType'] = responseType - except NameError: - pass - try: - userConfig - modulesetup['userConfig'] = userConfig - except NameError: - pass - try: - outputFileExtension - modulesetup['outputFileExtension'] = outputFileExtension - except NameError: - pass - try: - inputSource - modulesetup['inputSource'] = inputSource - except NameError: - pass - return modulesetup + modulesetup = {} + try: + responseType + modulesetup['responseType'] = responseType + except NameError: + pass + try: + userConfig + modulesetup['userConfig'] = userConfig + except NameError: + pass + try: + outputFileExtension + modulesetup['outputFileExtension'] = outputFileExtension + except NameError: + pass + try: + inputSource + modulesetup['inputSource'] = inputSource + except NameError: + pass + return modulesetup def version(): - moduleinfo['config'] = moduleconfig - return moduleinfo + moduleinfo['config'] = moduleconfig + return moduleinfo From 639534f152f9e94f529bc0c8a55847ddcd1d00f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 16 Apr 2019 11:25:53 +0200 Subject: [PATCH 19/59] chg: Bump Dependencies. --- Pipfile.lock | 106 +++++++++++++++++++++++++-------------------------- 1 file changed, 53 insertions(+), 53 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index b5faca1..428ae11 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -111,10 +111,10 @@ }, "click-plugins": { "hashes": [ - "sha256:b1ee1ccc9421c73007fe290680d97984eb6eaf5f4512b7620c6aa46031d6cb6b", - "sha256:dfed74b5063546a137de99baaaf742b4de4337ad2b3e1df5ec7c8a256adc0847" + "sha256:46ab999744a9d831159c3411bb0c79346d94a444df9a3a3742e9ed63645f264b", + "sha256:5d262006d3222f5057fd81e1623d4443e41dcda5dc815c06b442aa3c02889fc8" ], - "version": "==1.0.4" + "version": "==1.1.1" }, "colorama": { "hashes": [ @@ -300,7 +300,7 @@ "pybgpranking": { "editable": true, "git": "https://github.com/D4-project/BGP-Ranking.git/", - "ref": "019ef1c40aad1e5bb5c5072c9a998c6a8f0271f3", + "ref": "86180c080ba5f83a5160a8295ebc263709aa4eba", "subdirectory": "client" }, "pydnstrails": { @@ -331,13 +331,13 @@ "pyipasnhistory": { "editable": true, "git": "https://github.com/D4-project/IPASN-History.git/", - "ref": "0c4f11792061417b77ca6e22d2ece18109d74c75", + "ref": "1009124c39e9cc3b362f0f9ce4d272817b9a5186", "subdirectory": "client" }, "pymisp": { "editable": true, "git": "https://github.com/MISP/PyMISP.git", - "ref": "64bcaad0e578129543cdffad532a232722615f6c" + "ref": "13445b51748524195bb15aec2c1c6176bcb6bd95" }, "pyonyphe": { "editable": true, @@ -346,10 +346,10 @@ }, "pyparsing": { "hashes": [ - "sha256:66c9268862641abcac4a96ba74506e594c884e3f57690a696d21ad8210ed667a", - "sha256:f6c5ef0d7480ad048c054c37632c67fca55299990fff127850181659eea33fc3" + "sha256:1873c03321fc118f4e9746baf201ff990ceb915f433f23b395f5580d1840cb2a", + "sha256:9b6323ef4ab914af344ba97510e966d64ba91055d6b9afa6b30799340e89cc03" ], - "version": "==2.3.1" + "version": "==2.4.0" }, "pypdns": { "hashes": [ @@ -417,37 +417,37 @@ }, "reportlab": { "hashes": [ - "sha256:0135bc54a463db5315c93bba4182fb83dc088fefaa7da18784ecd2a0c4a9c068", - "sha256:09e167e01458ea1e0cf3acff634ae9ecc1f1757e7585060d039c90b762859cfd", - "sha256:0dfcea18ba3ca1fac55cb273d056a8a43a48bd04d419299b3267e1994c72455a", - "sha256:1a61e56593ea1a8a38135eedfb40f79dcad13164fff034313ebf2a30e200ca79", - "sha256:1bdd871c2087d3853a0e9a3a573b1a7535500f3341944b1e34e68f3213cd28b8", - "sha256:26878a4b9c45f046c635b5695681188c19806f08b04129ea01c9ed51c7754039", - "sha256:27c62264c758aa30113df105da816223d149e4e87ee778ad49469725b79be2eb", - "sha256:29a9dd3954465b9e4efb129ffda9ab3e6a4f06488e8aa2efd5aff8ad332f13c2", - "sha256:5740e3218ca98c1bc86bd2d2e2a8c1d23e7c97d949d6377ac30aaf449f01c363", - "sha256:605892bb3f822a1e7342ce2b461d645ab8e4d13875127c0ae5377f76853db422", - "sha256:6dacc72552bc0dd50286e856f09a5e646a007d9345598bf6f75b117a200bfd9d", - "sha256:7021b7c8ba6d8e69e4c68c9473067482aaa40b9094270b45dbf798fcb0e09bd4", - "sha256:8acd950dad5b20a417579d1253c1065222dde48f9412e71533b052ab3dd98632", - "sha256:8b8fb3b0dd1e2124aba24544a02c95bff1fffa966b0581f30abf4fb28e414005", - "sha256:920c61c942eb1cc446e1647a04978f4afe31993ed403b74576a018c3ca526394", - "sha256:928e8d99befe064e28e9a29a4fd9afcf2066dcd758b0903280e67e221527422a", - "sha256:a04787eee401a74c80b65e539b5fe9226fdeabe25caa3d216c21dc990b2f8a01", - "sha256:a5bb6bd7753cba854425fcf7ecf04627a17de78d47ef9e8fac615887c5658da3", - "sha256:a70d970619014dc83b4406bcfed7e2f9d5aaf5f521aad808f5560d90ea896fb4", - "sha256:ae468fe82c8af3d1987113f03c1f87d01daa5b4c85c1f10da126be84423a744d", - "sha256:b278d83a7f76410bd310b368309e6e4b19664ffa686abfa9f0696130b09c17d3", - "sha256:b6623e9a96db3edc4b384e036e67c7bc87bbd7e5dc2d72ce66efa0043f9383b0", - "sha256:dc15cfa577bb25f0a598d483cf6dcc5ecad576ba723fe9bec63b6ec720dab2a3", - "sha256:dffdb4f6b34ce791e67365f3f96ab3c45b4cdd2c70d212fac98fb146dc75ac80", - "sha256:e84020e3482856da733e1359cb7b84e6bac09179bd3af860e70468a9c3cb43e3", - "sha256:edda09668e8474d5acb1a37fb64599557b43a714f1469bd49a058e95b5b410ff", - "sha256:f77e9835873931d25f836a3c107e53e0f7d3c0b4906b13063815308cf5ca1fac", - "sha256:f91d16ff07d5d3c92303f64c6864d74d3b6a491dde186bfef90c58088f932998" + "sha256:1c228a3ac2c405f7fc16eac43ba92aec448bc25438902f30590ad021e8828097", + "sha256:2210fafd3bb06308a84876fe6d19172b645373edce2b6d7501378cb9c768f825", + "sha256:232fb2037b7c3df259685f1c5ecb7826f55742dc81f0713837b84a152307483e", + "sha256:2c4f25e63fa75f3064871cf435696a4e19b7bd4901d922b766ae58a447b5b6da", + "sha256:47951166d897b60e9e7ca349db82a2b689e6478ac6078e2c7c88ca8becbb0c7d", + "sha256:526ab1193ea8e97c4838135917890e66de5f777d04283008007229b139f3c094", + "sha256:5a9cc8470623ec5b76c7e59f56b7d1fcf0254896cd61842dbdbd278934cc50f4", + "sha256:5ddc1a4a74f225e35a7f60e2eae10de6878dddc9960dad2d9cadc49092f8850d", + "sha256:6b594f6d7d71bc5778e19adb1c699a598c69b9a7bcf97fa638d8762279f9d80a", + "sha256:6e8c89b46cfaf9ae40b7db87e9f29c9e5d32d18d25f9cd10d423a5241e8ec453", + "sha256:71f4f3e3975b91ddbfc1b36a537b46d07533ca7f31945e990a75db5f9bd7a0ba", + "sha256:763654dc346eeb66fa726a88d27f911339950d20a25303dfc098f3b59ba26614", + "sha256:7bae4b33363f44343e0fac5004c8e44576c3ed00885be4eee1f2260802c116c3", + "sha256:8a4b8a0fd0547f3b436b548284aa604ba183bfac26f41a7ffb23d0ff5db8c658", + "sha256:8b08d68e4cb498eabf85411beda5c32e591ef8d0a6d18c948c3f80ed5d2c6e31", + "sha256:9840f27948b54aefa3c6386e5ed0f124d641eb54fa2f2bc9aebcb270598487fc", + "sha256:9ae8f822370e47486ba1880f7580669058a41e64bdaa41019f4617317489f884", + "sha256:9db49197080646a113059eba1c0758161164de1bc57315e7422bbf8c86e03dcf", + "sha256:a08d23fa3f23f13a1cc6dca3b3c431d08ae48e52384e6bf47bbefb22fde58e61", + "sha256:ac111bc47733dbfa3e34d61282c91b69b1f66800b0c72b7b86dc2534faa09bef", + "sha256:bc3c69707c0bf9308193612d34ca87249d6fc91a35ce0873102321395d39024a", + "sha256:c375759a763c1c93d5b4f36620390440d9fa6dec6fcf88bce8234701d88b339c", + "sha256:c8a5988d73ec93a54f22660b64c5f3d2018163dd9ca4a5cdde8022a7e4fcb345", + "sha256:eba2bc7c28a3b2b0a3c24caff33e4d8708db008f480b03a6ea39c28661663746", + "sha256:ee187977d587b9b81929e08022f385eb11274efd75795d59d99eb23b3fa9b055", + "sha256:f3ef7616ffc27c150ffec61ac820739495f6a9ca5d8532047102756ebb27e8d1", + "sha256:f46f223fcae09c8bf2746b4eb2f351294faae04b262429cc480d34c69b133fd9", + "sha256:fd9f6429a68a246fb466696d97d1240752c889b5bfdc219fea15ae787cf366a6" ], "index": "pypi", - "version": "==3.5.17" + "version": "==3.5.19" }, "requests": { "hashes": [ @@ -466,10 +466,10 @@ }, "shodan": { "hashes": [ - "sha256:f93b7199e89eecf5c84647f66316c2c044c3aebfc1fe4d9caa43dfda07f74c4e" + "sha256:c30baebce853ad67677bf002dde96a1ca1a9729bdd300fbb3c5e5d889547a639" ], "index": "pypi", - "version": "==1.11.1" + "version": "==1.12.1" }, "sigmatools": { "hashes": [ @@ -487,10 +487,10 @@ }, "soupsieve": { "hashes": [ - "sha256:3aef141566afd07201b525c17bfaadd07580a8066f82b57f7c9417f26adbd0a3", - "sha256:e41a65e99bd125972d84221022beb1e4b5cfc68fa12c170c39834ce32d1b294c" + "sha256:6898e82ecb03772a0d82bd0d0a10c0d6dcc342f77e0701d0ec4a8271be465ece", + "sha256:b20eff5e564529711544066d7dc0f7661df41232ae263619dede5059799cdfca" ], - "version": "==1.9" + "version": "==1.9.1" }, "sparqlwrapper": { "hashes": [ @@ -555,12 +555,12 @@ }, "vulners": { "hashes": [ - "sha256:6617d5904b5369507bc34105071d312e9e1c38d73654505e7b15b9a3f1325915", - "sha256:8b05d12a9dd7cbc07198a13281299a6e014ec348522e214b1efd097e194b7568", - "sha256:a19b02e0a112d70951e10c5abc1993f7f029234212828e1b617ab35f4e460a24" + "sha256:6506f3ad45bf3fd72f9cd1ebd5fbc13f814e7cf62faed6897e33db949fe4584a", + "sha256:a0e86015343ecf1c3313f6101567749988b5eb5299a672f19fd2974121817444", + "sha256:f243fe025a84b85bd6f37e45d6cf693e24697d30661695fe5d29b652dff6a5a1" ], "index": "pypi", - "version": "==1.4.7" + "version": "==1.4.9" }, "wand": { "hashes": [ @@ -572,10 +572,10 @@ }, "xlsxwriter": { "hashes": [ - "sha256:de9ef46088489915eaaee00c7088cff93cf613e9990b46b933c98eb46f21b47f", - "sha256:df96eafc3136d9e790e35d6725b473e46ada6f585c1f6519da69b27f5c8873f7" + "sha256:3a4e4a24a6753f046dc5a5e5bc5f443fce6a18988486885a258db6963eb54163", + "sha256:92a2ba339ca939815f0e125fcde728e94ccdb3e97e1acd3275ecf25a3cacfdc6" ], - "version": "==1.1.5" + "version": "==1.1.6" }, "yara-python": { "hashes": [ @@ -760,11 +760,11 @@ }, "pytest": { "hashes": [ - "sha256:13c5e9fb5ec5179995e9357111ab089af350d788cbc944c628f3cde72285809b", - "sha256:f21d2f1fb8200830dcbb5d8ec466a9c9120e20d8b53c7585d180125cce1d297a" + "sha256:3773f4c235918987d51daf1db66d51c99fac654c81d6f2f709a046ab446d5e5d", + "sha256:b7802283b70ca24d7119b32915efa7c409982f59913c1a6c0640aacf118b95f5" ], "index": "pypi", - "version": "==4.4.0" + "version": "==4.4.1" }, "requests": { "hashes": [ From eefa35c65db9915af7e7a47a6ca4ba80ac29742a Mon Sep 17 00:00:00 2001 From: Evert0x <35029353+Evert0x@users.noreply.github.com> Date: Thu, 18 Apr 2019 00:23:38 +0200 Subject: [PATCH 20/59] Create cuckoo_submit.py --- .../modules/expansion/cuckoo_submit.py | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 misp_modules/modules/expansion/cuckoo_submit.py diff --git a/misp_modules/modules/expansion/cuckoo_submit.py b/misp_modules/modules/expansion/cuckoo_submit.py new file mode 100644 index 0000000..d0cb236 --- /dev/null +++ b/misp_modules/modules/expansion/cuckoo_submit.py @@ -0,0 +1,62 @@ +import json +import requests + +misperrors = {'error': 'Error'} +mispattributes = { + 'input': ['url'], + 'output': ['text'] +} + +# possible module-types: 'expansion', 'hover' or both +moduleinfo = {'version': '1', 'author': 'Evert Kors', + 'description': 'MODULE_DESCRIPTION', + 'module-type': ['expansion', 'hover']} + +# config fields that your code expects from the site admin +moduleconfig = ['cuckoo_api'] + + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + config = request.get('config') + if config is None: + misperrors['error'] = 'config is missing' + return misperrors + + cuck = config.get('cuckoo_api') + if cuck is None: + misperrors['error'] = 'cuckoo api url is missing' + return misperrors + + # The url to submit + url = request.get('url') + + HEADERS = {"Authorization": "Bearer S4MPL3"} + + urls = [ + url + ] + + try: + r = requests.post( + "%s/tasks/create/submit" % (cuck), + headers=HEADERS, + data={"strings": "\n".join(urls)} + ) + except Exception as e: + misperrors['error'] = str(e) + return misperrors + + r = {'results': [{'types': "text", 'values': "cool"}]} + return r + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo From e243edb50341899a4feedbd90ec3bd4e5df5fd00 Mon Sep 17 00:00:00 2001 From: Evert0x <35029353+Evert0x@users.noreply.github.com> Date: Thu, 18 Apr 2019 14:25:05 +0200 Subject: [PATCH 21/59] Update __init__.py --- misp_modules/modules/expansion/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py index 42f74a3..df83ca9 100644 --- a/misp_modules/modules/expansion/__init__.py +++ b/misp_modules/modules/expansion/__init__.py @@ -1,6 +1,6 @@ from . import _vmray # noqa -__all__ = ['vmray_submit', 'bgpranking', 'circl_passivedns', 'circl_passivessl', +__all__ = ['cuckoo_submit', 'vmray_submit', 'bgpranking', 'circl_passivedns', 'circl_passivessl', 'countrycode', 'cve', 'dns', 'btc_steroids', 'domaintools', 'eupi', 'farsight_passivedns', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal', 'whois', 'shodan', 'reversedns', 'geoip_country', 'wiki', 'iprep', From 49acb5374538a4fe1bafbb4af44d422f8ca0287d Mon Sep 17 00:00:00 2001 From: Ricardo van Zutphen Date: Fri, 19 Apr 2019 14:06:35 +0200 Subject: [PATCH 22/59] Update Cuckoo module to support files and URLs --- .../modules/expansion/cuckoo_submit.py | 151 ++++++++++++++---- 1 file changed, 119 insertions(+), 32 deletions(-) diff --git a/misp_modules/modules/expansion/cuckoo_submit.py b/misp_modules/modules/expansion/cuckoo_submit.py index d0cb236..e368fbd 100644 --- a/misp_modules/modules/expansion/cuckoo_submit.py +++ b/misp_modules/modules/expansion/cuckoo_submit.py @@ -1,56 +1,143 @@ +import base64 +import io import json +import logging import requests +import sys +import urllib.parse +import zipfile -misperrors = {'error': 'Error'} +from requests.exceptions import RequestException + +log = logging.getLogger('cuckoo_submit') +log.setLevel(logging.DEBUG) +sh = logging.StreamHandler(sys.stdout) +sh.setLevel(logging.DEBUG) +fmt = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s' +) +sh.setFormatter(fmt) +log.addHandler(sh) + +moduleinfo = { + "version": "0.1", 'author': "Evert Kors", + "description": "Submit files and URLs to Cuckoo Sandbox", + "module-type": ["expansion", "hover"] +} +misperrors = {"error": "Error"} +moduleconfig = ["cuckoo_api", "api_key"] mispattributes = { - 'input': ['url'], - 'output': ['text'] + "input": ["attachment', 'malware-sample", "url", "domain"], + "output": ["text"] } -# possible module-types: 'expansion', 'hover' or both -moduleinfo = {'version': '1', 'author': 'Evert Kors', - 'description': 'MODULE_DESCRIPTION', - 'module-type': ['expansion', 'hover']} -# config fields that your code expects from the site admin -moduleconfig = ['cuckoo_api'] +class APIKeyError(RequestException): + """Raised if the Cuckoo API returns a 401. This means no or an invalid + bearer token was supplied.""" + pass + + +class CuckooAPI(object): + + def __init__(self, api_url, api_key=""): + self.api_key = api_key + if not api_url.startswith("http"): + api_url = "https://{}".format(api_url) + + self.api_url = api_url + + def _post_api(self, endpoint, files=None, data={}): + data.update({ + "owner": "MISP" + }) + + try: + response = requests.post( + urllib.parse.urljoin(self.api_url, endpoint), + files=files, data=data, + headers={"Authorization: Bearer {}".format(self.api_key)} + ) + except RequestException as e: + log.error("Failed to submit sample to Cuckoo Sandbox. %s", e) + return None + + if response.status_code == 401: + raise APIKeyError("Invalid or no Cuckoo Sandbox API key provided") + + return response.json() + + def create_task(self, filename, fp): + response = self._post_api( + "/tasks/create/file", files={"file": (filename, fp)} + ) + if not response: + return False + + return response["task_id"] + + def create_url(self, url): + response = self._post_api( + "/tasks/create/url", data={"url": url} + ) + if not response: + return False + + return response["task_id"] def handler(q=False): if q is False: return False + request = json.loads(q) - config = request.get('config') - if config is None: - misperrors['error'] = 'config is missing' + + # See if the API URL was provided. The API key is optional, as it can + # be disabled in the Cuckoo API settings. + api_url = request["config"].get("api_url") + api_key = request["config"].get("api_key", "") + if not api_url: + misperrors["error"] = "No Cuckoo API URL provided" return misperrors - cuck = config.get('cuckoo_api') - if cuck is None: - misperrors['error'] = 'cuckoo api url is missing' - return misperrors + url = request.get("url") or request.get("domain") + data = request.get("data") + filename = None + if data: + data = base64.b64decode(data) - # The url to submit - url = request.get('url') + if "malware-sample" in request: + filename = request.get("malware-sample").split("|", 1)[0] + with zipfile.ZipFile(io.BytesIO(data)) as zipf: + data = zipf.read(zipf.namelist()[0], pwd=b"infected") - HEADERS = {"Authorization": "Bearer S4MPL3"} - - urls = [ - url - ] + elif "attachment" in request: + filename = request.get("attachment") + cuckoo_api = CuckooAPI(api_url=api_url, api_key=api_key) + task_id = None try: - r = requests.post( - "%s/tasks/create/submit" % (cuck), - headers=HEADERS, - data={"strings": "\n".join(urls)} - ) - except Exception as e: - misperrors['error'] = str(e) + if url: + log.debug("Submitting URL to Cuckoo Sandbox %s", api_url) + task_id = cuckoo_api.create_url(url) + elif data and filename: + log.debug("Submitting file to Cuckoo Sandbox %s", api_url) + task_id = cuckoo_api.create_task( + filename=filename, fp=io.BytesIO(data) + ) + except APIKeyError as e: + misperrors["error"] = "Failed to submit to Cuckoo: {}".format(e) return misperrors - r = {'results': [{'types': "text", 'values': "cool"}]} - return r + if not task_id: + misperrors["error"] = "File or URL submission failed" + return misperrors + + return { + "results": [ + {"types": "text", "values": "Cuckoo task id: {}".format(task_id)} + ] + } def introspection(): From e6326185d568575a200147a2866aa8cd9769ac28 Mon Sep 17 00:00:00 2001 From: Ricardo van Zutphen Date: Fri, 19 Apr 2019 16:24:30 +0200 Subject: [PATCH 23/59] Use double quotes and provide headers correctly --- .../modules/expansion/cuckoo_submit.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/misp_modules/modules/expansion/cuckoo_submit.py b/misp_modules/modules/expansion/cuckoo_submit.py index e368fbd..c1ded90 100644 --- a/misp_modules/modules/expansion/cuckoo_submit.py +++ b/misp_modules/modules/expansion/cuckoo_submit.py @@ -9,25 +9,25 @@ import zipfile from requests.exceptions import RequestException -log = logging.getLogger('cuckoo_submit') +log = logging.getLogger("cuckoo_submit") log.setLevel(logging.DEBUG) sh = logging.StreamHandler(sys.stdout) sh.setLevel(logging.DEBUG) fmt = logging.Formatter( - '%(asctime)s - %(name)s - %(levelname)s - %(message)s' + "%(asctime)s - %(name)s - %(levelname)s - %(message)s" ) sh.setFormatter(fmt) log.addHandler(sh) moduleinfo = { - "version": "0.1", 'author': "Evert Kors", + "version": "0.1", "author": "Evert Kors", "description": "Submit files and URLs to Cuckoo Sandbox", "module-type": ["expansion", "hover"] } misperrors = {"error": "Error"} -moduleconfig = ["cuckoo_api", "api_key"] +moduleconfig = ["api_url", "api_key"] mispattributes = { - "input": ["attachment', 'malware-sample", "url", "domain"], + "input": ["attachment", "malware-sample", "url", "domain"], "output": ["text"] } @@ -56,7 +56,7 @@ class CuckooAPI(object): response = requests.post( urllib.parse.urljoin(self.api_url, endpoint), files=files, data=data, - headers={"Authorization: Bearer {}".format(self.api_key)} + headers={"Authorization": "Bearer {}".format(self.api_key)} ) except RequestException as e: log.error("Failed to submit sample to Cuckoo Sandbox. %s", e) @@ -65,6 +65,10 @@ class CuckooAPI(object): if response.status_code == 401: raise APIKeyError("Invalid or no Cuckoo Sandbox API key provided") + if response.status_code != 200: + log.error("Invalid Cuckoo API response") + return None + return response.json() def create_task(self, filename, fp): @@ -145,5 +149,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo From 7fefbd2a4c074c43b73d803f6c87491aa564996c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 19 Apr 2019 22:41:06 +0200 Subject: [PATCH 24/59] chg: Bump dependencies Fix CVE-2019-11324 (urllib3) --- Pipfile.lock | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index 428ae11..4a5c092 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -300,7 +300,7 @@ "pybgpranking": { "editable": true, "git": "https://github.com/D4-project/BGP-Ranking.git/", - "ref": "86180c080ba5f83a5160a8295ebc263709aa4eba", + "ref": "4e0741056bcc0077de1120b8724a31330b26033e", "subdirectory": "client" }, "pydnstrails": { @@ -331,13 +331,13 @@ "pyipasnhistory": { "editable": true, "git": "https://github.com/D4-project/IPASN-History.git/", - "ref": "1009124c39e9cc3b362f0f9ce4d272817b9a5186", + "ref": "c0c2bbf8d70811982dad065ea463a7e01593a38d", "subdirectory": "client" }, "pymisp": { "editable": true, "git": "https://github.com/MISP/PyMISP.git", - "ref": "13445b51748524195bb15aec2c1c6176bcb6bd95" + "ref": "921f414e0e026a3a4b77112012cf930242a33b04" }, "pyonyphe": { "editable": true, @@ -459,10 +459,10 @@ }, "requests-cache": { "hashes": [ - "sha256:e9270030becc739b0a7f7f834234c73a878b2d794122bf76f40055a22419eb67", - "sha256:fe561ca119879bbcfb51f03a35e35b425e18f338248e59fd5cf2166c77f457a2" + "sha256:6822f788c5ee248995c4bfbd725de2002ad710182ba26a666e85b64981866060", + "sha256:73a7211870f7d67af5fd81cad2f67cfe1cd3eb4ee6a85155e07613968cc72dfc" ], - "version": "==0.4.13" + "version": "==0.5.0" }, "shodan": { "hashes": [ @@ -494,12 +494,12 @@ }, "sparqlwrapper": { "hashes": [ - "sha256:2a95fdede2833be660b81092934c4a0054ff85f2693098556762a2759ea486f1", - "sha256:7f4c8d38ea1bfcffbc358c9a05de35a3fd7152cc3e8ea57963ee7a0a242f7a5e", - "sha256:acf6d60f0a3684cb673653b07871acb0c350a974b891f20f8ac94926ff9eb2ff" + "sha256:14ec551f0d60b4a496ffcc31f15337e844c085b8ead8cbe9a7178748a6de3794", + "sha256:21928e7a97f565e772cdeeb0abad428960f4307e3a13dbdd8f6d3da8a6a506c9", + "sha256:abc3e7eadcad32fa69a85c003853e2f6f73bda6cc999853838f401a5a1ea1109" ], "index": "pypi", - "version": "==1.8.2" + "version": "==1.8.4" }, "stix2-patterns": { "hashes": [ @@ -542,10 +542,10 @@ }, "urllib3": { "hashes": [ - "sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39", - "sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22" + "sha256:4c291ca23bbb55c76518905869ef34bdd5f0e46af7afe6861e8375643ffee1a0", + "sha256:9a247273df709c4fedb38c711e44292304f73f39ab01beda9f6b9fc375669ac3" ], - "version": "==1.24.1" + "version": "==1.24.2" }, "uwhois": { "editable": true, @@ -783,10 +783,10 @@ }, "urllib3": { "hashes": [ - "sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39", - "sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22" + "sha256:4c291ca23bbb55c76518905869ef34bdd5f0e46af7afe6861e8375643ffee1a0", + "sha256:9a247273df709c4fedb38c711e44292304f73f39ab01beda9f6b9fc375669ac3" ], - "version": "==1.24.1" + "version": "==1.24.2" } } } From 5367bcd409e55eff39ecc498aa28ceab39988f6d Mon Sep 17 00:00:00 2001 From: Ricardo van Zutphen Date: Mon, 22 Apr 2019 22:38:03 +0200 Subject: [PATCH 25/59] Document Cuckoo expansion module --- doc/expansion/cuckoo_submit.json | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 doc/expansion/cuckoo_submit.json diff --git a/doc/expansion/cuckoo_submit.json b/doc/expansion/cuckoo_submit.json new file mode 100644 index 0000000..7fe8067 --- /dev/null +++ b/doc/expansion/cuckoo_submit.json @@ -0,0 +1,9 @@ +{ + "description": "An expansion module to submit files and URLs to Cuckoo Sandbox.", + "logo": "logos/cuckoo.png", + "requirements": ["Access to a Cuckoo Sandbox API and an API key if the API requires it. (api_url and api_key)"], + "input": "A malware-sample or attachment for files. A url or domain for URLs.", + "output": "A text field containing 'Cuckoo task id: '", + "references": ["https://cuckoosandbox.org/", "https://cuckoo.sh/docs/"], + "features": "The module takes a malware-sample, attachment, url or domain and submits it to Cuckoo Sandbox.\n The returned task id can be used to retrieve results when the analysis completed." +} From cafa1a6229649358ac04e2d1326bc07747628046 Mon Sep 17 00:00:00 2001 From: Ricardo van Zutphen Date: Mon, 22 Apr 2019 22:45:38 +0200 Subject: [PATCH 26/59] Generate latest version of documentation --- doc/README.md | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/doc/README.md b/doc/README.md index c32a8c4..02506ab 100644 --- a/doc/README.md +++ b/doc/README.md @@ -178,6 +178,25 @@ Module to query Crowdstrike Falcon. ----- +#### [cuckoo_submit](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/cuckoo_submit.py) + + + +An expansion module to submit files and URLs to Cuckoo Sandbox. +- **features**: +>The module takes a malware-sample, attachment, url or domain and submits it to Cuckoo Sandbox. +> The returned task id can be used to retrieve results when the analysis completed. +- **input**: +>A malware-sample or attachment for files. A url or domain for URLs. +- **output**: +>A text field containing 'Cuckoo task id: ' +- **references**: +>https://cuckoosandbox.org/, https://cuckoo.sh/docs/ +- **requirements**: +>Access to a Cuckoo Sandbox API and an API key if the API requires it. (api_url and api_key) + +----- + #### [cve](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/cve.py) @@ -1081,7 +1100,13 @@ OSQuery export of a MISP event. Simple export of a MISP event to PDF. - **features**: ->The module takes care of the PDF file building, and work with any MISP Event. Except the requirement of asciidoctor, used to create the file, there is no special feature concerning the Event. +>The module takes care of the PDF file building, and work with any MISP Event. Except the requirement of reportlab, used to create the file, there is no special feature concerning the Event. Some parameters can be given through the config dict. 'MISP_base_url_for_dynamic_link' is your MISP URL, to attach an hyperlink to your event on your MISP instance from the PDF. Keep it clear to avoid hyperlinks in the generated pdf. +> 'MISP_name_for_metadata' is your CERT or MISP instance name. Used as text in the PDF' metadata +> 'Activate_textual_description' is a boolean (True or void) to activate the textual description/header abstract of an event +> 'Activate_galaxy_description' is a boolean (True or void) to activate the description of event related galaxies. +> 'Activate_related_events' is a boolean (True or void) to activate the description of related event. Be aware this might leak information on confidential events linked to the current event ! +> 'Activate_internationalization_fonts' is a boolean (True or void) to activate Noto fonts instead of default fonts (Helvetica). This allows the support of CJK alphabet. Be sure to have followed the procedure to download Noto fonts (~70Mo) in the right place (/tools/pdf_fonts/Noto_TTF), to allow PyMisp to find and use them during PDF generation. +> 'Custom_fonts_path' is a text (path or void) to the TTF file of your choice, to create the PDF with it. Be aware the PDF won't support bold/italic/special style anymore with this option - **input**: >MISP Event - **output**: @@ -1089,7 +1114,7 @@ Simple export of a MISP event to PDF. - **references**: >https://acrobat.adobe.com/us/en/acrobat/about-adobe-pdf.html - **requirements**: ->PyMISP, asciidoctor +>PyMISP, reportlab ----- From c85ab8d93ca9485e5534cd779cc91e5ba8c9d2c8 Mon Sep 17 00:00:00 2001 From: Sascha Rommelfangen Date: Tue, 23 Apr 2019 11:38:56 +0200 Subject: [PATCH 27/59] initial version of QR code reader Module accepts attachments and processes pictures. It tries to identify and analyze an existing QR code. Identified values can be inserted into the event. --- misp_modules/modules/expansion/qrcode.py | 86 ++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 misp_modules/modules/expansion/qrcode.py diff --git a/misp_modules/modules/expansion/qrcode.py b/misp_modules/modules/expansion/qrcode.py new file mode 100644 index 0000000..5357e6d --- /dev/null +++ b/misp_modules/modules/expansion/qrcode.py @@ -0,0 +1,86 @@ +import json +from pyzbar import pyzbar +import cv2 +import re +import binascii +import np + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['attachment'], + 'output': ['url', 'btc']} +moduleinfo = {'version': '0.1', 'author': 'Sascha Rommelfangen', + 'description': 'QR code decoder', + 'module-type': ['expansion', 'hover']} + +debug = True +debug_prefix = "[DEBUG] QR Code module: " +# format example: bitcoin:1GXZ6v7FZzYBEnoRaG77SJxhu7QkvQmFuh?amount=0.15424 +# format example: http://example.com +cryptocurrencies = {'bitcoin'} +schemas = {'http://', 'https://', 'ftp://'} +moduleconfig = [] + +def handler(q=False): + if q is False: + return False + q = json.loads(q) + filename = q['attachment'] + try: + img_array = np.fromstring(binascii.a2b_base64(q['data']), np.uint8) + except: + err = "Couldn't fetch attachment (JSON 'data' is empty). Are you using the 'Query enrichment' action?" + misperrors['error'] = err + print(err) + return misperrors + image = cv2.imdecode(img_array, cv2.IMREAD_COLOR) + if q: + barcodes = pyzbar.decode(image) + for item in barcodes: + try: + result = item.data.decode() + except Exception as e: + print(e) + return + if debug: + print(debug_prefix + result) + for item in cryptocurrencies: + if item in result: + try: + currency, address, extra = re.split('\:|\?', result) + except Exception as e: + print(e) + if currency in cryptocurrencies: + try: + amount = re.split('=', extra)[1] + if debug: + print(debug_prefix + address) + print(debug_prefix + amount) + return {'results': [{'types': ['btc'], 'values': address, 'comment': "BTC: " + amount + " from file " + filename}]} + except Exception as e: + print(e) + else: + print(address) + for item in schemas: + if item in result: + try: + url = result + if debug: + print(debug_prefix + url) + return {'results': [{'types': ['url'], 'values': url, 'comment': "from QR code of file " + filename}]} + except Exception as e: + print(e) + else: + try: + return {'results': [{'types': ['text'], 'values': result, 'comment': "from QR code of file " + filename}]} + except Exception as e: + print(e) + misperrors['error'] = "Couldn't decode QR code in attachment." + return misperrors + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo From d5180e7e79584cf256ad88db1422f3b155445122 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Tue, 23 Apr 2019 14:37:27 +0200 Subject: [PATCH 28/59] chg: [qrcode] various fixes to make it PEP compliant --- misp_modules/modules/expansion/qrcode.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/misp_modules/modules/expansion/qrcode.py b/misp_modules/modules/expansion/qrcode.py index 5357e6d..82f4f88 100644 --- a/misp_modules/modules/expansion/qrcode.py +++ b/misp_modules/modules/expansion/qrcode.py @@ -20,6 +20,7 @@ cryptocurrencies = {'bitcoin'} schemas = {'http://', 'https://', 'ftp://'} moduleconfig = [] + def handler(q=False): if q is False: return False @@ -27,7 +28,7 @@ def handler(q=False): filename = q['attachment'] try: img_array = np.fromstring(binascii.a2b_base64(q['data']), np.uint8) - except: + except Exception as e: err = "Couldn't fetch attachment (JSON 'data' is empty). Are you using the 'Query enrichment' action?" misperrors['error'] = err print(err) @@ -46,7 +47,7 @@ def handler(q=False): for item in cryptocurrencies: if item in result: try: - currency, address, extra = re.split('\:|\?', result) + currency, address, extra = re.split('\:|\?', result) except Exception as e: print(e) if currency in cryptocurrencies: @@ -60,7 +61,7 @@ def handler(q=False): print(e) else: print(address) - for item in schemas: + for item in schemas: if item in result: try: url = result @@ -77,6 +78,7 @@ def handler(q=False): misperrors['error'] = "Couldn't decode QR code in attachment." return misperrors + def introspection(): return mispattributes From 44050ec4da5b830161b2f9397e7910309cf1bdd5 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Tue, 23 Apr 2019 14:44:00 +0200 Subject: [PATCH 29/59] chg: [qrcode] flake8 needs some drugs --- misp_modules/modules/expansion/qrcode.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/qrcode.py b/misp_modules/modules/expansion/qrcode.py index 82f4f88..437de60 100644 --- a/misp_modules/modules/expansion/qrcode.py +++ b/misp_modules/modules/expansion/qrcode.py @@ -32,6 +32,7 @@ def handler(q=False): err = "Couldn't fetch attachment (JSON 'data' is empty). Are you using the 'Query enrichment' action?" misperrors['error'] = err print(err) + print(e) return misperrors image = cv2.imdecode(img_array, cv2.IMREAD_COLOR) if q: @@ -47,7 +48,7 @@ def handler(q=False): for item in cryptocurrencies: if item in result: try: - currency, address, extra = re.split('\:|\?', result) + currency, address, extra = re.split(r'\:|\?', result) except Exception as e: print(e) if currency in cryptocurrencies: From e55ae11a1e126ad040eef1e17ae6836c770f2629 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Tue, 23 Apr 2019 14:45:12 +0200 Subject: [PATCH 30/59] chg: [qrcode] added to the __init__ --- misp_modules/modules/expansion/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py index 42f74a3..ff847a4 100644 --- a/misp_modules/modules/expansion/__init__.py +++ b/misp_modules/modules/expansion/__init__.py @@ -8,4 +8,4 @@ __all__ = ['vmray_submit', 'bgpranking', 'circl_passivedns', 'circl_passivessl', 'yara_syntax_validator', 'hashdd', 'onyphe', 'onyphe_full', 'rbl', 'xforceexchange', 'sigma_syntax_validator', 'stix2_pattern_syntax_validator', 'sigma_queries', 'dbl_spamhaus', 'vulners', 'yara_query', 'macaddress_io', - 'intel471', 'backscatter_io', 'btc_scam_check', 'hibp', 'greynoise', 'macvendors'] + 'intel471', 'backscatter_io', 'btc_scam_check', 'hibp', 'greynoise', 'macvendors', 'qrcode'] From 32430a15cb1794bb41596e5a4b80a2f2fce77109 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Tue, 23 Apr 2019 14:49:02 +0200 Subject: [PATCH 31/59] chg: [qrcode] add requirements --- Pipfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Pipfile b/Pipfile index 2f2d172..6a88fcc 100644 --- a/Pipfile +++ b/Pipfile @@ -42,6 +42,9 @@ misp-modules = {editable = true,path = "."} pybgpranking = {editable = true,git = "https://github.com/D4-project/BGP-Ranking.git/",subdirectory = "client"} pyipasnhistory = {editable = true,git = "https://github.com/D4-project/IPASN-History.git/",subdirectory = "client"} backscatter = "*" +pyzbar = "*" +opencv-python = "*" +np = "*" [requires] python_version = "3.6" From 5adb9bfcfab220f5db5af4f0ec89a9611984db8b Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Tue, 23 Apr 2019 14:54:05 +0200 Subject: [PATCH 32/59] chg: [doc] qrcode and Cisco FireSight added --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 18b5548..94a5778 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,7 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/ * [onyphe_full](misp_modules/modules/expansion/onyphe_full.py) - a modules to process full queries on Onyphe. * [OTX](misp_modules/modules/expansion/otx.py) - an expansion module for [OTX](https://otx.alienvault.com/). * [passivetotal](misp_modules/modules/expansion/passivetotal.py) - a [passivetotal](https://www.passivetotal.org/) module that queries a number of different PassiveTotal datasets. +* [qrcode](misp_modules/modules/expansion/qrcode.py) - a module decode QR code, barcode and similar codes from an image and enrich with the decoded values. * [rbl](misp_modules/modules/expansion/rbl.py) - a module to get RBL (Real-Time Blackhost List) values from an attribute. * [reversedns](misp_modules/modules/expansion/reversedns.py) - Simple Reverse DNS expansion service to resolve reverse DNS from MISP attributes. * [securitytrails](misp_modules/modules/expansion/securitytrails.py) - an expansion module for [securitytrails](https://securitytrails.com/). @@ -68,6 +69,7 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/ ### Export modules * [CEF](misp_modules/modules/export_mod/cef_export.py) module to export Common Event Format (CEF). +* [Cisco FireSight Manager ACL rule](misp_modules/modules/export_mod/cisco_firesight_manager_ACL_rule_export.py) module to export as rule for the Cisco FireSight manager ACL. * [GoAML export](misp_modules/modules/export_mod/goamlexport.py) module to export in [GoAML format](http://goaml.unodc.org/goaml/en/index.html). * [Lite Export](misp_modules/modules/export_mod/liteexport.py) module to export a lite event. * [PDF export](misp_modules/modules/export_mod/pdfexport.py) module to export an event in PDF. From 8acbb1762d07669d41ab9215f563507535a288c1 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Tue, 23 Apr 2019 14:59:42 +0200 Subject: [PATCH 33/59] chg: [travis] because everyone need a bar --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index e8fea8e..b0d73ed 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,7 @@ python: - "3.7-dev" install: + - apt-get install libzbar0 libzbar-dev - pip install pipenv - pipenv install --dev From 72cd5e3c1f0ccfae85e30f3644cef578d1239f06 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Tue, 23 Apr 2019 15:02:32 +0200 Subject: [PATCH 34/59] chg: [travis] because we all need sudo --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b0d73ed..52a7742 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,7 @@ python: - "3.7-dev" install: - - apt-get install libzbar0 libzbar-dev + - sudo apt-get install libzbar0 libzbar-dev - pip install pipenv - pipenv install --dev From 2d8aaf09c20849b1ce6f251dfe07417341b0d29a Mon Sep 17 00:00:00 2001 From: Sascha Rommelfangen Date: Tue, 23 Apr 2019 15:40:22 +0200 Subject: [PATCH 35/59] brackets are difficult... --- misp_modules/modules/expansion/qrcode.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/qrcode.py b/misp_modules/modules/expansion/qrcode.py index 437de60..9a62827 100644 --- a/misp_modules/modules/expansion/qrcode.py +++ b/misp_modules/modules/expansion/qrcode.py @@ -16,8 +16,8 @@ debug = True debug_prefix = "[DEBUG] QR Code module: " # format example: bitcoin:1GXZ6v7FZzYBEnoRaG77SJxhu7QkvQmFuh?amount=0.15424 # format example: http://example.com -cryptocurrencies = {'bitcoin'} -schemas = {'http://', 'https://', 'ftp://'} +cryptocurrencies = ['bitcoin'] +schemas = ['http://', 'https://', 'ftp://'] moduleconfig = [] From b787aa7961976f560c638db9eff3223a6be07d29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 23 Apr 2019 17:02:21 +0200 Subject: [PATCH 36/59] chg: Require python3 instead of python 3.6 --- Pipfile | 2 +- Pipfile.lock | 99 ++++++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 90 insertions(+), 11 deletions(-) diff --git a/Pipfile b/Pipfile index 6a88fcc..ba650fe 100644 --- a/Pipfile +++ b/Pipfile @@ -47,4 +47,4 @@ opencv-python = "*" np = "*" [requires] -python_version = "3.6" +python_version = "3" diff --git a/Pipfile.lock b/Pipfile.lock index 4a5c092..3191d4b 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,11 +1,11 @@ { "_meta": { "hash": { - "sha256": "23dec0fa6400c828e294ea9981b433903c17358ca61d7abdaec8df5a1c89f08c" + "sha256": "7fee9399d8a7151a79b6f8bbce64564062fd562b0a091fd45a875884d3fb954e" }, "pipfile-spec": 6, "requires": { - "python_version": "3.6" + "python_version": "3" }, "sources": [ { @@ -158,9 +158,10 @@ }, "httplib2": { "hashes": [ - "sha256:4ba6b8fd77d0038769bf3c33c9a96a6f752bc4cdf739701fdcaf210121f399d4" + "sha256:23914b5487dfe8ef09db6656d6d63afb0cf3054ad9ebc50868ddc8e166b5f8e8", + "sha256:a18121c7c72a56689efbf1aef990139ad940fee1e64c6f2458831736cd593600" ], - "version": "==0.12.1" + "version": "==0.12.3" }, "idna": { "hashes": [ @@ -236,6 +237,41 @@ ], "version": "==4.5.2" }, + "np": { + "hashes": [ + "sha256:781265283f3823663ad8fb48741aae62abcf4c78bc19f908f8aa7c1d3eb132f8" + ], + "index": "pypi", + "version": "==1.0.2" + }, + "numpy": { + "hashes": [ + "sha256:0e2eed77804b2a6a88741f8fcac02c5499bba3953ec9c71e8b217fad4912c56c", + "sha256:1c666f04553ef70fda54adf097dbae7080645435fc273e2397f26bbf1d127bbb", + "sha256:1f46532afa7b2903bfb1b79becca2954c0a04389d19e03dc73f06b039048ac40", + "sha256:315fa1b1dfc16ae0f03f8fd1c55f23fd15368710f641d570236f3d78af55e340", + "sha256:3d5fcea4f5ed40c3280791d54da3ad2ecf896f4c87c877b113576b8280c59441", + "sha256:48241759b99d60aba63b0e590332c600fc4b46ad597c9b0a53f350b871ef0634", + "sha256:4b4f2924b36d857cf302aec369caac61e43500c17eeef0d7baacad1084c0ee84", + "sha256:54fe3b7ed9e7eb928bbc4318f954d133851865f062fa4bbb02ef8940bc67b5d2", + "sha256:5a8f021c70e6206c317974c93eaaf9bc2b56295b6b1cacccf88846e44a1f33fc", + "sha256:754a6be26d938e6ca91942804eb209307b73f806a1721176278a6038869a1686", + "sha256:771147e654e8b95eea1293174a94f34e2e77d5729ad44aefb62fbf8a79747a15", + "sha256:78a6f89da87eeb48014ec652a65c4ffde370c036d780a995edaeb121d3625621", + "sha256:7fde5c2a3a682a9e101e61d97696687ebdba47637611378b4127fe7e47fdf2bf", + "sha256:80d99399c97f646e873dd8ce87c38cfdbb668956bbc39bc1e6cac4b515bba2a0", + "sha256:88a72c1e45a0ae24d1f249a529d9f71fe82e6fa6a3fd61414b829396ec585900", + "sha256:a4f4460877a16ac73302a9c077ca545498d9fe64e6a81398d8e1a67e4695e3df", + "sha256:a61255a765b3ac73ee4b110b28fccfbf758c985677f526c2b4b39c48cc4b509d", + "sha256:ab4896a8c910b9a04c0142871d8800c76c8a2e5ff44763513e1dd9d9631ce897", + "sha256:abbd6b1c2ef6199f4b7ca9f818eb6b31f17b73a6110aadc4e4298c3f00fab24e", + "sha256:b16d88da290334e33ea992c56492326ea3b06233a00a1855414360b77ca72f26", + "sha256:b78a1defedb0e8f6ae1eb55fa6ac74ab42acc4569c3a2eacc2a407ee5d42ebcb", + "sha256:cfef82c43b8b29ca436560d51b2251d5117818a8d1fb74a8384a83c096745dad", + "sha256:d160e57731fcdec2beda807ebcabf39823c47e9409485b5a3a1db3a8c6ce763e" + ], + "version": "==1.16.3" + }, "oauth2": { "hashes": [ "sha256:15b5c42301f46dd63113f1214b0d81a8b16254f65a86d3c32a1b52297f3266e6", @@ -244,6 +280,39 @@ "index": "pypi", "version": "==1.9.0.post1" }, + "opencv-python": { + "hashes": [ + "sha256:1703a296a96d3d46615e5053f224867977accb4240bcaa0fcabcb0768bf5ac13", + "sha256:1777ce7535ee7a1995cae168a107a1320e9df13648b930e72a1a2c2eccd64cda", + "sha256:1e5520482fb18fbd64d079e7f17ac0018f195fd75f6360a53bb82d7903106b50", + "sha256:25522dcf2529614750a71112a6659759080b4bdc2323f19d47f4d895960fd796", + "sha256:2af5f2842ad44c65ae2647377e0ff198719e1a1cfc9c6a19bc0c525c035d4bd8", + "sha256:31ec48d7eca13fc25c287dea7cecab453976e372cad8f50d55c054a247efda21", + "sha256:47cf48ff5dbd554e9f58cc9e98cf0b5de3f6a971172612bffa06bc5fb79ce872", + "sha256:494f98366bb5d6c2ac7e50e6617139f353704fd97a6d12ec9d392e72817d5cb0", + "sha256:4a9845870739e640e3350a8d98d511c92c087fe3d66090e83be7bf94e0ac64f7", + "sha256:4ac29cc0847d948a6636899014e84e165c30cc8779d6218394d44363462a01ce", + "sha256:5857ace03b7854221abf8072462d306c2c2ce4e366190b21d90ee8ee8aaf5bb4", + "sha256:5b4a23d99d5a2874767034466f5a8fd37b9f93ac14955a01b1a208983c76b9ad", + "sha256:734d87a5021c037064beb62133e135e66c7128e401a63b8b842b809ae2093749", + "sha256:78005c1c5d15ef4e32e0f485557bd15b5b6d87f49c19db7fe3e9246a61ebe7e4", + "sha256:81ae2283225c5c52fc3d72debd4241c30ccff2bb922578bf7867f9851cce3acb", + "sha256:88dbf900f297fdae0f62b899d6a784d8868ec2135854c5f8a9abbad00a6f0c5b", + "sha256:8c98ea7b8d327a31cd6028782a06147d0e0329ae8e829e881fb5d02f7ed8aec9", + "sha256:937d4686fef6967921145290f5b50c01c00c5b5d3542a6519e8a85cd88448723", + "sha256:a057958c0e362b3c4f03b9af1cbdb6d5af035fd22ecd7fd794eba8fdeb049eb8", + "sha256:c41eab31fa2c641226c6187caa391a688d064c99f078d604574f1912296b771f", + "sha256:cf4f7e62d1f80d1fa85a1693a3500def5cde54b2b75212b3609e552e4c25acfb", + "sha256:d90d60143e18334330c149f293071c9f2f3c79c896f33dc4ec65099e58baaaa7", + "sha256:db3106b7ca86999a7bd1f2fcc93e49314e5e6e451356774e421a69428df5020b", + "sha256:dbaf264db56f4771dfac6624f438bc4dc670aa94f61a6138848fcab7e9e77380", + "sha256:e65206c4cf651dc9cf0829962fae8bec986767c9f123d6a1ad17f9356bf7257e", + "sha256:eac94ddc78c58e891cff7180274317dad2938a4ddfc6ced1c04846c7f50e77e9", + "sha256:f2e828711f044a965509c862b3a59b3181e9c56c145a950cb53d43fec54e66d2" + ], + "index": "pypi", + "version": "==4.1.0.25" + }, "passivetotal": { "hashes": [ "sha256:d745a6519ec04e3a354682978ebf07778bf7602beac30307cbad075ff1a4418d" @@ -401,6 +470,15 @@ ], "version": "==5.1" }, + "pyzbar": { + "hashes": [ + "sha256:0e204b904e093e5e75aa85e0203bb0e02888105732a509b51f31cff400f34265", + "sha256:496249b546be70ec98c0ff0ad9151e73daaffff129266df86150a15dcd8dac4c", + "sha256:7d6c01d2c0a352fa994aa91b5540d1caeaeaac466656eb41468ca5df33be9f2e" + ], + "index": "pypi", + "version": "==0.1.8" + }, "rdflib": { "hashes": [ "sha256:58d5994610105a457cff7fdfe3d683d87786c5028a45ae032982498a7e913d6f", @@ -564,18 +642,19 @@ }, "wand": { "hashes": [ - "sha256:91810d241ab0851d40e67c946beb960b869c4f4160c397eac291ec6283ee3e3f", - "sha256:ae7c0958509a22f531b7b97e93adfd3f1208f0ac1c593af9e5f0cffa4ac06d5b" + "sha256:63ab24dee0264a44f5f045d4ecc0d392bc1cc195e5a2f80ce537b2c205c3033b", + "sha256:a2c318993791fab4fcfd460045415176f81d42f8c6fd8a88fb8d74d2f0f34b97", + "sha256:f68f32f2e4eca663a361d36148f06372de560442dcf8c785a53a64ee282572c9" ], "index": "pypi", - "version": "==0.5.2" + "version": "==0.5.3" }, "xlsxwriter": { "hashes": [ - "sha256:3a4e4a24a6753f046dc5a5e5bc5f443fce6a18988486885a258db6963eb54163", - "sha256:92a2ba339ca939815f0e125fcde728e94ccdb3e97e1acd3275ecf25a3cacfdc6" + "sha256:2a40b427dac0f640031e5b33abe97e761de6e0f12d4d346e7b2e2b67cf6ee927", + "sha256:431edc9ba1132eec1996939aa83fffe41885d3042ab09d47c3086f41a156c430" ], - "version": "==1.1.6" + "version": "==1.1.7" }, "yara-python": { "hashes": [ From 4631c17286dacc52c71eda700a26a2403162b85b Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Tue, 23 Apr 2019 19:49:58 +0200 Subject: [PATCH 37/59] chg: [doc] cuckoo_submit module added --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 94a5778..081120d 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/ * [countrycode](misp_modules/modules/expansion/countrycode.py) - a hover module to tell you what country a URL belongs to. * [CrowdStrike Falcon](misp_modules/modules/expansion/crowdstrike_falcon.py) - an expansion module to expand using CrowdStrike Falcon Intel Indicator API. * [CVE](misp_modules/modules/expansion/cve.py) - a hover module to give more information about a vulnerability (CVE). +* [Cuckoo submit](misp_modules/modules/expansion/cuckoo_submit.py) - A hover module to submit malware sample, url, attachment, domain to Cuckoo Sandbox. * [DBL Spamhaus](misp_modules/modules/expansion/dbl_spamhaus.py) - a hover module to check Spamhaus DBL for a domain name. * [DNS](misp_modules/modules/expansion/dns.py) - a simple module to resolve MISP attributes like hostname and domain to expand IP addresses attributes. * [DomainTools](misp_modules/modules/expansion/domaintools.py) - a hover and expansion module to get information from [DomainTools](http://www.domaintools.com/) whois. From e893a17583a36c84d8dd6deb112eaf8c41476eb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 24 Apr 2019 11:29:16 +0200 Subject: [PATCH 38/59] chg: Bump dependencies, update REQUIREMENTS file --- Pipfile.lock | 8 ++++---- REQUIREMENTS | 47 +++++++++++++++++++++++++---------------------- 2 files changed, 29 insertions(+), 26 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index 3191d4b..8a073a9 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -633,12 +633,12 @@ }, "vulners": { "hashes": [ - "sha256:6506f3ad45bf3fd72f9cd1ebd5fbc13f814e7cf62faed6897e33db949fe4584a", - "sha256:a0e86015343ecf1c3313f6101567749988b5eb5299a672f19fd2974121817444", - "sha256:f243fe025a84b85bd6f37e45d6cf693e24697d30661695fe5d29b652dff6a5a1" + "sha256:146ef130f215b50cdff790b06b4886c7edb325c075e9fce4bf1d3ab8d64a10d0", + "sha256:53406a86126159eaee9575fa667c99459bfdf9dd8c06bd0ce73fbe536b305e30", + "sha256:a258ccdbaee586207bc80d3590f0315ff151cfe16ea54f2e1629a6018fd9f2a3" ], "index": "pypi", - "version": "==1.4.9" + "version": "==1.5.0" }, "wand": { "hashes": [ diff --git a/REQUIREMENTS b/REQUIREMENTS index 99e1c02..b9d198a 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -1,9 +1,9 @@ -i https://pypi.org/simple -e . --e git+https://github.com/D4-project/BGP-Ranking.git/@37c97ae252ec4bf1d67733a49d4895c8cb009cf9#egg=pybgpranking&subdirectory=client --e git+https://github.com/D4-project/IPASN-History.git/@e846cd36fe1ed6b22f60890bba89f84e61b62e59#egg=pyipasnhistory&subdirectory=client +-e git+https://github.com/D4-project/BGP-Ranking.git/@4e0741056bcc0077de1120b8724a31330b26033e#egg=pybgpranking&subdirectory=client +-e git+https://github.com/D4-project/IPASN-History.git/@c0c2bbf8d70811982dad065ea463a7e01593a38d#egg=pyipasnhistory&subdirectory=client -e git+https://github.com/MISP/PyIntel471.git@0df8d51f1c1425de66714b3a5a45edb69b8cc2fc#egg=pyintel471 --e git+https://github.com/MISP/PyMISP.git@b8759673b91e733c307698abdc0d5ed82fd7e0de#egg=pymisp +-e git+https://github.com/MISP/PyMISP.git@921f414e0e026a3a4b77112012cf930242a33b04#egg=pymisp -e git+https://github.com/Rafiot/uwhoisd.git@411572840eba4c72dc321c549b36a54ed5cea9de#egg=uwhois&subdirectory=client -e git+https://github.com/sebdraven/pydnstrails@48c1f740025c51289f43a24863d1845ff12fd21a#egg=pydnstrails -e git+https://github.com/sebdraven/pyonyphe@cbb0168d5cb28a9f71f7ab3773164a7039ccdb12#egg=pyonyphe @@ -16,7 +16,7 @@ beautifulsoup4==4.7.1 blockchain==1.4.4 certifi==2019.3.9 chardet==3.0.4 -click-plugins==1.0.4 +click-plugins==1.1.1 click==7.0 colorama==0.4.1 dnspython==1.16.0 @@ -24,44 +24,47 @@ domaintools-api==0.3.3 enum-compat==0.0.2 ez-setup==0.9 future==0.17.1 -httplib2==0.12.1 -idna-ssl==1.1.0 ; python_version < '3.7' +httplib2==0.12.3 idna==2.8 isodate==0.6.0 jsonschema==3.0.1 maclookup==1.0.3 multidict==4.5.2 +np==1.0.2 +numpy==1.16.3 oauth2==1.9.0.post1 +opencv-python==4.1.0.25 passivetotal==1.0.30 -pillow==5.4.1 -psutil==5.6.0 +pillow==6.0.0 +psutil==5.6.1 pyeupi==1.0 pygeoip==0.3.2 -pyparsing==2.3.1 +pyparsing==2.4.0 pypdns==1.3 pypssl==2.1 pyrsistent==0.14.11 pytesseract==0.2.6 python-dateutil==2.8.0 -pyyaml==3.13 +pyyaml==5.1 +pyzbar==0.1.8 rdflib==4.2.2 -redis==3.2.0 -reportlab==3.5.13 -requests-cache==0.4.13 +redis==3.2.1 +reportlab==3.5.19 +requests-cache==0.5.0 requests==2.21.0 -shodan==1.11.1 -sigmatools==0.9 +shodan==1.12.1 +sigmatools==0.10 six==1.12.0 -soupsieve==1.8 -sparqlwrapper==1.8.2 +soupsieve==1.9.1 +sparqlwrapper==1.8.4 stix2-patterns==1.1.0 tabulate==0.8.3 -tornado==6.0.1 +tornado==6.0.2 url-normalize==1.4.1 urlarchiver==0.2 -urllib3==1.24.1 -vulners==1.4.5 -wand==0.5.1 -xlsxwriter==1.1.5 +urllib3==1.24.2 +vulners==1.5.0 +wand==0.5.3 +xlsxwriter==1.1.7 yara-python==3.8.1 yarl==1.3.0 From 7171c8ce92432e7850b8f5d6a51420c3543a36b0 Mon Sep 17 00:00:00 2001 From: Sascha Rommelfangen Date: Wed, 24 Apr 2019 13:54:21 +0200 Subject: [PATCH 39/59] initial version of OCR expansion module --- misp_modules/modules/expansion/__init__.py | 3 +- misp_modules/modules/expansion/ocr.py | 51 ++++++++++++++++++++++ 2 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 misp_modules/modules/expansion/ocr.py diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py index 994b289..ec78e9b 100644 --- a/misp_modules/modules/expansion/__init__.py +++ b/misp_modules/modules/expansion/__init__.py @@ -8,4 +8,5 @@ __all__ = ['cuckoo_submit', 'vmray_submit', 'bgpranking', 'circl_passivedns', 'c 'yara_syntax_validator', 'hashdd', 'onyphe', 'onyphe_full', 'rbl', 'xforceexchange', 'sigma_syntax_validator', 'stix2_pattern_syntax_validator', 'sigma_queries', 'dbl_spamhaus', 'vulners', 'yara_query', 'macaddress_io', - 'intel471', 'backscatter_io', 'btc_scam_check', 'hibp', 'greynoise', 'macvendors', 'qrcode'] + 'intel471', 'backscatter_io', 'btc_scam_check', 'hibp', 'greynoise', 'macvendors', + 'qrcode', 'ocr'] diff --git a/misp_modules/modules/expansion/ocr.py b/misp_modules/modules/expansion/ocr.py new file mode 100644 index 0000000..afdf343 --- /dev/null +++ b/misp_modules/modules/expansion/ocr.py @@ -0,0 +1,51 @@ +import json +import re +import binascii +import cv2 +import np +import pytesseract + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['attachment'], + 'output': ['freetext', 'text']} +moduleinfo = {'version': '0.1', 'author': 'Sascha Rommelfangen', + 'description': 'OCR decoder', + 'module-type': ['expansion']} + +moduleconfig = [] + + +def handler(q=False): + if q is False: + return False + q = json.loads(q) + filename = q['attachment'] + try: + img_array = np.frombuffer(binascii.a2b_base64(q['data']), np.uint8) + except Exception as e: + print(e) + err = "Couldn't fetch attachment (JSON 'data' is empty). Are you using the 'Query enrichment' action?" + misperrors['error'] = err + print(err) + return misperrors + + image = img_array + image = cv2.imdecode(img_array, cv2.IMREAD_COLOR) + try: + decoded = pytesseract.image_to_string(image) + return {'results': [{'types': ['freetext'], 'values': decoded, 'comment': "OCR from file " + filename}, + {'types': ['text'], 'values': decoded, 'comment': "ORC from file " + filename}]} + except Exception as e: + print(e) + err = "Couldn't analyze file type. Only images are supported right now." + misperrors['error'] = err + return misperrors + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo From 614fc1354b49bf281b746d3c6ca0bbbffcc21bff Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Wed, 24 Apr 2019 14:01:08 +0200 Subject: [PATCH 40/59] chg: [ocr] re module not used - removed --- misp_modules/modules/expansion/ocr.py | 1 - 1 file changed, 1 deletion(-) diff --git a/misp_modules/modules/expansion/ocr.py b/misp_modules/modules/expansion/ocr.py index afdf343..cd6baca 100644 --- a/misp_modules/modules/expansion/ocr.py +++ b/misp_modules/modules/expansion/ocr.py @@ -1,5 +1,4 @@ import json -import re import binascii import cv2 import np From 81b0082ae5cb2a42303c4629dd53af954713cb25 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Wed, 24 Apr 2019 14:01:48 +0200 Subject: [PATCH 41/59] chg: [init] removed trailing whitespace --- misp_modules/modules/expansion/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py index ec78e9b..03a7899 100644 --- a/misp_modules/modules/expansion/__init__.py +++ b/misp_modules/modules/expansion/__init__.py @@ -8,5 +8,5 @@ __all__ = ['cuckoo_submit', 'vmray_submit', 'bgpranking', 'circl_passivedns', 'c 'yara_syntax_validator', 'hashdd', 'onyphe', 'onyphe_full', 'rbl', 'xforceexchange', 'sigma_syntax_validator', 'stix2_pattern_syntax_validator', 'sigma_queries', 'dbl_spamhaus', 'vulners', 'yara_query', 'macaddress_io', - 'intel471', 'backscatter_io', 'btc_scam_check', 'hibp', 'greynoise', 'macvendors', + 'intel471', 'backscatter_io', 'btc_scam_check', 'hibp', 'greynoise', 'macvendors', 'qrcode', 'ocr'] From 5104bce4519d994aa868f9b576b803e393ca1ce7 Mon Sep 17 00:00:00 2001 From: Sascha Rommelfangen Date: Wed, 24 Apr 2019 14:53:03 +0200 Subject: [PATCH 42/59] renamed module --- misp_modules/modules/expansion/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py index ec78e9b..38d59ad 100644 --- a/misp_modules/modules/expansion/__init__.py +++ b/misp_modules/modules/expansion/__init__.py @@ -9,4 +9,4 @@ __all__ = ['cuckoo_submit', 'vmray_submit', 'bgpranking', 'circl_passivedns', 'c 'xforceexchange', 'sigma_syntax_validator', 'stix2_pattern_syntax_validator', 'sigma_queries', 'dbl_spamhaus', 'vulners', 'yara_query', 'macaddress_io', 'intel471', 'backscatter_io', 'btc_scam_check', 'hibp', 'greynoise', 'macvendors', - 'qrcode', 'ocr'] + 'qrcode', 'ocr-enrich'] From 07f759b07a5994e2fea4410a4a230e2304159009 Mon Sep 17 00:00:00 2001 From: Sascha Rommelfangen Date: Wed, 24 Apr 2019 14:53:16 +0200 Subject: [PATCH 43/59] renamed file --- misp_modules/modules/expansion/{ocr.py => ocr-enrich.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename misp_modules/modules/expansion/{ocr.py => ocr-enrich.py} (100%) diff --git a/misp_modules/modules/expansion/ocr.py b/misp_modules/modules/expansion/ocr-enrich.py similarity index 100% rename from misp_modules/modules/expansion/ocr.py rename to misp_modules/modules/expansion/ocr-enrich.py From 29e57dfcc6afa59b3d68d5ccb3c65ee649001a09 Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Thu, 25 Apr 2019 17:36:32 +0900 Subject: [PATCH 44/59] chg: [doc] Added new dependencies and updated RHEL/CentOS howto. --- README.md | 39 ++++++++++++++------------------------- 1 file changed, 14 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 169151e..64d0960 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,7 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/ ## How to install and start MISP modules in a Python virtualenv? (recommended) ~~~~bash -sudo apt-get install python3-dev python3-pip libpq5 libjpeg-dev tesseract-ocr imagemagick virtualenv +sudo apt-get install python3-dev python3-pip libpq5 libjpeg-dev tesseract-ocr imagemagick virtualenv libopencv-dev zbar-tools sudo -u www-data virtualenv -p python3 /var/www/MISP/venv cd /usr/local/src/ sudo git clone https://github.com/MISP/misp-modules.git @@ -107,36 +107,24 @@ sudo systemctl enable --now misp-modules /var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s & #to start the modules ~~~~ -## How to install and start MISP modules on Debian-based distributions ? - -~~~~bash -sudo apt-get install python3-dev python3-pip libpq5 libjpeg-dev tesseract-ocr imagemagick -cd /usr/local/src/ -sudo git clone https://github.com/MISP/misp-modules.git -cd misp-modules -sudo pip3 install -I -r REQUIREMENTS -sudo pip3 install -I . -# Start misp-modules as a service -sudo cp etc/systemd/system/misp-modules.service /etc/systemd/system/ -sudo systemctl daemon-reload -sudo systemctl enable --now misp-modules -/var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s & #to start the modules -~~~~ - ## How to install and start MISP modules on RHEL-based distributions ? As of this writing, the official RHEL repositories only contain Ruby 2.0.0 and Ruby 2.1 or higher is required. As such, this guide installs Ruby 2.2 from the [SCL](https://access.redhat.com/documentation/en-us/red_hat_software_collections/3/html/3.2_release_notes/chap-installation#sect-Installation-Subscribe) repository. + ~~~~bash -yum install rh-ruby22 +sudo yum install rh-ruby22 +sudo yum install openjpeg-devel +sudo yum install rubygem-rouge rubygem-asciidoctor zbar-devel opencv-devel cd /var/www/MISP git clone https://github.com/MISP/misp-modules.git cd misp-modules -scl enable rh-python36 ‘python3 –m pip install cryptography’ -scl enable rh-python36 ‘python3 –m pip install -I -r REQUIREMENTS’ -scl enable rh-python36 ‘python3 –m pip install –I .’ +sudo -u apache /usr/bin/scl enable rh-python36 "virtualenv -p python3 /var/www/MISP/venv" +sudo -u apache /var/www/MISP/venv/bin/pip install -U -I -r REQUIREMENTS +sudo -u apache /var/www/MISP/venv/bin/pip install -U . ~~~~ + Create the service file /etc/systemd/system/misp-modules.service : ~~~~ -[Unit] +echo "[Unit] Description=MISP's modules After=misp-workers.service @@ -144,15 +132,16 @@ After=misp-workers.service Type=simple User=apache Group=apache -ExecStart=/usr/bin/scl enable rh-python36 rh-ruby22 ‘/opt/rh/rh-python36/root/bin/misp-modules –l 127.0.0.1 –s’ +ExecStart=/usr/bin/scl enable rh-python36 rh-ruby22 '/opt/rh/rh-python36/root/bin/misp-modules –l 127.0.0.1 –s' Restart=always RestartSec=10 [Install] -WantedBy=multi-user.target +WantedBy=multi-user.target" | sudo tee /etc/systemd/system/misp-modules.service ~~~~ + The `After=misp-workers.service` must be changed or removed if you have not created a misp-workers service. -Then, enable the misp-modules service and start it ; +Then, enable the misp-modules service and start it: ~~~~bash systemctl daemon-reload systemctl enable --now misp-modules From 2c64e5ca67ee37e9d18131a8ef1e8b4c50f360b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 26 Apr 2019 11:35:03 +0200 Subject: [PATCH 45/59] fix: CTRL+C is working again Fix #292 --- misp_modules/__init__.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/misp_modules/__init__.py b/misp_modules/__init__.py index d933dc9..68a2c69 100644 --- a/misp_modules/__init__.py +++ b/misp_modules/__init__.py @@ -55,7 +55,7 @@ log = logging.getLogger('misp-modules') def handle_signal(sig, frame): - IOLoop.instance().add_callback(IOLoop.instance().stop) + IOLoop.instance().add_callback_from_signal(IOLoop.instance().stop) def init_logger(level=False): @@ -266,8 +266,11 @@ def main(): if args.t: log.info('MISP modules started in test-mode, quitting immediately.') sys.exit() - IOLoop.instance().start() - IOLoop.instance().stop() + try: + IOLoop.instance().start() + finally: + IOLoop.instance().stop() + return 0 From c825cabbbe104fe3b3151f98047c1942a1825079 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 26 Apr 2019 11:40:25 +0200 Subject: [PATCH 46/59] chg: Bump dependencies --- Pipfile.lock | 91 ++++++++++++++++++++++++---------------------------- REQUIREMENTS | 8 ++--- 2 files changed, 46 insertions(+), 53 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index 8a073a9..5d395d5 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -170,13 +170,6 @@ ], "version": "==2.8" }, - "idna-ssl": { - "hashes": [ - "sha256:a933e3bb13da54383f9e8f35dc4f9cb9eb9b3b78c6b36f311254d6d0d92c6c7c" - ], - "markers": "python_version < '3.7'", - "version": "==1.1.0" - }, "isodate": { "hashes": [ "sha256:2e364a3d5759479cdb2d37cce6b9376ea504db2ff90252a2e5b7cc89cc9ff2d8", @@ -354,17 +347,17 @@ }, "psutil": { "hashes": [ - "sha256:23e9cd90db94fbced5151eaaf9033ae9667c033dffe9e709da761c20138d25b6", - "sha256:27858d688a58cbfdd4434e1c40f6c79eb5014b709e725c180488ccdf2f721729", - "sha256:354601a1d1a1322ae5920ba397c58d06c29728a15113598d1a8158647aaa5385", - "sha256:9c3a768486194b4592c7ae9374faa55b37b9877fd9746fb4028cb0ac38fd4c60", - "sha256:c1fd45931889dc1812ba61a517630d126f6185f688eac1693171c6524901b7de", - "sha256:d463a142298112426ebd57351b45c39adb41341b91f033aa903fa4c6f76abecc", - "sha256:e1494d20ffe7891d07d8cb9a8b306c1a38d48b13575265d090fc08910c56d474", - "sha256:ec4b4b638b84d42fc48139f9352f6c6587ee1018d55253542ee28db7480cc653", - "sha256:fa0a570e0a30b9dd618bffbece590ae15726b47f9f1eaf7518dfb35f4d7dcd21" + "sha256:206eb909aa8878101d0eca07f4b31889c748f34ed6820a12eb3168c7aa17478e", + "sha256:649f7ffc02114dced8fbd08afcd021af75f5f5b2311bc0e69e53e8f100fe296f", + "sha256:6ebf2b9c996bb8c7198b385bade468ac8068ad8b78c54a58ff288cd5f61992c7", + "sha256:753c5988edc07da00dafd6d3d279d41f98c62cd4d3a548c4d05741a023b0c2e7", + "sha256:76fb0956d6d50e68e3f22e7cc983acf4e243dc0fcc32fd693d398cb21c928802", + "sha256:828e1c3ca6756c54ac00f1427fdac8b12e21b8a068c3bb9b631a1734cada25ed", + "sha256:a4c62319ec6bf2b3570487dd72d471307ae5495ce3802c1be81b8a22e438b4bc", + "sha256:acba1df9da3983ec3c9c963adaaf530fcb4be0cd400a8294f1ecc2db56499ddd", + "sha256:ef342cb7d9b60e6100364f50c57fa3a77d02ff8665d5b956746ac01901247ac4" ], - "version": "==5.6.1" + "version": "==5.6.2" }, "pybgpranking": { "editable": true, @@ -406,7 +399,7 @@ "pymisp": { "editable": true, "git": "https://github.com/MISP/PyMISP.git", - "ref": "921f414e0e026a3a4b77112012cf930242a33b04" + "ref": "582dda0ce2a8ca8e1dd2cf3842e0491caca51c62" }, "pyonyphe": { "editable": true, @@ -436,9 +429,9 @@ }, "pyrsistent": { "hashes": [ - "sha256:3ca82748918eb65e2d89f222b702277099aca77e34843c5eb9d52451173970e2" + "sha256:5403d37f4d55ff4572b5b5676890589f367a9569529c6f728c11046c4ea4272b" ], - "version": "==0.14.11" + "version": "==0.15.1" }, "pytesseract": { "hashes": [ @@ -495,37 +488,37 @@ }, "reportlab": { "hashes": [ - "sha256:1c228a3ac2c405f7fc16eac43ba92aec448bc25438902f30590ad021e8828097", - "sha256:2210fafd3bb06308a84876fe6d19172b645373edce2b6d7501378cb9c768f825", - "sha256:232fb2037b7c3df259685f1c5ecb7826f55742dc81f0713837b84a152307483e", - "sha256:2c4f25e63fa75f3064871cf435696a4e19b7bd4901d922b766ae58a447b5b6da", - "sha256:47951166d897b60e9e7ca349db82a2b689e6478ac6078e2c7c88ca8becbb0c7d", - "sha256:526ab1193ea8e97c4838135917890e66de5f777d04283008007229b139f3c094", - "sha256:5a9cc8470623ec5b76c7e59f56b7d1fcf0254896cd61842dbdbd278934cc50f4", - "sha256:5ddc1a4a74f225e35a7f60e2eae10de6878dddc9960dad2d9cadc49092f8850d", - "sha256:6b594f6d7d71bc5778e19adb1c699a598c69b9a7bcf97fa638d8762279f9d80a", - "sha256:6e8c89b46cfaf9ae40b7db87e9f29c9e5d32d18d25f9cd10d423a5241e8ec453", - "sha256:71f4f3e3975b91ddbfc1b36a537b46d07533ca7f31945e990a75db5f9bd7a0ba", - "sha256:763654dc346eeb66fa726a88d27f911339950d20a25303dfc098f3b59ba26614", - "sha256:7bae4b33363f44343e0fac5004c8e44576c3ed00885be4eee1f2260802c116c3", - "sha256:8a4b8a0fd0547f3b436b548284aa604ba183bfac26f41a7ffb23d0ff5db8c658", - "sha256:8b08d68e4cb498eabf85411beda5c32e591ef8d0a6d18c948c3f80ed5d2c6e31", - "sha256:9840f27948b54aefa3c6386e5ed0f124d641eb54fa2f2bc9aebcb270598487fc", - "sha256:9ae8f822370e47486ba1880f7580669058a41e64bdaa41019f4617317489f884", - "sha256:9db49197080646a113059eba1c0758161164de1bc57315e7422bbf8c86e03dcf", - "sha256:a08d23fa3f23f13a1cc6dca3b3c431d08ae48e52384e6bf47bbefb22fde58e61", - "sha256:ac111bc47733dbfa3e34d61282c91b69b1f66800b0c72b7b86dc2534faa09bef", - "sha256:bc3c69707c0bf9308193612d34ca87249d6fc91a35ce0873102321395d39024a", - "sha256:c375759a763c1c93d5b4f36620390440d9fa6dec6fcf88bce8234701d88b339c", - "sha256:c8a5988d73ec93a54f22660b64c5f3d2018163dd9ca4a5cdde8022a7e4fcb345", - "sha256:eba2bc7c28a3b2b0a3c24caff33e4d8708db008f480b03a6ea39c28661663746", - "sha256:ee187977d587b9b81929e08022f385eb11274efd75795d59d99eb23b3fa9b055", - "sha256:f3ef7616ffc27c150ffec61ac820739495f6a9ca5d8532047102756ebb27e8d1", - "sha256:f46f223fcae09c8bf2746b4eb2f351294faae04b262429cc480d34c69b133fd9", - "sha256:fd9f6429a68a246fb466696d97d1240752c889b5bfdc219fea15ae787cf366a6" + "sha256:13714baa9753bfca94df67716cccb3eedcaaa30cf7bc40b282d338a718e0b610", + "sha256:16c1bb717a1a0e2ed065aa31eb5968dc03b34b728926216ef282cefeebf50c1b", + "sha256:2d9d66770880e8d112b6b925458593d34b84947c355847578cd974df0a3e3b8b", + "sha256:3334a30e477e1dfa0276eb41ed5bfd2a684c9917e55c6acb30d91abac46555f6", + "sha256:33796ea88d20c05958903c11ff34d896e462381f4a0f550854aabe6dd07cc189", + "sha256:5184f53c0babeedb4ebe297eb97794822cb122456ca03411c68256730c998d48", + "sha256:53589c5db35041920cd7a92a171506ff4eb5542ab8415af272fe4558927399a8", + "sha256:58ba0a9ca51d666d55ec7ecd83ab14763b79e7e5e0775b7717694e94c2fbbf18", + "sha256:6998652beba357da9687eba13b46ceccd0a7a2153d656cf8a03b7494c915e077", + "sha256:6c3b07c8a94ee9609c31a51e4131891c8330ffd379db23ab582fd225a06a4e59", + "sha256:7b248d2d9d4ab6d4cad91eb2b153b2c4c7b3fced89cb5a5b5bfbc7d09593871a", + "sha256:81d991c9994a576ea053b281b8c9afc28b12261197d478e72055d381f60fa26f", + "sha256:8a9a8be6841b88b13aa9c0f7d193c6d24b04b10c2e7cbf6657b1807bac5b1e9f", + "sha256:8de3107436e68014890adcec446207fd98d60c26e7feae6f550eea9eab3a622d", + "sha256:90f85afb68f7cd4fd8681af3123d23488274e4d1c3fea8d3831ef7257d9733c8", + "sha256:94857052c951ffa56de95cfce483fdf3de19587db4b1bc4f6a4043fb1c4af772", + "sha256:a47603d9b975e8870ed30ade22db3478b030dd7a6041c8272c3719d9bbeaef34", + "sha256:a5671b152d3f85963d8450e956ddecfb5d30af62dd1f73207fab9aa32a0240d2", + "sha256:a745cd1a4368fac093deff3b65614f052eced6afa9ed7fe223da2a52813f2e23", + "sha256:af454e8e844e3eeace5aead02701748b2a908f3e8cbc386cc5ddc185cef8c57f", + "sha256:c3c6c1234eed451111e969c776688e866554cb362250b88f782ab80ea62f9114", + "sha256:cc1cf8ba1b2f1669f5d873a7cfdb9e07a920243d74a66a78f8afa2cf78587864", + "sha256:cce3c9b0e115ea5553615a647b6644e5724bdc776e778593ffa5f383d907afb2", + "sha256:d137feacef83627d10adb869aa6998f29eb7af4cff3101c9fc94a1d73943b6cc", + "sha256:d7213814d050ca3b0bf7e018f94ed947e90477cd36aff298ff5932b849a0f36a", + "sha256:e381d08675807d2bb465717f69818040173351650af82730d721ecad429279a6", + "sha256:e39c6efdd64027be56ce991f7ffb86c7cee47da8c844c3544bbd68ef842831a0", + "sha256:f8526cfbbad599d22de6eb59b5a43610ba9b28f74ac2406125fe803f31a262a6" ], "index": "pypi", - "version": "==3.5.19" + "version": "==3.5.20" }, "requests": { "hashes": [ diff --git a/REQUIREMENTS b/REQUIREMENTS index b9d198a..37c5a68 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -3,7 +3,7 @@ -e git+https://github.com/D4-project/BGP-Ranking.git/@4e0741056bcc0077de1120b8724a31330b26033e#egg=pybgpranking&subdirectory=client -e git+https://github.com/D4-project/IPASN-History.git/@c0c2bbf8d70811982dad065ea463a7e01593a38d#egg=pyipasnhistory&subdirectory=client -e git+https://github.com/MISP/PyIntel471.git@0df8d51f1c1425de66714b3a5a45edb69b8cc2fc#egg=pyintel471 --e git+https://github.com/MISP/PyMISP.git@921f414e0e026a3a4b77112012cf930242a33b04#egg=pymisp +-e git+https://github.com/MISP/PyMISP.git@582dda0ce2a8ca8e1dd2cf3842e0491caca51c62#egg=pymisp -e git+https://github.com/Rafiot/uwhoisd.git@411572840eba4c72dc321c549b36a54ed5cea9de#egg=uwhois&subdirectory=client -e git+https://github.com/sebdraven/pydnstrails@48c1f740025c51289f43a24863d1845ff12fd21a#egg=pydnstrails -e git+https://github.com/sebdraven/pyonyphe@cbb0168d5cb28a9f71f7ab3773164a7039ccdb12#egg=pyonyphe @@ -36,20 +36,20 @@ oauth2==1.9.0.post1 opencv-python==4.1.0.25 passivetotal==1.0.30 pillow==6.0.0 -psutil==5.6.1 +psutil==5.6.2 pyeupi==1.0 pygeoip==0.3.2 pyparsing==2.4.0 pypdns==1.3 pypssl==2.1 -pyrsistent==0.14.11 +pyrsistent==0.15.1 pytesseract==0.2.6 python-dateutil==2.8.0 pyyaml==5.1 pyzbar==0.1.8 rdflib==4.2.2 redis==3.2.1 -reportlab==3.5.19 +reportlab==3.5.20 requests-cache==0.5.0 requests==2.21.0 shodan==1.12.1 From f55d7946df192032f6b8b8a7a0e78efd30547f14 Mon Sep 17 00:00:00 2001 From: Sascha Rommelfangen Date: Fri, 26 Apr 2019 12:07:55 +0200 Subject: [PATCH 47/59] introduction of new modules --- misp_modules/modules/expansion/docx-enrich.py | 61 +++++++++++++++++++ misp_modules/modules/expansion/ods-enrich.py | 56 +++++++++++++++++ misp_modules/modules/expansion/odt-enrich.py | 51 ++++++++++++++++ misp_modules/modules/expansion/pdf-enrich.py | 50 +++++++++++++++ misp_modules/modules/expansion/pptx-enrich.py | 55 +++++++++++++++++ misp_modules/modules/expansion/xlsx-enrich.py | 53 ++++++++++++++++ 6 files changed, 326 insertions(+) create mode 100644 misp_modules/modules/expansion/docx-enrich.py create mode 100644 misp_modules/modules/expansion/ods-enrich.py create mode 100644 misp_modules/modules/expansion/odt-enrich.py create mode 100644 misp_modules/modules/expansion/pdf-enrich.py create mode 100644 misp_modules/modules/expansion/pptx-enrich.py create mode 100644 misp_modules/modules/expansion/xlsx-enrich.py diff --git a/misp_modules/modules/expansion/docx-enrich.py b/misp_modules/modules/expansion/docx-enrich.py new file mode 100644 index 0000000..6380557 --- /dev/null +++ b/misp_modules/modules/expansion/docx-enrich.py @@ -0,0 +1,61 @@ +import json +import binascii +import np +import docx +import io + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['attachment'], + 'output': ['freetext', 'text']} +moduleinfo = {'version': '0.1', 'author': 'Sascha Rommelfangen', + 'description': '.docx to freetext-import IOC extractor', + 'module-type': ['expansion']} + +moduleconfig = [] + + +def handler(q=False): + if q is False: + return False + q = json.loads(q) + filename = q['attachment'] + try: + docx_array = np.frombuffer(binascii.a2b_base64(q['data']), np.uint8) + except Exception as e: + print(e) + err = "Couldn't fetch attachment (JSON 'data' is empty). Are you using the 'Query enrichment' action?" + misperrors['error'] = err + print(err) + return misperrors + + doc_content = "" + doc_file = io.BytesIO(docx_array) + try: + doc = docx.Document(doc_file) + for para in doc.paragraphs: + print(para.text) + doc_content = doc_content + "\n" + para.text + tables = doc.tables + for table in tables: + for row in table.rows: + for cell in row.cells: + for para in cell.paragraphs: + print(para.text) + doc_content = doc_content + "\n" + para.text + print(doc_content) + return {'results': [{'types': ['freetext'], 'values': doc_content, 'comment': ".docx-to-text from file " + filename}, + {'types': ['text'], 'values': doc_content, 'comment': ".docx-to-text from file " + filename}]} + except Exception as e: + print(e) + err = "Couldn't analyze file as .docx. Error was: " + str(e) + misperrors['error'] = err + return misperrors + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/expansion/ods-enrich.py b/misp_modules/modules/expansion/ods-enrich.py new file mode 100644 index 0000000..06d7ff5 --- /dev/null +++ b/misp_modules/modules/expansion/ods-enrich.py @@ -0,0 +1,56 @@ +import json +import binascii +import np +import ezodf +import pandas_ods_reader +import io + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['attachment'], + 'output': ['freetext', 'text']} +moduleinfo = {'version': '0.1', 'author': 'Sascha Rommelfangen', + 'description': '.ods to freetext-import IOC extractor', + 'module-type': ['expansion']} + +moduleconfig = [] + + +def handler(q=False): + if q is False: + return False + q = json.loads(q) + filename = q['attachment'] + try: + ods_array = np.frombuffer(binascii.a2b_base64(q['data']), np.uint8) + except Exception as e: + print(e) + err = "Couldn't fetch attachment (JSON 'data' is empty). Are you using the 'Query enrichment' action?" + misperrors['error'] = err + print(err) + return misperrors + + ods_content = "" + ods_file = io.BytesIO(ods_array) + doc = ezodf.opendoc(ods_file) + num_sheets = len(doc.sheets) + try: + for i in range(0, num_sheets): + ods = pandas_ods_reader.read_ods(ods_file, i, headers=False) + ods_content = ods_content + "\n" + ods.to_string(max_rows=None) + print(ods_content) + return {'results': [{'types': ['freetext'], 'values': ods_content, 'comment': ".ods-to-text from file " + filename}, + {'types': ['text'], 'values': ods_content, 'comment': ".ods-to-text from file " + filename}]} + except Exception as e: + print(e) + err = "Couldn't analyze file as .ods. Error was: " + str(e) + misperrors['error'] = err + return misperrors + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/expansion/odt-enrich.py b/misp_modules/modules/expansion/odt-enrich.py new file mode 100644 index 0000000..6dac555 --- /dev/null +++ b/misp_modules/modules/expansion/odt-enrich.py @@ -0,0 +1,51 @@ +import json +import binascii +import np +from ODTReader.odtreader import odtToText +import io + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['attachment'], + 'output': ['freetext', 'text']} +moduleinfo = {'version': '0.1', 'author': 'Sascha Rommelfangen', + 'description': '.odt to freetext-import IOC extractor', + 'module-type': ['expansion']} + +moduleconfig = [] + + +def handler(q=False): + if q is False: + return False + q = json.loads(q) + filename = q['attachment'] + try: + odt_array = np.frombuffer(binascii.a2b_base64(q['data']), np.uint8) + except Exception as e: + print(e) + err = "Couldn't fetch attachment (JSON 'data' is empty). Are you using the 'Query enrichment' action?" + misperrors['error'] = err + print(err) + return misperrors + + odt_content = "" + odt_file = io.BytesIO(odt_array) + try: + odt_content = odtToText(odt_file) + print(odt_content) + return {'results': [{'types': ['freetext'], 'values': odt_content, 'comment': ".odt-to-text from file " + filename}, + {'types': ['text'], 'values': odt_content, 'comment': ".odt-to-text from file " + filename}]} + except Exception as e: + print(e) + err = "Couldn't analyze file as .odt. Error was: " + str(e) + misperrors['error'] = err + return misperrors + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/expansion/pdf-enrich.py b/misp_modules/modules/expansion/pdf-enrich.py new file mode 100644 index 0000000..aeb5d9b --- /dev/null +++ b/misp_modules/modules/expansion/pdf-enrich.py @@ -0,0 +1,50 @@ +import json +import binascii +import np +import pytesseract +import pdftotext +import io +import collections + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['attachment'], + 'output': ['freetext', 'text']} +moduleinfo = {'version': '0.1', 'author': 'Sascha Rommelfangen', + 'description': 'PDF to freetext-import IOC extractor', + 'module-type': ['expansion']} + +moduleconfig = [] + + +def handler(q=False): + if q is False: + return False + q = json.loads(q) + filename = q['attachment'] + try: + pdf_array = np.frombuffer(binascii.a2b_base64(q['data']), np.uint8) + except Exception as e: + print(e) + err = "Couldn't fetch attachment (JSON 'data' is empty). Are you using the 'Query enrichment' action?" + misperrors['error'] = err + print(err) + return misperrors + + pdf_file = io.BytesIO(pdf_array) + try: + pdf_content = "\n\n".join(pdftotext.PDF(pdf_file)) + return {'results': [{'types': ['freetext'], 'values': pdf_content, 'comment': "PDF-to-text from file " + filename}]} + except Exception as e: + print(e) + err = "Couldn't analyze file as PDF. Error was: " + str(e) + misperrors['error'] = err + return misperrors + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/expansion/pptx-enrich.py b/misp_modules/modules/expansion/pptx-enrich.py new file mode 100644 index 0000000..9381262 --- /dev/null +++ b/misp_modules/modules/expansion/pptx-enrich.py @@ -0,0 +1,55 @@ +import json +import binascii +import np +from pptx import Presentation +import io + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['attachment'], + 'output': ['freetext', 'text']} +moduleinfo = {'version': '0.1', 'author': 'Sascha Rommelfangen', + 'description': '.pptx to freetext-import IOC extractor', + 'module-type': ['expansion']} + +moduleconfig = [] + + +def handler(q=False): + if q is False: + return False + q = json.loads(q) + filename = q['attachment'] + try: + pptx_array = np.frombuffer(binascii.a2b_base64(q['data']), np.uint8) + except Exception as e: + print(e) + err = "Couldn't fetch attachment (JSON 'data' is empty). Are you using the 'Query enrichment' action?" + misperrors['error'] = err + print(err) + return misperrors + + ppt_content = "" + ppt_file = io.BytesIO(pptx_array) + try: + ppt = Presentation(ppt_file) + for slide in ppt.slides: + for shape in slide.shapes: + if hasattr(shape, "text"): + print(shape.text) + ppt_content = ppt_content + "\n" + shape.text + return {'results': [{'types': ['freetext'], 'values': ppt_content, 'comment': ".pptx-to-text from file " + filename}, + {'types': ['text'], 'values': ppt_content, 'comment': ".pptx-to-text from file " + filename}]} + except Exception as e: + print(e) + err = "Couldn't analyze file as .pptx. Error was: " + str(e) + misperrors['error'] = err + return misperrors + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/expansion/xlsx-enrich.py b/misp_modules/modules/expansion/xlsx-enrich.py new file mode 100644 index 0000000..66df3e1 --- /dev/null +++ b/misp_modules/modules/expansion/xlsx-enrich.py @@ -0,0 +1,53 @@ +import json +import binascii +import np +import pandas +import io + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['attachment'], + 'output': ['freetext', 'text']} +moduleinfo = {'version': '0.1', 'author': 'Sascha Rommelfangen', + 'description': '.xlsx to freetext-import IOC extractor', + 'module-type': ['expansion']} + +moduleconfig = [] + + +def handler(q=False): + if q is False: + return False + q = json.loads(q) + filename = q['attachment'] + try: + xlsx_array = np.frombuffer(binascii.a2b_base64(q['data']), np.uint8) + except Exception as e: + print(e) + err = "Couldn't fetch attachment (JSON 'data' is empty). Are you using the 'Query enrichment' action?" + misperrors['error'] = err + print(err) + return misperrors + + xls_content = "" + xls_file = io.BytesIO(xlsx_array) + pandas.set_option('display.max_colwidth', -1) + try: + xls = pandas.read_excel(xls_file) + xls_content = xls.to_string(max_rows=None) + print(xls_content) + return {'results': [{'types': ['freetext'], 'values': xls_content, 'comment': ".xlsx-to-text from file " + filename}, + {'types': ['text'], 'values': xls_content, 'comment': ".xlsx-to-text from file " + filename}]} + except Exception as e: + print(e) + err = "Couldn't analyze file as .xlsx. Error was: " + str(e) + misperrors['error'] = err + return misperrors + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo From 3c0319b8bc80a60e2e33d0aebd56a2036d98d791 Mon Sep 17 00:00:00 2001 From: Sascha Rommelfangen Date: Fri, 26 Apr 2019 12:08:52 +0200 Subject: [PATCH 48/59] new requirements for new modules --- .travis.yml | 2 +- Pipfile | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 52a7742..18c02c6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,7 @@ python: - "3.7-dev" install: - - sudo apt-get install libzbar0 libzbar-dev + - sudo apt-get install libzbar0 libzbar-dev libpoppler-cpp-dev - pip install pipenv - pipenv install --dev diff --git a/Pipfile b/Pipfile index ba650fe..8529abc 100644 --- a/Pipfile +++ b/Pipfile @@ -45,6 +45,16 @@ backscatter = "*" pyzbar = "*" opencv-python = "*" np = "*" +ODTReader = {editable = true,git = "https://github.com/cartertemm/ODTReader.git"} +python-pptx = "*" +collections = "*" +python-docx = "*" +ezodf = "*" +pandas = "*" +pandas_ods_reader = "*" +pdftotext = "*" +lxml = "*" +xlrd = "*" [requires] python_version = "3" From 1d4f8a6989b2741ae0f58d11f960b0f8aa0dafcf Mon Sep 17 00:00:00 2001 From: Sascha Rommelfangen Date: Fri, 26 Apr 2019 12:09:16 +0200 Subject: [PATCH 49/59] new modules added --- misp_modules/modules/expansion/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py index 5c3edee..c4b2fdc 100644 --- a/misp_modules/modules/expansion/__init__.py +++ b/misp_modules/modules/expansion/__init__.py @@ -9,4 +9,5 @@ __all__ = ['cuckoo_submit', 'vmray_submit', 'bgpranking', 'circl_passivedns', 'c 'xforceexchange', 'sigma_syntax_validator', 'stix2_pattern_syntax_validator', 'sigma_queries', 'dbl_spamhaus', 'vulners', 'yara_query', 'macaddress_io', 'intel471', 'backscatter_io', 'btc_scam_check', 'hibp', 'greynoise', 'macvendors', - 'qrcode', 'ocr-enrich'] + 'qrcode', 'ocr-enrich', 'pdf-enrich', 'docx-enrich', 'xlsx-enrich', 'pptx-enrich', + 'ods-enrich', 'odt-enrich'] From fc339c888db7cb844ce8233129c97c1e41ec7664 Mon Sep 17 00:00:00 2001 From: Sascha Rommelfangen Date: Fri, 26 Apr 2019 12:14:56 +0200 Subject: [PATCH 50/59] removed trailing whitespaces --- misp_modules/modules/expansion/docx-enrich.py | 2 +- misp_modules/modules/expansion/ods-enrich.py | 4 ++-- misp_modules/modules/expansion/odt-enrich.py | 2 +- misp_modules/modules/expansion/pdf-enrich.py | 2 +- misp_modules/modules/expansion/pptx-enrich.py | 2 +- misp_modules/modules/expansion/xlsx-enrich.py | 6 +++--- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/misp_modules/modules/expansion/docx-enrich.py b/misp_modules/modules/expansion/docx-enrich.py index 6380557..d5da3f8 100644 --- a/misp_modules/modules/expansion/docx-enrich.py +++ b/misp_modules/modules/expansion/docx-enrich.py @@ -28,7 +28,7 @@ def handler(q=False): print(err) return misperrors - doc_content = "" + doc_content = "" doc_file = io.BytesIO(docx_array) try: doc = docx.Document(doc_file) diff --git a/misp_modules/modules/expansion/ods-enrich.py b/misp_modules/modules/expansion/ods-enrich.py index 06d7ff5..b247c44 100644 --- a/misp_modules/modules/expansion/ods-enrich.py +++ b/misp_modules/modules/expansion/ods-enrich.py @@ -29,14 +29,14 @@ def handler(q=False): print(err) return misperrors - ods_content = "" + ods_content = "" ods_file = io.BytesIO(ods_array) doc = ezodf.opendoc(ods_file) num_sheets = len(doc.sheets) try: for i in range(0, num_sheets): ods = pandas_ods_reader.read_ods(ods_file, i, headers=False) - ods_content = ods_content + "\n" + ods.to_string(max_rows=None) + ods_content = ods_content + "\n" + ods.to_string(max_rows=None) print(ods_content) return {'results': [{'types': ['freetext'], 'values': ods_content, 'comment': ".ods-to-text from file " + filename}, {'types': ['text'], 'values': ods_content, 'comment': ".ods-to-text from file " + filename}]} diff --git a/misp_modules/modules/expansion/odt-enrich.py b/misp_modules/modules/expansion/odt-enrich.py index 6dac555..c4513ae 100644 --- a/misp_modules/modules/expansion/odt-enrich.py +++ b/misp_modules/modules/expansion/odt-enrich.py @@ -28,7 +28,7 @@ def handler(q=False): print(err) return misperrors - odt_content = "" + odt_content = "" odt_file = io.BytesIO(odt_array) try: odt_content = odtToText(odt_file) diff --git a/misp_modules/modules/expansion/pdf-enrich.py b/misp_modules/modules/expansion/pdf-enrich.py index aeb5d9b..59d003e 100644 --- a/misp_modules/modules/expansion/pdf-enrich.py +++ b/misp_modules/modules/expansion/pdf-enrich.py @@ -2,7 +2,7 @@ import json import binascii import np import pytesseract -import pdftotext +import pdftotext import io import collections diff --git a/misp_modules/modules/expansion/pptx-enrich.py b/misp_modules/modules/expansion/pptx-enrich.py index 9381262..816e439 100644 --- a/misp_modules/modules/expansion/pptx-enrich.py +++ b/misp_modules/modules/expansion/pptx-enrich.py @@ -28,7 +28,7 @@ def handler(q=False): print(err) return misperrors - ppt_content = "" + ppt_content = "" ppt_file = io.BytesIO(pptx_array) try: ppt = Presentation(ppt_file) diff --git a/misp_modules/modules/expansion/xlsx-enrich.py b/misp_modules/modules/expansion/xlsx-enrich.py index 66df3e1..6e0ee73 100644 --- a/misp_modules/modules/expansion/xlsx-enrich.py +++ b/misp_modules/modules/expansion/xlsx-enrich.py @@ -1,7 +1,7 @@ import json import binascii import np -import pandas +import pandas import io misperrors = {'error': 'Error'} @@ -28,12 +28,12 @@ def handler(q=False): print(err) return misperrors - xls_content = "" + xls_content = "" xls_file = io.BytesIO(xlsx_array) pandas.set_option('display.max_colwidth', -1) try: xls = pandas.read_excel(xls_file) - xls_content = xls.to_string(max_rows=None) + xls_content = xls.to_string(max_rows=None) print(xls_content) return {'results': [{'types': ['freetext'], 'values': xls_content, 'comment': ".xlsx-to-text from file " + filename}, {'types': ['text'], 'values': xls_content, 'comment': ".xlsx-to-text from file " + filename}]} From 73067c8b238b563a11721281a3d58c96a678694b Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Fri, 26 Apr 2019 13:28:16 +0200 Subject: [PATCH 51/59] chg: [Pipfile] collection removed --- Pipfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Pipfile b/Pipfile index 8529abc..16ac02e 100644 --- a/Pipfile +++ b/Pipfile @@ -47,7 +47,6 @@ opencv-python = "*" np = "*" ODTReader = {editable = true,git = "https://github.com/cartertemm/ODTReader.git"} python-pptx = "*" -collections = "*" python-docx = "*" ezodf = "*" pandas = "*" From 63c12f34e6eb5b3e1998eec4dd762a1dc7a82beb Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Fri, 26 Apr 2019 13:36:07 +0200 Subject: [PATCH 52/59] chg: [pdf-enrich] updated --- misp_modules/modules/expansion/pdf-enrich.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/misp_modules/modules/expansion/pdf-enrich.py b/misp_modules/modules/expansion/pdf-enrich.py index 59d003e..ef85fde 100644 --- a/misp_modules/modules/expansion/pdf-enrich.py +++ b/misp_modules/modules/expansion/pdf-enrich.py @@ -1,10 +1,8 @@ import json import binascii import np -import pytesseract import pdftotext import io -import collections misperrors = {'error': 'Error'} mispattributes = {'input': ['attachment'], From ec766f571cf00429b43becec9d3a618c603281b7 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Fri, 26 Apr 2019 13:36:53 +0200 Subject: [PATCH 53/59] chg: [init] cleanup for pep --- misp_modules/modules/expansion/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py index c4b2fdc..70aca68 100644 --- a/misp_modules/modules/expansion/__init__.py +++ b/misp_modules/modules/expansion/__init__.py @@ -9,5 +9,5 @@ __all__ = ['cuckoo_submit', 'vmray_submit', 'bgpranking', 'circl_passivedns', 'c 'xforceexchange', 'sigma_syntax_validator', 'stix2_pattern_syntax_validator', 'sigma_queries', 'dbl_spamhaus', 'vulners', 'yara_query', 'macaddress_io', 'intel471', 'backscatter_io', 'btc_scam_check', 'hibp', 'greynoise', 'macvendors', - 'qrcode', 'ocr-enrich', 'pdf-enrich', 'docx-enrich', 'xlsx-enrich', 'pptx-enrich', + 'qrcode', 'ocr-enrich', 'pdf-enrich', 'docx-enrich', 'xlsx-enrich', 'pptx-enrich', 'ods-enrich', 'odt-enrich'] From 48c158271b5e3b0285ec8a4b915d1cf9d16784b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 26 Apr 2019 13:48:38 +0200 Subject: [PATCH 54/59] new: Devel mode. Fix #293 --- misp_modules/__init__.py | 52 ++++++++++++++++++++++++++++------------ 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/misp_modules/__init__.py b/misp_modules/__init__.py index 68a2c69..440ad3f 100644 --- a/misp_modules/__init__.py +++ b/misp_modules/__init__.py @@ -210,37 +210,59 @@ class QueryModule(tornado.web.RequestHandler): self.finish() +def _launch_from_current_dir(): + log.info('Launch MISP modules server from current directory.') + os.chdir(os.path.dirname(__file__)) + modulesdir = 'modules' + helpersdir = 'helpers' + load_helpers(helpersdir=helpersdir) + return load_modules(modulesdir) + + def main(): global mhandlers global loaded_modules signal.signal(signal.SIGINT, handle_signal) signal.signal(signal.SIGTERM, handle_signal) - argParser = argparse.ArgumentParser(description='misp-modules server') + argParser = argparse.ArgumentParser(description='misp-modules server', formatter_class=argparse.RawTextHelpFormatter) argParser.add_argument('-t', default=False, action='store_true', help='Test mode') argParser.add_argument('-s', default=False, action='store_true', help='Run a system install (package installed via pip)') argParser.add_argument('-d', default=False, action='store_true', help='Enable debugging') argParser.add_argument('-p', default=6666, help='misp-modules TCP port (default 6666)') argParser.add_argument('-l', default='localhost', help='misp-modules listen address (default localhost)') argParser.add_argument('-m', default=[], action='append', help='Register a custom module') + argParser.add_argument('--devel', default=False, action='store_true', help='''Start in development mode, enable debug, start only the module(s) listed in -m.\nExample: -m misp_modules.modules.expansion.bgpranking''') args = argParser.parse_args() port = args.p listen = args.l - log = init_logger(level=args.d) - if args.s: - log.info('Launch MISP modules server from package.') - load_package_helpers() - mhandlers, loaded_modules = load_package_modules() + if args.devel: + log = init_logger(level=True) + log.info('Launch MISP modules server in developement mode. Enable debug, load a list of modules is -m is used.') + if args.m: + mhandlers = {} + modules = [] + for module in args.m: + splitted = module.split(".") + modulename = splitted[-1] + moduletype = splitted[2] + mhandlers[modulename] = importlib.import_module(module) + mhandlers['type:' + modulename] = moduletype + modules.append(modulename) + log.info('MISP modules {0} imported'.format(modulename)) + else: + mhandlers, loaded_modules = _launch_from_current_dir() else: - log.info('Launch MISP modules server from current directory.') - os.chdir(os.path.dirname(__file__)) - modulesdir = 'modules' - helpersdir = 'helpers' - load_helpers(helpersdir=helpersdir) - mhandlers, loaded_modules = load_modules(modulesdir) + log = init_logger(level=args.d) + if args.s: + log.info('Launch MISP modules server from package.') + load_package_helpers() + mhandlers, loaded_modules = load_package_modules() + else: + mhandlers, loaded_modules = _launch_from_current_dir() - for module in args.m: - mispmod = importlib.import_module(module) - mispmod.register(mhandlers, loaded_modules) + for module in args.m: + mispmod = importlib.import_module(module) + mispmod.register(mhandlers, loaded_modules) service = [(r'/modules', ListModules), (r'/query', QueryModule)] From 929dbd2463f159f23de7df4b0089409af5670b8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 26 Apr 2019 13:49:16 +0200 Subject: [PATCH 55/59] chg: Bump dependencies. --- Pipfile | 2 +- Pipfile.lock | 116 ++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 116 insertions(+), 2 deletions(-) diff --git a/Pipfile b/Pipfile index 16ac02e..762e4c5 100644 --- a/Pipfile +++ b/Pipfile @@ -45,7 +45,7 @@ backscatter = "*" pyzbar = "*" opencv-python = "*" np = "*" -ODTReader = {editable = true,git = "https://github.com/cartertemm/ODTReader.git"} +ODTReader = {editable = true,git = "https://github.com/cartertemm/ODTReader.git/"} python-pptx = "*" python-docx = "*" ezodf = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 5d395d5..6ac6493 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "7fee9399d8a7151a79b6f8bbce64564062fd562b0a091fd45a875884d3fb954e" + "sha256": "1d4c69ced012268e1ab20cee76f76652e1acd9f5133636551d5686f15748d3b4" }, "pipfile-spec": 6, "requires": { @@ -150,6 +150,13 @@ ], "version": "==0.9" }, + "ezodf": { + "hashes": [ + "sha256:000da534f689c6d55297a08f9e2ed7eada9810d194d31d164388162fb391122d" + ], + "index": "pypi", + "version": "==0.3.2" + }, "future": { "hashes": [ "sha256:67045236dcfd6816dc439556d009594abf643e5eb48992e36beac09c2ca659b8" @@ -184,6 +191,38 @@ ], "version": "==3.0.1" }, + "lxml": { + "hashes": [ + "sha256:03984196d00670b2ab14ae0ea83d5cc0cfa4f5a42558afa9ab5fa745995328f5", + "sha256:0815b0c9f897468de6a386dc15917a0becf48cc92425613aa8bbfc7f0f82951f", + "sha256:175f3825f075cf02d15099eb52658457cf0ff103dcf11512b5d2583e1d40f58b", + "sha256:30e14c62d88d1e01a26936ecd1c6e784d4afc9aa002bba4321c5897937112616", + "sha256:3210da6f36cf4b835ff1be853962b22cc354d506f493b67a4303c88bbb40d57b", + "sha256:40f60819fbd5bad6e191ba1329bfafa09ab7f3f174b3d034d413ef5266963294", + "sha256:43b26a865a61549919f8a42e094dfdb62847113cf776d84bd6b60e4e3fc20ea3", + "sha256:4a03dd682f8e35a10234904e0b9508d705ff98cf962c5851ed052e9340df3d90", + "sha256:62f382cddf3d2e52cf266e161aa522d54fd624b8cc567bc18f573d9d50d40e8e", + "sha256:7b98f0325be8450da70aa4a796c4f06852949fe031878b4aa1d6c417a412f314", + "sha256:846a0739e595871041385d86d12af4b6999f921359b38affb99cdd6b54219a8f", + "sha256:a3080470559938a09a5d0ec558c005282e99ac77bf8211fb7b9a5c66390acd8d", + "sha256:ad841b78a476623955da270ab8d207c3c694aa5eba71f4792f65926dc46c6ee8", + "sha256:afdd75d9735e44c639ffd6258ce04a2de3b208f148072c02478162d0944d9da3", + "sha256:b4fbf9b552faff54742bcd0791ab1da5863363fb19047e68f6592be1ac2dab33", + "sha256:b90c4e32d6ec089d3fa3518436bdf5ce4d902a0787dbd9bb09f37afe8b994317", + "sha256:b91cfe4438c741aeff662d413fd2808ac901cc6229c838236840d11de4586d63", + "sha256:bdb0593a42070b0a5f138b79b872289ee73c8e25b3f0bea6564e795b55b6bcdd", + "sha256:c4e4bca2bb68ce22320297dfa1a7bf070a5b20bcbaec4ee023f83d2f6e76496f", + "sha256:cec4ab14af9eae8501be3266ff50c3c2aecc017ba1e86c160209bb4f0423df6a", + "sha256:e83b4b2bf029f5104bc1227dbb7bf5ace6fd8fabaebffcd4f8106fafc69fc45f", + "sha256:e995b3734a46d41ae60b6097f7c51ba9958648c6d1e0935b7e0ee446ee4abe22", + "sha256:f679d93dec7f7210575c85379a31322df4c46496f184ef650d3aba1484b38a2d", + "sha256:fd213bb5166e46974f113c8228daaef1732abc47cb561ce9c4c8eaed4bd3b09b", + "sha256:fdcb57b906dbc1f80666e6290e794ab8fb959a2e17aa5aee1758a85d1da4533f", + "sha256:ff424b01d090ffe1947ec7432b07f536912e0300458f9a7f48ea217dd8362b86" + ], + "index": "pypi", + "version": "==4.3.3" + }, "maclookup": { "hashes": [ "sha256:33bf8eaebe3b1e4ab4ae9277dd93c78024e0ebf6b3c42f76c37695bc26ce287a", @@ -273,6 +312,11 @@ "index": "pypi", "version": "==1.9.0.post1" }, + "odtreader": { + "editable": true, + "git": "https://github.com/cartertemm/ODTReader.git/", + "ref": "49d6938693f6faa3ff09998f86dba551ae3a996b" + }, "opencv-python": { "hashes": [ "sha256:1703a296a96d3d46615e5053f224867977accb4240bcaa0fcabcb0768bf5ac13", @@ -306,6 +350,40 @@ "index": "pypi", "version": "==4.1.0.25" }, + "pandas": { + "hashes": [ + "sha256:071e42b89b57baa17031af8c6b6bbd2e9a5c68c595bc6bf9adabd7a9ed125d3b", + "sha256:17450e25ae69e2e6b303817bdf26b2cd57f69595d8550a77c308be0cd0fd58fa", + "sha256:17916d818592c9ec891cbef2e90f98cc85e0f1e89ed0924c9b5220dc3209c846", + "sha256:2538f099ab0e9f9c9d09bbcd94b47fd889bad06dc7ae96b1ed583f1dc1a7a822", + "sha256:366f30710172cb45a6b4f43b66c220653b1ea50303fbbd94e50571637ffb9167", + "sha256:42e5ad741a0d09232efbc7fc648226ed93306551772fc8aecc6dce9f0e676794", + "sha256:4e718e7f395ba5bfe8b6f6aaf2ff1c65a09bb77a36af6394621434e7cc813204", + "sha256:4f919f409c433577a501e023943e582c57355d50a724c589e78bc1d551a535a2", + "sha256:4fe0d7e6438212e839fc5010c78b822664f1a824c0d263fd858f44131d9166e2", + "sha256:5149a6db3e74f23dc3f5a216c2c9ae2e12920aa2d4a5b77e44e5b804a5f93248", + "sha256:627594338d6dd995cfc0bacd8e654cd9e1252d2a7c959449228df6740d737eb8", + "sha256:83c702615052f2a0a7fb1dd289726e29ec87a27272d775cb77affe749cca28f8", + "sha256:8c872f7fdf3018b7891e1e3e86c55b190e6c5cee70cab771e8f246c855001296", + "sha256:90f116086063934afd51e61a802a943826d2aac572b2f7d55caaac51c13db5b5", + "sha256:a3352bacac12e1fc646213b998bce586f965c9d431773d9e91db27c7c48a1f7d", + "sha256:bcdd06007cca02d51350f96debe51331dec429ac8f93930a43eb8fb5639e3eb5", + "sha256:c1bd07ebc15285535f61ddd8c0c75d0d6293e80e1ee6d9a8d73f3f36954342d0", + "sha256:c9a4b7c55115eb278c19aa14b34fcf5920c8fe7797a09b7b053ddd6195ea89b3", + "sha256:cc8fc0c7a8d5951dc738f1c1447f71c43734244453616f32b8aa0ef6013a5dfb", + "sha256:d7b460bc316064540ce0c41c1438c416a40746fd8a4fb2999668bf18f3c4acf1" + ], + "index": "pypi", + "version": "==0.24.2" + }, + "pandas-ods-reader": { + "hashes": [ + "sha256:0f7d510639c8957a06aa1227b9f84d1be47a437dfd306464ce803b91cf5eeec4", + "sha256:d85ef58fc3aeac1616028d22954b6ef2e8983ab9bae015e1e90ce3979d138553" + ], + "index": "pypi", + "version": "==0.0.6" + }, "passivetotal": { "hashes": [ "sha256:d745a6519ec04e3a354682978ebf07778bf7602beac30307cbad075ff1a4418d" @@ -313,6 +391,13 @@ "index": "pypi", "version": "==1.0.30" }, + "pdftotext": { + "hashes": [ + "sha256:e3ad11efe0aa22cbfc46aa1296b2ea5a52ad208b778288311f2801adef178ccb" + ], + "index": "pypi", + "version": "==2.1.1" + }, "pillow": { "hashes": [ "sha256:15c056bfa284c30a7f265a41ac4cbbc93bdbfc0dfe0613b9cb8a8581b51a9e55", @@ -447,6 +532,27 @@ ], "version": "==2.8.0" }, + "python-docx": { + "hashes": [ + "sha256:bc76ecac6b2d00ce6442a69d03a6f35c71cd72293cd8405a7472dfe317920024" + ], + "index": "pypi", + "version": "==0.8.10" + }, + "python-pptx": { + "hashes": [ + "sha256:1f2d5d1d923d91f50a1f0ed794935e7d670993fdcb6c12c81cc83977c1f23e14" + ], + "index": "pypi", + "version": "==0.6.17" + }, + "pytz": { + "hashes": [ + "sha256:303879e36b721603cc54604edcac9d20401bdbe31e1e4fdee5b9f98d5d31dfda", + "sha256:d747dd3d23d77ef44c6a3526e274af6efeb0a6f1afd5a69ba4d5be4098c8e141" + ], + "version": "==2019.1" + }, "pyyaml": { "hashes": [ "sha256:1adecc22f88d38052fb787d959f003811ca858b799590a5eaa70e63dca50308c", @@ -642,6 +748,14 @@ "index": "pypi", "version": "==0.5.3" }, + "xlrd": { + "hashes": [ + "sha256:546eb36cee8db40c3eaa46c351e67ffee6eeb5fa2650b71bc4c758a29a1b29b2", + "sha256:e551fb498759fa3a5384a94ccd4c3c02eb7c00ea424426e212ac0c57be9dfbde" + ], + "index": "pypi", + "version": "==1.2.0" + }, "xlsxwriter": { "hashes": [ "sha256:2a40b427dac0f640031e5b33abe97e761de6e0f12d4d346e7b2e2b67cf6ee927", From 980760790f9aa596c87694015b9f90cef2303f15 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Fri, 26 Apr 2019 13:51:17 +0200 Subject: [PATCH 56/59] chg: [doc] new MISP expansion modules added for PDF, OCR, DOCX, XLSX, PPTX , ODS and ODT. --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 64d0960..cce1923 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/ * [Cuckoo submit](misp_modules/modules/expansion/cuckoo_submit.py) - A hover module to submit malware sample, url, attachment, domain to Cuckoo Sandbox. * [DBL Spamhaus](misp_modules/modules/expansion/dbl_spamhaus.py) - a hover module to check Spamhaus DBL for a domain name. * [DNS](misp_modules/modules/expansion/dns.py) - a simple module to resolve MISP attributes like hostname and domain to expand IP addresses attributes. +* [docx-enrich](misp_modules/modules/expansion/docx-enrich.py) - an enrichment module to get text out of Word document into MISP (using free-text parser). * [DomainTools](misp_modules/modules/expansion/domaintools.py) - a hover and expansion module to get information from [DomainTools](http://www.domaintools.com/) whois. * [EUPI](misp_modules/modules/expansion/eupi.py) - a hover and expansion module to get information about an URL from the [Phishing Initiative project](https://phishing-initiative.eu/?lang=en). * [Farsight DNSDB Passive DNS](misp_modules/modules/expansion/farsight_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information. @@ -41,10 +42,15 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/ * [iprep](misp_modules/modules/expansion/iprep.py) - an expansion module to get IP reputation from packetmail.net. * [macaddress.io](misp_modules/modules/expansion/macaddress_io.py) - a hover module to retrieve vendor details and other information regarding a given MAC address or an OUI from [MAC address Vendor Lookup](https://macaddress.io). See [integration tutorial here](https://macaddress.io/integrations/MISP-module). * [macvendors](misp_modules/modules/expansion/macvendors.py) - a hover module to retrieve mac vendor information. +* [ocr-enrich](misp_modules/modules/expansion/ocr-enrich.py) - an enrichment module to get OCRized data from images into MISP. +* [ods-enrich](misp_modules/modules/expansion/ods-enrich.py) - an enrichment module to get text out of OpenOffice spreadsheet document into MISP (using free-text parser). +* [odt-enrich](misp_modules/modules/expansion/odt-enrich.py) - an enrichment module to get text out of OpenOffice document into MISP (using free-text parser). * [onyphe](misp_modules/modules/expansion/onyphe.py) - a modules to process queries on Onyphe. * [onyphe_full](misp_modules/modules/expansion/onyphe_full.py) - a modules to process full queries on Onyphe. * [OTX](misp_modules/modules/expansion/otx.py) - an expansion module for [OTX](https://otx.alienvault.com/). * [passivetotal](misp_modules/modules/expansion/passivetotal.py) - a [passivetotal](https://www.passivetotal.org/) module that queries a number of different PassiveTotal datasets. +* [pdf-enrich](misp_modules/modules/expansion/pdf-enrich.py) - an enrichment module to extract text from PDF into MISP (using free-text parser). +* [pptx-enrich](misp_modules/modules/expansion/pptx-enrich.py) - an enrichment module to get text out of PowerPoint document into MISP (using free-text parser). * [qrcode](misp_modules/modules/expansion/qrcode.py) - a module decode QR code, barcode and similar codes from an image and enrich with the decoded values. * [rbl](misp_modules/modules/expansion/rbl.py) - a module to get RBL (Real-Time Blackhost List) values from an attribute. * [reversedns](misp_modules/modules/expansion/reversedns.py) - Simple Reverse DNS expansion service to resolve reverse DNS from MISP attributes. @@ -64,6 +70,7 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/ * [whois](misp_modules/modules/expansion) - a module to query a local instance of [uwhois](https://github.com/rafiot/uwhoisd). * [wikidata](misp_modules/modules/expansion/wiki.py) - a [wikidata](https://www.wikidata.org) expansion module. * [xforce](misp_modules/modules/expansion/xforceexchange.py) - an IBM X-Force Exchange expansion module. +* [xlsx-enrich](misp_modules/modules/expansion/xlsx-enrich.py) - an enrichment module to get text out of an Excel document into MISP (using free-text parser). * [YARA query](misp_modules/modules/expansion/yara_query.py) - a module to create YARA rules from single hash attributes. * [YARA syntax validator](misp_modules/modules/expansion/yara_syntax_validator.py) - YARA syntax validator. From d77fdabeb2fab2e7cdf010d8023f1678a6dba3f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 26 Apr 2019 13:59:36 +0200 Subject: [PATCH 57/59] fix: Re-enable python 3.6 support --- Pipfile | 1 + Pipfile.lock | 10 +++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/Pipfile b/Pipfile index 762e4c5..36f059e 100644 --- a/Pipfile +++ b/Pipfile @@ -54,6 +54,7 @@ pandas_ods_reader = "*" pdftotext = "*" lxml = "*" xlrd = "*" +idna-ssl = {markers="python_version < '3.7'"} [requires] python_version = "3" diff --git a/Pipfile.lock b/Pipfile.lock index 6ac6493..fa07b4c 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "1d4c69ced012268e1ab20cee76f76652e1acd9f5133636551d5686f15748d3b4" + "sha256": "9aac0a9c45df16b9502c13f9468095cf5ffdb8bc407fe2b55faee3ff53d8eba3" }, "pipfile-spec": 6, "requires": { @@ -177,6 +177,14 @@ ], "version": "==2.8" }, + "idna-ssl": { + "hashes": [ + "sha256:a933e3bb13da54383f9e8f35dc4f9cb9eb9b3b78c6b36f311254d6d0d92c6c7c" + ], + "index": "pypi", + "markers": "python_version < '3.7'", + "version": "==1.1.0" + }, "isodate": { "hashes": [ "sha256:2e364a3d5759479cdb2d37cce6b9376ea504db2ff90252a2e5b7cc89cc9ff2d8", From c9281e605da93f7ab9923b798e1e92e4b51fc26a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 26 Apr 2019 14:05:02 +0200 Subject: [PATCH 58/59] chg: Bump REQUIREMENTS --- REQUIREMENTS | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/REQUIREMENTS b/REQUIREMENTS index 37c5a68..9447c47 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -5,6 +5,7 @@ -e git+https://github.com/MISP/PyIntel471.git@0df8d51f1c1425de66714b3a5a45edb69b8cc2fc#egg=pyintel471 -e git+https://github.com/MISP/PyMISP.git@582dda0ce2a8ca8e1dd2cf3842e0491caca51c62#egg=pymisp -e git+https://github.com/Rafiot/uwhoisd.git@411572840eba4c72dc321c549b36a54ed5cea9de#egg=uwhois&subdirectory=client +-e git+https://github.com/cartertemm/ODTReader.git/@49d6938693f6faa3ff09998f86dba551ae3a996b#egg=odtreader -e git+https://github.com/sebdraven/pydnstrails@48c1f740025c51289f43a24863d1845ff12fd21a#egg=pydnstrails -e git+https://github.com/sebdraven/pyonyphe@cbb0168d5cb28a9f71f7ab3773164a7039ccdb12#egg=pyonyphe aiohttp==3.4.4 @@ -23,18 +24,24 @@ dnspython==1.16.0 domaintools-api==0.3.3 enum-compat==0.0.2 ez-setup==0.9 +ezodf==0.3.2 future==0.17.1 httplib2==0.12.3 +idna-ssl==1.1.0 ; python_version < '3.7' idna==2.8 isodate==0.6.0 jsonschema==3.0.1 +lxml==4.3.3 maclookup==1.0.3 multidict==4.5.2 np==1.0.2 numpy==1.16.3 oauth2==1.9.0.post1 opencv-python==4.1.0.25 +pandas-ods-reader==0.0.6 +pandas==0.24.2 passivetotal==1.0.30 +pdftotext==2.1.1 pillow==6.0.0 psutil==5.6.2 pyeupi==1.0 @@ -45,6 +52,9 @@ pypssl==2.1 pyrsistent==0.15.1 pytesseract==0.2.6 python-dateutil==2.8.0 +python-docx==0.8.10 +python-pptx==0.6.17 +pytz==2019.1 pyyaml==5.1 pyzbar==0.1.8 rdflib==4.2.2 @@ -65,6 +75,7 @@ urlarchiver==0.2 urllib3==1.24.2 vulners==1.5.0 wand==0.5.3 +xlrd==1.2.0 xlsxwriter==1.1.7 yara-python==3.8.1 yarl==1.3.0 From c5cbfaedf63d02620f1c142576f035e64f3a1679 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Sat, 27 Apr 2019 09:08:33 +0200 Subject: [PATCH 59/59] chg: [doc] install of deps updated --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cce1923..6ec5bc6 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,7 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/ ## How to install and start MISP modules in a Python virtualenv? (recommended) ~~~~bash -sudo apt-get install python3-dev python3-pip libpq5 libjpeg-dev tesseract-ocr imagemagick virtualenv libopencv-dev zbar-tools +sudo apt-get install python3-dev python3-pip libpq5 libjpeg-dev tesseract-ocr libpoppler-cpp-dev imagemagick virtualenv libopencv-dev zbar-tools libzbar0 libzbar-dev libfuzzy-dev -y sudo -u www-data virtualenv -p python3 /var/www/MISP/venv cd /usr/local/src/ sudo git clone https://github.com/MISP/misp-modules.git