From a770bfc5934157f895f211539911f6dcd7481e64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 26 Feb 2019 16:43:08 -0800 Subject: [PATCH 01/24] chg: Bump dependencies, add update script --- Pipfile.lock | 2 +- REQUIREMENTS | 32 ++++++++++++++++---------------- setup.py | 1 + tools/update_misp_modules.sh | 35 +++++++++++++++++++++++++++++++++++ 4 files changed, 53 insertions(+), 17 deletions(-) create mode 100755 tools/update_misp_modules.sh diff --git a/Pipfile.lock b/Pipfile.lock index 9e6265d..85cd9db 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -333,7 +333,7 @@ "pymisp": { "editable": true, "git": "https://github.com/MISP/PyMISP.git", - "ref": "634ecc3ac308d01ebf5f5fbb9aace7746a2b8707" + "ref": "62e047f3c1972e21aa36a8882bebf4488cdc1f84" }, "pyonyphe": { "editable": true, diff --git a/REQUIREMENTS b/REQUIREMENTS index 4891c60..4709747 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -1,17 +1,16 @@ -i https://pypi.org/simple -e . --e git+https://github.com/D4-project/BGP-Ranking.git/@7e698f87366e6f99b4d0d11852737db28e3ddc62#egg=pybgpranking&subdirectory=client +-e git+https://github.com/D4-project/BGP-Ranking.git/@37c97ae252ec4bf1d67733a49d4895c8cb009cf9#egg=pybgpranking&subdirectory=client -e git+https://github.com/D4-project/IPASN-History.git/@e846cd36fe1ed6b22f60890bba89f84e61b62e59#egg=pyipasnhistory&subdirectory=client -e git+https://github.com/MISP/PyIntel471.git@0df8d51f1c1425de66714b3a5a45edb69b8cc2fc#egg=pyintel471 --e git+https://github.com/MISP/PyMISP.git@634ecc3ac308d01ebf5f5fbb9aace7746a2b8707#egg=pymisp --e git+https://github.com/Rafiot/uwhoisd.git@f6f035e52213c8abc20f2084d28cfffb399457cb#egg=uwhois&subdirectory=client +-e git+https://github.com/MISP/PyMISP.git@62e047f3c1972e21aa36a8882bebf4488cdc1f84#egg=pymisp +-e git+https://github.com/Rafiot/uwhoisd.git@411572840eba4c72dc321c549b36a54ed5cea9de#egg=uwhois&subdirectory=client -e git+https://github.com/sebdraven/pydnstrails@48c1f740025c51289f43a24863d1845ff12fd21a#egg=pydnstrails --e git+https://github.com/sebdraven/pyonyphe@66329baeee7cab844f2203c047c2551828eaf14d#egg=pyonyphe +-e git+https://github.com/sebdraven/pyonyphe@cbb0168d5cb28a9f71f7ab3773164a7039ccdb12#egg=pyonyphe aiohttp==3.4.4 antlr4-python3-runtime==4.7.2 ; python_version >= '3' async-timeout==3.0.1 attrs==18.2.0 -backscatter==0.2.3 beautifulsoup4==4.7.1 blockchain==1.4.4 certifi==2018.11.29 @@ -24,42 +23,43 @@ domaintools-api==0.3.3 enum-compat==0.0.2 ez-setup==0.9 future==0.17.1 -httplib2==0.12.0 +httplib2==0.12.1 idna-ssl==1.1.0 ; python_version < '3.7' idna==2.8 isodate==0.6.0 -jsonschema==2.6.0 +jsonschema==3.0.0 maclookup==1.0.3 multidict==4.5.2 oauth2==1.9.0.post1 passivetotal==1.0.30 pillow==5.4.1 -psutil==5.5.0 +psutil==5.5.1 pyeupi==1.0 pygeoip==0.3.2 pyparsing==2.3.1 pypdns==1.3 pypssl==2.1 +pyrsistent==0.14.11 pytesseract==0.2.6 -python-dateutil==2.7.5 +python-dateutil==2.8.0 pyyaml==3.13 rdflib==4.2.2 -redis==3.1.0 +redis==3.2.0 +reportlab==3.5.13 requests-cache==0.4.13 requests==2.21.0 -reportlab -shodan==1.10.4 +shodan==1.11.1 sigmatools==0.7.1 six==1.12.0 -soupsieve==1.7.3 +soupsieve==1.8 sparqlwrapper==1.8.2 stix2-patterns==1.1.0 tornado==5.1.1 url-normalize==1.4.1 urlarchiver==0.2 urllib3==1.24.1 -vulners==1.4.0 -wand==0.5.0 -xlsxwriter==1.1.2 +vulners==1.4.4 +wand==0.5.1 +xlsxwriter==1.1.5 yara-python==3.8.1 yarl==1.3.0 diff --git a/setup.py b/setup.py index fc78750..55ed8b7 100644 --- a/setup.py +++ b/setup.py @@ -12,6 +12,7 @@ setup( description='MISP modules are autonomous modules that can be used for expansion and other services in MISP', packages=find_packages(), entry_points={'console_scripts': ['misp-modules = misp_modules:main']}, + scripts=['tools/update_misp_modules.sh'], test_suite="tests", classifiers=[ 'License :: OSI Approved :: GNU Affero General Public License v3', diff --git a/tools/update_misp_modules.sh b/tools/update_misp_modules.sh new file mode 100755 index 0000000..e0578fd --- /dev/null +++ b/tools/update_misp_modules.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +set -e +set -x + +# Updates the MISP Modules while respecting the current permissions +# It aims to support the two following installation methods: +# * Everything is runinng on the same machine following the MISP installation guide. +# * The modules are installed using pipenv on a different machine from the one where MISP is running. + +if [ -d "/var/www/MISP" ] && [ -d "/usr/local/src/misp-modules" ] +then + echo "MISP is installed on the same machine, following the recommanded install script. Using MISP virtualenv." + PATH_TO_MISP="/var/www/MISP" + PATH_TO_MISP_MODULES="/usr/local/src/misp-modules" + + pushd ${PATH_TO_MISP_MODULES} + USER=`stat -c "%U" .` + sudo -H -u ${USER} git pull + sudo -H -u ${USER} ${PATH_TO_MISP}/venv/bin/pip install -U -r REQUIREMENTS + sudo -H -u ${USER} ${PATH_TO_MISP}/venv/bin/pip install -U -e . + + popd +else + if ! [ -x "$(command -v pipenv)" ]; then + echo 'Error: pipenv not available, unable to automatically update.' >&2 + exit 1 + fi + + echo "Standalone mode, use pipenv from the current directory." + git pull + pipenv install +fi + + From 75953c32a7c093401a88e0f05d2447c9f4dbbac4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 26 Feb 2019 16:48:11 -0800 Subject: [PATCH 02/24] chr: Restart the modules after update --- tools/update_misp_modules.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/update_misp_modules.sh b/tools/update_misp_modules.sh index e0578fd..372d146 100755 --- a/tools/update_misp_modules.sh +++ b/tools/update_misp_modules.sh @@ -20,6 +20,8 @@ then sudo -H -u ${USER} ${PATH_TO_MISP}/venv/bin/pip install -U -r REQUIREMENTS sudo -H -u ${USER} ${PATH_TO_MISP}/venv/bin/pip install -U -e . + service misp-modules restart + popd else if ! [ -x "$(command -v pipenv)" ]; then From a937b7c85dae6a22f7f3d5373e628e9f36aae384 Mon Sep 17 00:00:00 2001 From: Falconieri Date: Wed, 27 Feb 2019 12:45:22 +0100 Subject: [PATCH 03/24] fix: [reportlab] Textual description parameter --- misp_modules/modules/export_mod/pdfexport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/export_mod/pdfexport.py b/misp_modules/modules/export_mod/pdfexport.py index 6b0c12f..c143b5e 100755 --- a/misp_modules/modules/export_mod/pdfexport.py +++ b/misp_modules/modules/export_mod/pdfexport.py @@ -15,7 +15,7 @@ moduleinfo = {'version': '2', 'require_standard_format': True} # config fields that your code expects from the site admin -moduleconfig = ["MISP_base_url_for_dynamic_link", "MISP_name_for_metadata"] +moduleconfig = ["MISP_base_url_for_dynamic_link", "MISP_name_for_metadata", "Activate_textual_description"] mispattributes = {} From a2716bc05d0c94a4c4d147591c2cb74a1a685882 Mon Sep 17 00:00:00 2001 From: Falconieri Date: Fri, 1 Mar 2019 09:11:34 +0100 Subject: [PATCH 04/24] fix: [exportpdf] add configmodule parameter for galaxy --- misp_modules/modules/export_mod/pdfexport.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/export_mod/pdfexport.py b/misp_modules/modules/export_mod/pdfexport.py index c143b5e..096992b 100755 --- a/misp_modules/modules/export_mod/pdfexport.py +++ b/misp_modules/modules/export_mod/pdfexport.py @@ -15,8 +15,8 @@ moduleinfo = {'version': '2', 'require_standard_format': True} # config fields that your code expects from the site admin -moduleconfig = ["MISP_base_url_for_dynamic_link", "MISP_name_for_metadata", "Activate_textual_description"] - +moduleconfig = ["MISP_base_url_for_dynamic_link", "MISP_name_for_metadata", "Activate_textual_description", + "Activate_galaxy_description"] mispattributes = {} outputFileExtension = "pdf" From aef8dbbe2eca411757bc3e56c6dd4040de0daec5 Mon Sep 17 00:00:00 2001 From: Falconieri Date: Fri, 1 Mar 2019 09:17:38 +0100 Subject: [PATCH 05/24] fix: [exportpdf] problem on one line --- misp_modules/modules/export_mod/pdfexport.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/misp_modules/modules/export_mod/pdfexport.py b/misp_modules/modules/export_mod/pdfexport.py index 096992b..d4e3be5 100755 --- a/misp_modules/modules/export_mod/pdfexport.py +++ b/misp_modules/modules/export_mod/pdfexport.py @@ -15,8 +15,7 @@ moduleinfo = {'version': '2', 'require_standard_format': True} # config fields that your code expects from the site admin -moduleconfig = ["MISP_base_url_for_dynamic_link", "MISP_name_for_metadata", "Activate_textual_description", - "Activate_galaxy_description"] +moduleconfig = ["MISP_base_url_for_dynamic_link", "MISP_name_for_metadata", "Activate_textual_description","Activate_galaxy_description"] mispattributes = {} outputFileExtension = "pdf" From 7d7c90143ef062cbb97e4f2baf88ea5e32632033 Mon Sep 17 00:00:00 2001 From: Falconieri Date: Fri, 1 Mar 2019 09:25:02 +0100 Subject: [PATCH 06/24] fix: [exportpdf] mising whitespace --- misp_modules/modules/export_mod/pdfexport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/export_mod/pdfexport.py b/misp_modules/modules/export_mod/pdfexport.py index d4e3be5..7402e27 100755 --- a/misp_modules/modules/export_mod/pdfexport.py +++ b/misp_modules/modules/export_mod/pdfexport.py @@ -15,7 +15,7 @@ moduleinfo = {'version': '2', 'require_standard_format': True} # config fields that your code expects from the site admin -moduleconfig = ["MISP_base_url_for_dynamic_link", "MISP_name_for_metadata", "Activate_textual_description","Activate_galaxy_description"] +moduleconfig = ["MISP_base_url_for_dynamic_link", "MISP_name_for_metadata", "Activate_textual_description", "Activate_galaxy_description"] mispattributes = {} outputFileExtension = "pdf" From 3b415cb53a2eff78dc1d3d56264764a81de6c3a9 Mon Sep 17 00:00:00 2001 From: cgi1 Date: Fri, 1 Mar 2019 12:13:27 +0100 Subject: [PATCH 07/24] Adding virtualenv to apt-get install --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 501e54f..6ef4bf4 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,7 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/ ## How to install and start MISP modules in a Python virtualenv? ~~~~bash -sudo apt-get install python3-dev python3-pip libpq5 libjpeg-dev tesseract-ocr imagemagick +sudo apt-get install python3-dev python3-pip libpq5 libjpeg-dev tesseract-ocr imagemagick virtualenv sudo -u www-data virtualenv -p python3 /var/www/MISP/venv cd /usr/local/src/ sudo git clone https://github.com/MISP/misp-modules.git From a30bcc5dd20f44c19d867e5cb9328113ad8ea980 Mon Sep 17 00:00:00 2001 From: Falconieri Date: Mon, 4 Mar 2019 12:36:18 +0100 Subject: [PATCH 08/24] fix: [exportpdf] add parameters --- misp_modules/modules/export_mod/pdfexport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/export_mod/pdfexport.py b/misp_modules/modules/export_mod/pdfexport.py index 7402e27..d12dece 100755 --- a/misp_modules/modules/export_mod/pdfexport.py +++ b/misp_modules/modules/export_mod/pdfexport.py @@ -15,7 +15,7 @@ moduleinfo = {'version': '2', 'require_standard_format': True} # config fields that your code expects from the site admin -moduleconfig = ["MISP_base_url_for_dynamic_link", "MISP_name_for_metadata", "Activate_textual_description", "Activate_galaxy_description"] +moduleconfig = ["MISP_base_url_for_dynamic_link", "MISP_name_for_metadata", "Activate_textual_description", "Activate_galaxy_description", "Activate_related_events", "Activate_internationalization_fonts"] mispattributes = {} outputFileExtension = "pdf" From e3ddbe66a62830f69a47c228209dcef6f3a6c14e Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Mon, 4 Mar 2019 23:08:58 +0100 Subject: [PATCH 09/24] chg: [doc] asciidoctor requirement removed (new PDF module use reportlab) --- README.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/README.md b/README.md index 6ef4bf4..951de64 100644 --- a/README.md +++ b/README.md @@ -94,8 +94,6 @@ sudo git clone https://github.com/MISP/misp-modules.git cd misp-modules sudo -u www-data /var/www/MISP/venv/bin/pip install -I -r REQUIREMENTS sudo -u www-data /var/www/MISP/venv/bin/pip install . -sudo apt install ruby-pygments.rb -y -sudo gem install asciidoctor-pdf --pre sudo sed -i -e '$i \sudo -u www-data /var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s > /tmp/misp-modules_rc.local.log &\n' /etc/rc.local /var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s & #to start the modules ~~~~ @@ -109,8 +107,6 @@ sudo git clone https://github.com/MISP/misp-modules.git cd misp-modules sudo pip3 install -I -r REQUIREMENTS sudo pip3 install -I . -sudo apt install ruby-pygments.rb -y -sudo gem install asciidoctor-pdf --pre sudo sed -i -e '$i \sudo -u www-data /var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s > /tmp/misp-modules_rc.local.log &\n' /etc/rc.local /var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s & #to start the modules ~~~~ @@ -125,7 +121,6 @@ cd misp-modules scl enable rh-python36 ‘python3 –m pip install cryptography’ scl enable rh-python36 ‘python3 –m pip install -I -r REQUIREMENTS’ scl enable rh-python36 ‘python3 –m pip install –I .’ -scl enable rh-ruby22 ‘gem install asciidoctor-pdf –pre’ ~~~~ Create the service file /etc/systemd/system/misp-workers.service : ~~~~ From 32e10ee27333410a1ed4d2c4afe10d8c3b7456be Mon Sep 17 00:00:00 2001 From: Falconieri Date: Tue, 5 Mar 2019 10:39:07 +0100 Subject: [PATCH 10/24] fix: [exportpdf] custom path parameter --- misp_modules/modules/export_mod/pdfexport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/export_mod/pdfexport.py b/misp_modules/modules/export_mod/pdfexport.py index d12dece..44b3bc9 100755 --- a/misp_modules/modules/export_mod/pdfexport.py +++ b/misp_modules/modules/export_mod/pdfexport.py @@ -15,7 +15,7 @@ moduleinfo = {'version': '2', 'require_standard_format': True} # config fields that your code expects from the site admin -moduleconfig = ["MISP_base_url_for_dynamic_link", "MISP_name_for_metadata", "Activate_textual_description", "Activate_galaxy_description", "Activate_related_events", "Activate_internationalization_fonts"] +moduleconfig = ["MISP_base_url_for_dynamic_link", "MISP_name_for_metadata", "Activate_textual_description", "Activate_galaxy_description", "Activate_related_events", "Activate_internationalization_fonts", "Custom_fonts_path"] mispattributes = {} outputFileExtension = "pdf" From 30c08708c62a773d24de1b36308cdb613d0f55a5 Mon Sep 17 00:00:00 2001 From: Falconieri Date: Tue, 5 Mar 2019 12:11:44 +0100 Subject: [PATCH 11/24] fix: [exportpdf] update documentation --- doc/export_mod/pdfexport.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/export_mod/pdfexport.json b/doc/export_mod/pdfexport.json index 9803c77..f1654dc 100644 --- a/doc/export_mod/pdfexport.json +++ b/doc/export_mod/pdfexport.json @@ -1,7 +1,7 @@ { "description": "Simple export of a MISP event to PDF.", - "requirements": ["PyMISP", "asciidoctor"], - "features": "The module takes care of the PDF file building, and work with any MISP Event. Except the requirement of asciidoctor, used to create the file, there is no special feature concerning the Event.", + "requirements": ["PyMISP", "reportlab"], + "features": "The module takes care of the PDF file building, and work with any MISP Event. Except the requirement of reportlab, used to create the file, there is no special feature concerning the Event. Some parameters can be given through the config dict. 'MISP_base_url_for_dynamic_link' is your MISP URL, to attach an hyperlink to your event on your MISP instance from the PDF. Keep it clear to avoid hyperlinks in the generated pdf.\n 'MISP_name_for_metadata' is your CERT or MISP instance name. Used as text in the PDF' metadata\n 'Activate_textual_description' is a boolean (True or void) to activate the textual description/header abstract of an event\n 'Activate_galaxy_description' is a boolean (True or void) to activate the description of event related galaxies.\n 'Activate_related_events' is a boolean (True or void) to activate the description of related event. Be aware this might leak information on confidential events linked to the current event !\n 'Activate_internationalization_fonts' is a boolean (True or void) to activate Noto fonts instead of default fonts (Helvetica). This allows the support of CJK alphabet. Be sure to have followed the procedure to download Noto fonts (~70Mo) in the right place (/tools/pdf_fonts/Noto_TTF), to allow PyMisp to find and use them during PDF generation.\n 'Custom_fonts_path' is a text (path or void) to the TTF file of your choice, to create the PDF with it. Be aware the PDF won't support bold/italic/special style anymore with this option ", "references": ["https://acrobat.adobe.com/us/en/acrobat/about-adobe-pdf.html"], "input": "MISP Event", "output": "MISP Event in a PDF file." From 9611c7f2a9a1bd84081ca3bc625f731d8185c842 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Sat, 9 Mar 2019 06:15:16 +0100 Subject: [PATCH 12/24] chg: Bump Requirements --- Pipfile.lock | 74 ++++++++++++++++++++++++++-------------------------- REQUIREMENTS | 14 +++++----- 2 files changed, 44 insertions(+), 44 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index 85cd9db..36d7c3c 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -59,10 +59,10 @@ }, "attrs": { "hashes": [ - "sha256:10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69", - "sha256:ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb" + "sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79", + "sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399" ], - "version": "==18.2.0" + "version": "==19.1.0" }, "beautifulsoup4": { "hashes": [ @@ -177,10 +177,10 @@ }, "jsonschema": { "hashes": [ - "sha256:acc8a90c31d11060516cfd0b414b9f8bcf4bc691b21f0f786ea57dd5255c79db", - "sha256:dd3f8ecb1b52d94d45eedb67cb86cac57b94ded562c5d98f63719e55ce58557b" + "sha256:0c0a81564f181de3212efa2d17de1910f8732fa1b71c42266d983cd74304e20d", + "sha256:a5f6559964a3851f59040d3b961de5e68e70971afb88ba519d27e6a039efff1a" ], - "version": "==3.0.0" + "version": "==3.0.1" }, "maclookup": { "hashes": [ @@ -281,17 +281,17 @@ }, "psutil": { "hashes": [ - "sha256:5ce6b5eb0267233459f4d3980c205828482f450999b8f5b684d9629fea98782a", - "sha256:72cebfaa422b7978a1d3632b65ff734a34c6b34f4578b68a5c204d633756b810", - "sha256:77c231b4dff8c1c329a4cd1c22b96c8976c597017ff5b09993cd148d6a94500c", - "sha256:8846ab0be0cdccd6cc92ecd1246a16e2f2e49f53bd73e522c3a75ac291e1b51d", - "sha256:a013b4250ccbddc9d22feca0f986a1afc71717ad026c0f2109bbffd007351191", - "sha256:ad43b83119eeea6d5751023298cd331637e542cbd332196464799e25a5519f8f", - "sha256:c177777c787d247d02dae6c855330f9ed3e1abf8ca1744c26dd5ff968949999a", - "sha256:ec1ef313530a9457e48d25e3fdb1723dfa636008bf1b970027462d46f2555d59", - "sha256:ef3e5e02b3c5d1df366abe7b4820400d5c427579668ad4465ff189d28ded5ebd" + "sha256:1020a37214c4138e34962881372b40f390582b5c8245680c04349c2afb785a25", + "sha256:151c9858c268a1523e16fab33e3bc3bae8a0e57b57cf7fcad85fb409cbac6baf", + "sha256:1c8e6444ca1cee9a60a1a35913b8409722f7474616e0e21004e4ffadba59964b", + "sha256:722dc0dcce5272f3c5c41609fdc2c8f0ee3f976550c2d2f2057e26ba760be9c0", + "sha256:86f61a1438c026c980a4c3e2dd88a5774a3a0f00d6d0954d6c5cf8d1921b804e", + "sha256:c4a2f42abee709ed97b4498c21aa608ac31fc1f7cc8aa60ebdcd3c80757a038d", + "sha256:d9cdc2e82aeb82200fff3640f375fac39d88b1bed27ce08377cd7fb0e3621cb7", + "sha256:da6676a484adec2fdd3e1ce1b70799881ffcb958e40208dd4c5beba0011f3589", + "sha256:dca71c08335fbfc6929438fe3a502f169ba96dd20e50b3544053d6be5cb19d82" ], - "version": "==5.5.1" + "version": "==5.6.0" }, "pybgpranking": { "editable": true, @@ -333,7 +333,7 @@ "pymisp": { "editable": true, "git": "https://github.com/MISP/PyMISP.git", - "ref": "62e047f3c1972e21aa36a8882bebf4488cdc1f84" + "ref": "b8759673b91e733c307698abdc0d5ed82fd7e0de" }, "pyonyphe": { "editable": true, @@ -469,10 +469,10 @@ }, "sigmatools": { "hashes": [ - "sha256:98c9897f27e7c99f398bff537bb6b0259599177d955f8b60a22db1b246f9cb0b" + "sha256:3bdbd2ee99c32f245e948d6b882219729ab379685dd7366e4d6149c390e08170" ], "index": "pypi", - "version": "==0.7.1" + "version": "==0.9" }, "six": { "hashes": [ @@ -506,15 +506,15 @@ }, "tornado": { "hashes": [ - "sha256:0662d28b1ca9f67108c7e3b77afabfb9c7e87bde174fbda78186ecedc2499a9d", - "sha256:4e5158d97583502a7e2739951553cbd88a72076f152b4b11b64b9a10c4c49409", - "sha256:732e836008c708de2e89a31cb2fa6c0e5a70cb60492bee6f1ea1047500feaf7f", - "sha256:8154ec22c450df4e06b35f131adc4f2f3a12ec85981a203301d310abf580500f", - "sha256:8e9d728c4579682e837c92fdd98036bd5cdefa1da2aaf6acf26947e6dd0c01c5", - "sha256:d4b3e5329f572f055b587efc57d29bd051589fb5a43ec8898c77a47ec2fa2bbb", - "sha256:e5f2585afccbff22390cddac29849df463b252b711aa2ce7c5f3f342a5b3b444" + "sha256:1a58f2d603476d5e462f7c28ca1dbb5ac7e51348b27a9cac849cdec3471101f8", + "sha256:33f93243cd46dd398e5d2bbdd75539564d1f13f25d704cfc7541db74066d6695", + "sha256:34e59401afcecf0381a28228daad8ed3275bcb726810654612d5e9c001f421b7", + "sha256:35817031611d2c296c69e5023ea1f9b5720be803e3bb119464bb2a0405d5cd70", + "sha256:666b335cef5cc2759c21b7394cff881f71559aaf7cb8c4458af5bb6cb7275b47", + "sha256:81203efb26debaaef7158187af45bc440796de9fb1df12a75b65fae11600a255", + "sha256:de274c65f45f6656c375cdf1759dbf0bc52902a1e999d12a35eb13020a641a53" ], - "version": "==5.1.1" + "version": "==6.0.1" }, "url-normalize": { "hashes": [ @@ -545,12 +545,12 @@ }, "vulners": { "hashes": [ - "sha256:40041bcf893fa1bfaf29c650369d9a249991911f28b4d8795f7bc06508013e14", - "sha256:6d00709300dcc7e2727499d8a60f51eaced1dc6b63cc19cb8a4b065b658c51aa", - "sha256:de8cef247c9852c39bd54434e63026b46bdb2bd4ca22813bf66626b7d359b0f3" + "sha256:08a7ccb2b210d45143354c6161c73fe209dc14fae8692e8b793b36b79330ad11", + "sha256:bfe2478cc11c69ba7e436d7a5df925e227565782c0bd603929fb3d612c73d78d", + "sha256:d035f6a883625878a1dc377830d17d9702ef138ca31569ac01cb8686874f89cd" ], "index": "pypi", - "version": "==1.4.4" + "version": "==1.4.5" }, "wand": { "hashes": [ @@ -611,10 +611,10 @@ }, "attrs": { "hashes": [ - "sha256:10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69", - "sha256:ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb" + "sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79", + "sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399" ], - "version": "==18.2.0" + "version": "==19.1.0" }, "certifi": { "hashes": [ @@ -743,10 +743,10 @@ }, "pyflakes": { "hashes": [ - "sha256:5e8c00e30c464c99e0b501dc160b13a14af7f27d4dffb529c556e30a159e231d", - "sha256:f277f9ca3e55de669fba45b7393a1449009cff5a37d1af10ebb76c52765269cd" + "sha256:17dbeb2e3f4d772725c777fabc446d5634d1038f234e77343108ce445ea69ce0", + "sha256:d976835886f8c5b31d47970ed689944a0262b5f3afa00a5a7b4dc81e5449f8a2" ], - "version": "==2.1.0" + "version": "==2.1.1" }, "pytest": { "hashes": [ diff --git a/REQUIREMENTS b/REQUIREMENTS index 4709747..d672411 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -3,14 +3,14 @@ -e git+https://github.com/D4-project/BGP-Ranking.git/@37c97ae252ec4bf1d67733a49d4895c8cb009cf9#egg=pybgpranking&subdirectory=client -e git+https://github.com/D4-project/IPASN-History.git/@e846cd36fe1ed6b22f60890bba89f84e61b62e59#egg=pyipasnhistory&subdirectory=client -e git+https://github.com/MISP/PyIntel471.git@0df8d51f1c1425de66714b3a5a45edb69b8cc2fc#egg=pyintel471 --e git+https://github.com/MISP/PyMISP.git@62e047f3c1972e21aa36a8882bebf4488cdc1f84#egg=pymisp +-e git+https://github.com/MISP/PyMISP.git@b8759673b91e733c307698abdc0d5ed82fd7e0de#egg=pymisp -e git+https://github.com/Rafiot/uwhoisd.git@411572840eba4c72dc321c549b36a54ed5cea9de#egg=uwhois&subdirectory=client -e git+https://github.com/sebdraven/pydnstrails@48c1f740025c51289f43a24863d1845ff12fd21a#egg=pydnstrails -e git+https://github.com/sebdraven/pyonyphe@cbb0168d5cb28a9f71f7ab3773164a7039ccdb12#egg=pyonyphe aiohttp==3.4.4 antlr4-python3-runtime==4.7.2 ; python_version >= '3' async-timeout==3.0.1 -attrs==18.2.0 +attrs==19.1.0 beautifulsoup4==4.7.1 blockchain==1.4.4 certifi==2018.11.29 @@ -27,13 +27,13 @@ httplib2==0.12.1 idna-ssl==1.1.0 ; python_version < '3.7' idna==2.8 isodate==0.6.0 -jsonschema==3.0.0 +jsonschema==3.0.1 maclookup==1.0.3 multidict==4.5.2 oauth2==1.9.0.post1 passivetotal==1.0.30 pillow==5.4.1 -psutil==5.5.1 +psutil==5.6.0 pyeupi==1.0 pygeoip==0.3.2 pyparsing==2.3.1 @@ -49,16 +49,16 @@ reportlab==3.5.13 requests-cache==0.4.13 requests==2.21.0 shodan==1.11.1 -sigmatools==0.7.1 +sigmatools==0.9 six==1.12.0 soupsieve==1.8 sparqlwrapper==1.8.2 stix2-patterns==1.1.0 -tornado==5.1.1 +tornado==6.0.1 url-normalize==1.4.1 urlarchiver==0.2 urllib3==1.24.1 -vulners==1.4.4 +vulners==1.4.5 wand==0.5.1 xlsxwriter==1.1.5 yara-python==3.8.1 From c4ced9dfbf49ec605f230b35564e17a94bcea6d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Sat, 9 Mar 2019 06:40:23 +0100 Subject: [PATCH 13/24] fix: Tornado expects a KILL now. --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index b574d4c..e8fea8e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,14 +19,14 @@ script: - pid=$! - sleep 5 - pipenv run nosetests --with-coverage --cover-package=misp_modules - - kill -s INT $pid + - kill -s KILL $pid - pushd ~/ - pipenv run coverage run -m --parallel-mode --source=misp_modules misp_modules.__init__ -s -l 127.0.0.1 & - pid=$! - popd - sleep 5 - pipenv run nosetests --with-coverage --cover-package=misp_modules - - kill -s INT $pid + - kill -s KILL $pid - pipenv run flake8 --ignore=E501,W503 misp_modules after_success: From 4b77cb5055e0a114c5bf361cadefc4aacbe0ae39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Sun, 10 Mar 2019 21:17:30 +0100 Subject: [PATCH 14/24] new: Add missing dependency (backscatter) --- Pipfile | 1 + Pipfile.lock | 92 ++++++++++++++++++++++++++++++---------------------- REQUIREMENTS | 4 ++- 3 files changed, 57 insertions(+), 40 deletions(-) diff --git a/Pipfile b/Pipfile index 45c05f5..2f2d172 100644 --- a/Pipfile +++ b/Pipfile @@ -41,6 +41,7 @@ domaintools_api = "*" misp-modules = {editable = true,path = "."} pybgpranking = {editable = true,git = "https://github.com/D4-project/BGP-Ranking.git/",subdirectory = "client"} pyipasnhistory = {editable = true,git = "https://github.com/D4-project/IPASN-History.git/",subdirectory = "client"} +backscatter = "*" [requires] python_version = "3.6" diff --git a/Pipfile.lock b/Pipfile.lock index 36d7c3c..3c902e7 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "d0cd64bfe7702365d3ea66d1f51a1ec8592df2490899e7e163fe38f97172561e" + "sha256": "23dec0fa6400c828e294ea9981b433903c17358ca61d7abdaec8df5a1c89f08c" }, "pipfile-spec": 6, "requires": { @@ -64,6 +64,14 @@ ], "version": "==19.1.0" }, + "backscatter": { + "hashes": [ + "sha256:7a0d1aa3661635de81e2a09b15d53e35cbe399a111cc58a70925f80e6874abd3", + "sha256:afb0efcf5d2551dac953ec4c38fb710b274b8e811775650e02c1ef42cafb14c8" + ], + "index": "pypi", + "version": "==0.2.4" + }, "beautifulsoup4": { "hashes": [ "sha256:034740f6cb549b4e932ae1ab975581e6103ac8f942200a0e9759065984391858", @@ -82,10 +90,10 @@ }, "certifi": { "hashes": [ - "sha256:47f9c83ef4c0c621eaef743f133f09fa8a74a9b75f037e8624f83bd1b6626cb7", - "sha256:993f830721089fef441cdfeb4b2c8c9df86f0c63239f06bd025a76a7daddb033" + "sha256:59b7658e26ca9c7339e00f8f4636cdfe59d34fa37b9b04f6f9e9926b3cece1a5", + "sha256:b26104d6835d1f5e49452a26eb2ff87fe7090b89dfcaee5ea2212697e1e1d7ae" ], - "version": "==2018.11.29" + "version": "==2019.3.9" }, "chardet": { "hashes": [ @@ -504,6 +512,12 @@ "index": "pypi", "version": "==1.1.0" }, + "tabulate": { + "hashes": [ + "sha256:8af07a39377cee1103a5c8b3330a421c2d99b9141e9cc5ddd2e3263fea416943" + ], + "version": "==0.8.3" + }, "tornado": { "hashes": [ "sha256:1a58f2d603476d5e462f7c28ca1dbb5ac7e51348b27a9cac849cdec3471101f8", @@ -618,10 +632,10 @@ }, "certifi": { "hashes": [ - "sha256:47f9c83ef4c0c621eaef743f133f09fa8a74a9b75f037e8624f83bd1b6626cb7", - "sha256:993f830721089fef441cdfeb4b2c8c9df86f0c63239f06bd025a76a7daddb033" + "sha256:59b7658e26ca9c7339e00f8f4636cdfe59d34fa37b9b04f6f9e9926b3cece1a5", + "sha256:b26104d6835d1f5e49452a26eb2ff87fe7090b89dfcaee5ea2212697e1e1d7ae" ], - "version": "==2018.11.29" + "version": "==2019.3.9" }, "chardet": { "hashes": [ @@ -640,39 +654,39 @@ }, "coverage": { "hashes": [ - "sha256:09e47c529ff77bf042ecfe858fb55c3e3eb97aac2c87f0349ab5a7efd6b3939f", - "sha256:0a1f9b0eb3aa15c990c328535655847b3420231af299386cfe5efc98f9c250fe", - "sha256:0cc941b37b8c2ececfed341444a456912e740ecf515d560de58b9a76562d966d", - "sha256:10e8af18d1315de936d67775d3a814cc81d0747a1a0312d84e27ae5610e313b0", - "sha256:1b4276550b86caa60606bd3572b52769860a81a70754a54acc8ba789ce74d607", - "sha256:1e8a2627c48266c7b813975335cfdea58c706fe36f607c97d9392e61502dc79d", - "sha256:2b224052bfd801beb7478b03e8a66f3f25ea56ea488922e98903914ac9ac930b", - "sha256:447c450a093766744ab53bf1e7063ec82866f27bcb4f4c907da25ad293bba7e3", - "sha256:46101fc20c6f6568561cdd15a54018bb42980954b79aa46da8ae6f008066a30e", - "sha256:4710dc676bb4b779c4361b54eb308bc84d64a2fa3d78e5f7228921eccce5d815", - "sha256:510986f9a280cd05189b42eee2b69fecdf5bf9651d4cd315ea21d24a964a3c36", - "sha256:5535dda5739257effef56e49a1c51c71f1d37a6e5607bb25a5eee507c59580d1", - "sha256:5a7524042014642b39b1fcae85fb37556c200e64ec90824ae9ecf7b667ccfc14", - "sha256:5f55028169ef85e1fa8e4b8b1b91c0b3b0fa3297c4fb22990d46ff01d22c2d6c", - "sha256:6694d5573e7790a0e8d3d177d7a416ca5f5c150742ee703f3c18df76260de794", - "sha256:6831e1ac20ac52634da606b658b0b2712d26984999c9d93f0c6e59fe62ca741b", - "sha256:77f0d9fa5e10d03aa4528436e33423bfa3718b86c646615f04616294c935f840", - "sha256:828ad813c7cdc2e71dcf141912c685bfe4b548c0e6d9540db6418b807c345ddd", - "sha256:85a06c61598b14b015d4df233d249cd5abfa61084ef5b9f64a48e997fd829a82", - "sha256:8cb4febad0f0b26c6f62e1628f2053954ad2c555d67660f28dfb1b0496711952", - "sha256:a5c58664b23b248b16b96253880b2868fb34358911400a7ba39d7f6399935389", - "sha256:aaa0f296e503cda4bc07566f592cd7a28779d433f3a23c48082af425d6d5a78f", - "sha256:ab235d9fe64833f12d1334d29b558aacedfbca2356dfb9691f2d0d38a8a7bfb4", - "sha256:b3b0c8f660fae65eac74fbf003f3103769b90012ae7a460863010539bb7a80da", - "sha256:bab8e6d510d2ea0f1d14f12642e3f35cefa47a9b2e4c7cea1852b52bc9c49647", - "sha256:c45297bbdbc8bb79b02cf41417d63352b70bcb76f1bbb1ee7d47b3e89e42f95d", - "sha256:d19bca47c8a01b92640c614a9147b081a1974f69168ecd494687c827109e8f42", - "sha256:d64b4340a0c488a9e79b66ec9f9d77d02b99b772c8b8afd46c1294c1d39ca478", - "sha256:da969da069a82bbb5300b59161d8d7c8d423bc4ccd3b410a9b4d8932aeefc14b", - "sha256:ed02c7539705696ecb7dc9d476d861f3904a8d2b7e894bd418994920935d36bb", - "sha256:ee5b8abc35b549012e03a7b1e86c09491457dba6c94112a2482b18589cc2bdb9" + "sha256:3684fabf6b87a369017756b551cef29e505cb155ddb892a7a29277b978da88b9", + "sha256:39e088da9b284f1bd17c750ac672103779f7954ce6125fd4382134ac8d152d74", + "sha256:3c205bc11cc4fcc57b761c2da73b9b72a59f8d5ca89979afb0c1c6f9e53c7390", + "sha256:465ce53a8c0f3a7950dfb836438442f833cf6663d407f37d8c52fe7b6e56d7e8", + "sha256:48020e343fc40f72a442c8a1334284620f81295256a6b6ca6d8aa1350c763bbe", + "sha256:5296fc86ab612ec12394565c500b412a43b328b3907c0d14358950d06fd83baf", + "sha256:5f61bed2f7d9b6a9ab935150a6b23d7f84b8055524e7be7715b6513f3328138e", + "sha256:68a43a9f9f83693ce0414d17e019daee7ab3f7113a70c79a3dd4c2f704e4d741", + "sha256:6b8033d47fe22506856fe450470ccb1d8ba1ffb8463494a15cfc96392a288c09", + "sha256:7ad7536066b28863e5835e8cfeaa794b7fe352d99a8cded9f43d1161be8e9fbd", + "sha256:7bacb89ccf4bedb30b277e96e4cc68cd1369ca6841bde7b005191b54d3dd1034", + "sha256:839dc7c36501254e14331bcb98b27002aa415e4af7ea039d9009409b9d2d5420", + "sha256:8f9a95b66969cdea53ec992ecea5406c5bd99c9221f539bca1e8406b200ae98c", + "sha256:932c03d2d565f75961ba1d3cec41ddde00e162c5b46d03f7423edcb807734eab", + "sha256:988529edadc49039d205e0aa6ce049c5ccda4acb2d6c3c5c550c17e8c02c05ba", + "sha256:998d7e73548fe395eeb294495a04d38942edb66d1fa61eb70418871bc621227e", + "sha256:9de60893fb447d1e797f6bf08fdf0dbcda0c1e34c1b06c92bd3a363c0ea8c609", + "sha256:9e80d45d0c7fcee54e22771db7f1b0b126fb4a6c0a2e5afa72f66827207ff2f2", + "sha256:a545a3dfe5082dc8e8c3eb7f8a2cf4f2870902ff1860bd99b6198cfd1f9d1f49", + "sha256:a5d8f29e5ec661143621a8f4de51adfb300d7a476224156a39a392254f70687b", + "sha256:aca06bfba4759bbdb09bf52ebb15ae20268ee1f6747417837926fae990ebc41d", + "sha256:bb23b7a6fd666e551a3094ab896a57809e010059540ad20acbeec03a154224ce", + "sha256:bfd1d0ae7e292105f29d7deaa9d8f2916ed8553ab9d5f39ec65bcf5deadff3f9", + "sha256:c62ca0a38958f541a73cf86acdab020c2091631c137bd359c4f5bddde7b75fd4", + "sha256:c709d8bda72cf4cd348ccec2a4881f2c5848fd72903c185f363d361b2737f773", + "sha256:c968a6aa7e0b56ecbd28531ddf439c2ec103610d3e2bf3b75b813304f8cb7723", + "sha256:df785d8cb80539d0b55fd47183264b7002077859028dfe3070cf6359bf8b2d9c", + "sha256:f406628ca51e0ae90ae76ea8398677a921b36f0bd71aab2099dfed08abd0322f", + "sha256:f46087bbd95ebae244a0eda01a618aff11ec7a069b15a3ef8f6b520db523dcf1", + "sha256:f8019c5279eb32360ca03e9fac40a12667715546eed5c5eb59eb381f2f501260", + "sha256:fc5f4d209733750afd2714e9109816a29500718b32dd9a5db01c0cb3a019b96a" ], - "version": "==4.5.2" + "version": "==4.5.3" }, "entrypoints": { "hashes": [ diff --git a/REQUIREMENTS b/REQUIREMENTS index d672411..99e1c02 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -11,9 +11,10 @@ aiohttp==3.4.4 antlr4-python3-runtime==4.7.2 ; python_version >= '3' async-timeout==3.0.1 attrs==19.1.0 +backscatter==0.2.4 beautifulsoup4==4.7.1 blockchain==1.4.4 -certifi==2018.11.29 +certifi==2019.3.9 chardet==3.0.4 click-plugins==1.0.4 click==7.0 @@ -54,6 +55,7 @@ six==1.12.0 soupsieve==1.8 sparqlwrapper==1.8.2 stix2-patterns==1.1.0 +tabulate==0.8.3 tornado==6.0.1 url-normalize==1.4.1 urlarchiver==0.2 From eb2dcca12b853ead10856b7de15b4d0fff4209f2 Mon Sep 17 00:00:00 2001 From: Sascha Rommelfangen Date: Thu, 14 Mar 2019 14:39:58 +0100 Subject: [PATCH 15/24] fixed a bug when checking malformed BTC addresses --- misp_modules/modules/expansion/btc_steroids.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/misp_modules/modules/expansion/btc_steroids.py b/misp_modules/modules/expansion/btc_steroids.py index 7011eda..5958502 100755 --- a/misp_modules/modules/expansion/btc_steroids.py +++ b/misp_modules/modules/expansion/btc_steroids.py @@ -91,6 +91,7 @@ def mprint(input): def handler(q=False): global result_text global conversion_rates + result_text = "" # start_time = time.time() # now = time.time() if q is False: @@ -105,7 +106,6 @@ def handler(q=False): btc = request['btc'] else: return False - mprint("\nAddress:\t" + btc) try: req = requests.get(blockchain_all.format(btc, "&limit=50")) @@ -113,8 +113,18 @@ def handler(q=False): except Exception: # print(e) print(req.text) - result_text = "" - sys.exit(1) + result_text = "Not a valid BTC address" + r = { + 'results': [ + { + 'types': ['text'], + 'values':[ + str(result_text) + ] + } + ] + } + return r n_tx = jreq['n_tx'] balance = float(jreq['final_balance'] / 100000000) From 97818e17d009f1c20806476aff8d10becb36f30e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D4=9C=D0=B5=D1=95?= <5124946+wesinator@users.noreply.github.com> Date: Thu, 14 Mar 2019 13:28:22 -0400 Subject: [PATCH 16/24] Fix command highlighting --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 951de64..e890433 100644 --- a/README.md +++ b/README.md @@ -505,14 +505,14 @@ sudo git checkout MyModBranch Remove the contents of the build directory and re-install misp-modules. -~~~python +~~~bash sudo rm -fr build/* sudo pip3 install --upgrade . ~~~ SSH in with a different terminal and run `misp-modules` with debugging enabled. -~~~python +~~~bash sudo killall misp-modules misp-modules -d ~~~ From 1c0984eaec5145643933779601390e811fb9084d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 15 Mar 2019 11:06:11 +0100 Subject: [PATCH 17/24] fix: Remove unused import --- misp_modules/modules/expansion/btc_steroids.py | 1 - 1 file changed, 1 deletion(-) diff --git a/misp_modules/modules/expansion/btc_steroids.py b/misp_modules/modules/expansion/btc_steroids.py index 2f3bebf..04b7138 100755 --- a/misp_modules/modules/expansion/btc_steroids.py +++ b/misp_modules/modules/expansion/btc_steroids.py @@ -1,4 +1,3 @@ -import sys import json import requests import time From 3e34f38cac78d6e495be8dcf4eee4d4b7dcae181 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 15 Mar 2019 11:12:19 +0100 Subject: [PATCH 18/24] chg: Bump dependencies. --- Pipfile.lock | 56 ++++++++++++++++++++++++++-------------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index 3c902e7..60f1d03 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -289,22 +289,22 @@ }, "psutil": { "hashes": [ - "sha256:1020a37214c4138e34962881372b40f390582b5c8245680c04349c2afb785a25", - "sha256:151c9858c268a1523e16fab33e3bc3bae8a0e57b57cf7fcad85fb409cbac6baf", - "sha256:1c8e6444ca1cee9a60a1a35913b8409722f7474616e0e21004e4ffadba59964b", - "sha256:722dc0dcce5272f3c5c41609fdc2c8f0ee3f976550c2d2f2057e26ba760be9c0", - "sha256:86f61a1438c026c980a4c3e2dd88a5774a3a0f00d6d0954d6c5cf8d1921b804e", - "sha256:c4a2f42abee709ed97b4498c21aa608ac31fc1f7cc8aa60ebdcd3c80757a038d", - "sha256:d9cdc2e82aeb82200fff3640f375fac39d88b1bed27ce08377cd7fb0e3621cb7", - "sha256:da6676a484adec2fdd3e1ce1b70799881ffcb958e40208dd4c5beba0011f3589", - "sha256:dca71c08335fbfc6929438fe3a502f169ba96dd20e50b3544053d6be5cb19d82" + "sha256:23e9cd90db94fbced5151eaaf9033ae9667c033dffe9e709da761c20138d25b6", + "sha256:27858d688a58cbfdd4434e1c40f6c79eb5014b709e725c180488ccdf2f721729", + "sha256:354601a1d1a1322ae5920ba397c58d06c29728a15113598d1a8158647aaa5385", + "sha256:9c3a768486194b4592c7ae9374faa55b37b9877fd9746fb4028cb0ac38fd4c60", + "sha256:c1fd45931889dc1812ba61a517630d126f6185f688eac1693171c6524901b7de", + "sha256:d463a142298112426ebd57351b45c39adb41341b91f033aa903fa4c6f76abecc", + "sha256:e1494d20ffe7891d07d8cb9a8b306c1a38d48b13575265d090fc08910c56d474", + "sha256:ec4b4b638b84d42fc48139f9352f6c6587ee1018d55253542ee28db7480cc653", + "sha256:fa0a570e0a30b9dd618bffbece590ae15726b47f9f1eaf7518dfb35f4d7dcd21" ], - "version": "==5.6.0" + "version": "==5.6.1" }, "pybgpranking": { "editable": true, "git": "https://github.com/D4-project/BGP-Ranking.git/", - "ref": "37c97ae252ec4bf1d67733a49d4895c8cb009cf9", + "ref": "4f2898af7c4e237b6497831d5acf3f4531ac14d8", "subdirectory": "client" }, "pydnstrails": { @@ -335,13 +335,13 @@ "pyipasnhistory": { "editable": true, "git": "https://github.com/D4-project/IPASN-History.git/", - "ref": "e846cd36fe1ed6b22f60890bba89f84e61b62e59", + "ref": "7ef09cf761fc58aa774ea305a33ba75959e39887", "subdirectory": "client" }, "pymisp": { "editable": true, "git": "https://github.com/MISP/PyMISP.git", - "ref": "b8759673b91e733c307698abdc0d5ed82fd7e0de" + "ref": "1dddfd72e08886673e57e23627064f9ea8303d4c" }, "pyonyphe": { "editable": true, @@ -391,19 +391,19 @@ }, "pyyaml": { "hashes": [ - "sha256:3d7da3009c0f3e783b2c873687652d83b1bbfd5c88e9813fb7e5b03c0dd3108b", - "sha256:3ef3092145e9b70e3ddd2c7ad59bdd0252a94dfe3949721633e41344de00a6bf", - "sha256:40c71b8e076d0550b2e6380bada1f1cd1017b882f7e16f09a65be98e017f211a", - "sha256:558dd60b890ba8fd982e05941927a3911dc409a63dcb8b634feaa0cda69330d3", - "sha256:a7c28b45d9f99102fa092bb213aa12e0aaf9a6a1f5e395d36166639c1f96c3a1", - "sha256:aa7dd4a6a427aed7df6fb7f08a580d68d9b118d90310374716ae90b710280af1", - "sha256:bc558586e6045763782014934bfaf39d48b8ae85a2713117d16c39864085c613", - "sha256:d46d7982b62e0729ad0175a9bc7e10a566fc07b224d2c79fafb5e032727eaa04", - "sha256:d5eef459e30b09f5a098b9cea68bebfeb268697f78d647bd255a085371ac7f3f", - "sha256:e01d3203230e1786cd91ccfdc8f8454c8069c91bee3962ad93b87a4b2860f537", - "sha256:e170a9e6fcfd19021dd29845af83bb79236068bf5fd4df3327c1be18182b2531" + "sha256:1adecc22f88d38052fb787d959f003811ca858b799590a5eaa70e63dca50308c", + "sha256:436bc774ecf7c103814098159fbb84c2715d25980175292c648f2da143909f95", + "sha256:460a5a4248763f6f37ea225d19d5c205677d8d525f6a83357ca622ed541830c2", + "sha256:5a22a9c84653debfbf198d02fe592c176ea548cccce47553f35f466e15cf2fd4", + "sha256:7a5d3f26b89d688db27822343dfa25c599627bc92093e788956372285c6298ad", + "sha256:9372b04a02080752d9e6f990179a4ab840227c6e2ce15b95e1278456664cf2ba", + "sha256:a5dcbebee834eaddf3fa7366316b880ff4062e4bcc9787b78c7fbb4a26ff2dd1", + "sha256:aee5bab92a176e7cd034e57f46e9df9a9862a71f8f37cad167c6fc74c65f5b4e", + "sha256:c51f642898c0bacd335fc119da60baae0824f2cde95b0330b56c0553439f0673", + "sha256:c68ea4d3ba1705da1e0d85da6684ac657912679a649e8868bd850d2c299cce13", + "sha256:e23d0cc5299223dcc37885dae624f382297717e459ea24053709675a976a3e19" ], - "version": "==3.13" + "version": "==5.1" }, "rdflib": { "hashes": [ @@ -764,11 +764,11 @@ }, "pytest": { "hashes": [ - "sha256:067a1d4bf827ffdd56ad21bd46674703fce77c5957f6c1eef731f6146bfcef1c", - "sha256:9687049d53695ad45cf5fdc7bbd51f0c49f1ea3ecfc4b7f3fde7501b541f17f4" + "sha256:592eaa2c33fae68c7d75aacf042efc9f77b27c08a6224a4f59beab8d9a420523", + "sha256:ad3ad5c450284819ecde191a654c09b0ec72257a2c711b9633d677c71c9850c4" ], "index": "pypi", - "version": "==4.3.0" + "version": "==4.3.1" }, "requests": { "hashes": [ From b89d068c041516f020877414bb18b0767fe43d97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 2 Apr 2019 15:30:11 +0200 Subject: [PATCH 19/24] new: Modules for greynoise, haveibeenpwned and macvendors Source: https://github.com/src7/misp-modules --- misp_modules/modules/expansion/greynoise.py | 40 ++++++++++++++ misp_modules/modules/expansion/hibp.py | 40 ++++++++++++++ misp_modules/modules/expansion/macvendors.py | 42 +++++++++++++++ tests/test_expansions.py | 57 ++++++++++++++++++++ 4 files changed, 179 insertions(+) create mode 100644 misp_modules/modules/expansion/greynoise.py create mode 100644 misp_modules/modules/expansion/hibp.py create mode 100644 misp_modules/modules/expansion/macvendors.py create mode 100644 tests/test_expansions.py diff --git a/misp_modules/modules/expansion/greynoise.py b/misp_modules/modules/expansion/greynoise.py new file mode 100644 index 0000000..324423d --- /dev/null +++ b/misp_modules/modules/expansion/greynoise.py @@ -0,0 +1,40 @@ +import requests +import json + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['ip-dst', 'ip-src'], 'output': ['text']} +moduleinfo = {'version': '0.1', 'author': 'Aurélien Schwab ', 'description': 'Module to access GreyNoise.io API.', 'module-type': ['hover']} +moduleconfig = ['user-agent']#TODO take this into account in the code + +greynoise_api_url = 'http://api.greynoise.io:8888/v1/query/ip' +default_user_agent = 'MISP-Module' + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + for input_type in mispattributes['input']: + if input_type in request: + ip = request[input_type] + break + else: + misperrors['error'] = "Unsupported attributes type" + return misperrors + data = {'ip':ip} + r = requests.post(greynoise_api_url, data=data, headers={'user-agent': default_user_agent})#Real request + if r.status_code == 200:#OK (record found) + response = json.loads(r.text) + if response: + return {'results': [{'types': mispattributes['output'], 'values': response}]} + elif r.status_code == 404:#Not found (not an error) + return {'results': [{'types': mispattributes['output'], 'values': 'No data'}]} + else:#Real error + misperrors['error'] = 'GreyNoise API not accessible (HTTP ' + str(r.status_code) + ')' + return misperrors['error'] + +def introspection(): + return mispattributes + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinf diff --git a/misp_modules/modules/expansion/hibp.py b/misp_modules/modules/expansion/hibp.py new file mode 100644 index 0000000..7010aee --- /dev/null +++ b/misp_modules/modules/expansion/hibp.py @@ -0,0 +1,40 @@ +import requests +import json + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['email-dst', 'email-src'], 'output': ['text']}#All mails as input +moduleinfo = {'version': '0.1', 'author': 'Aurélien Schwab', 'description': 'Module to access haveibeenpwned.com API.', 'module-type': ['hover']} +moduleconfig = ['user-agent']#TODO take this into account in the code + +haveibeenpwned_api_url = 'https://api.haveibeenpwned.com/api/v2/breachedaccount/' +default_user_agent = 'MISP-Module'#User agent (must be set, requiered by API)) + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + for input_type in mispattributes['input']: + if input_type in request: + email = request[input_type] + break + else: + misperrors['error'] = "Unsupported attributes type" + return misperrors + + r = requests.get(haveibeenpwned_api_url + email, headers={'user-agent': default_user_agent})#Real request + if r.status_code == 200:##OK (record found) + breaches = json.loads(r.text) + if breaches: + return {'results': [{'types': mispattributes['output'], 'values': breaches}]} + elif r.status_code == 404:#Not found (not an error) + return {'results': [{'types': mispattributes['output'], 'values': 'OK (Not Found)'}]} + else:#Real error + misperrors['error'] = 'haveibeenpwned.com API not accessible (HTTP ' + str(r.status_code) + ')' + return misperrors['error'] + +def introspection(): + return mispattributes + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinf diff --git a/misp_modules/modules/expansion/macvendors.py b/misp_modules/modules/expansion/macvendors.py new file mode 100644 index 0000000..55d0ef3 --- /dev/null +++ b/misp_modules/modules/expansion/macvendors.py @@ -0,0 +1,42 @@ +import requests +import json + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['mac-address'], 'output': ['text']} +moduleinfo = {'version': '0.1', 'author': 'Aurélien Schwab', 'description': 'Module to access Macvendors API.', 'module-type': ['hover']} +moduleconfig = ['user-agent'] # TODO take this into account in the code + +macvendors_api_url = 'https://api.macvendors.com/' +default_user_agent = 'MISP-Module' + + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + for input_type in mispattributes['input']: + if input_type in request: + mac = request[input_type] + break + else: + misperrors['error'] = "Unsupported attributes type" + return misperrors + r = requests.get(macvendors_api_url + mac, headers={'user-agent': default_user_agent}) # Real request + if r.status_code == 200: # OK (record found) + response = r.text + if response: + return {'results': [{'types': mispattributes['output'], 'values': response}]} + elif r.status_code == 404: # Not found (not an error) + return {'results': [{'types': mispattributes['output'], 'values': 'Not found'}]} + else: # Real error + misperrors['error'] = 'MacVendors API not accessible (HTTP ' + str(r.status_code) + ')' + return misperrors['error'] + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/tests/test_expansions.py b/tests/test_expansions.py new file mode 100644 index 0000000..d581a31 --- /dev/null +++ b/tests/test_expansions.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import unittest +import requests +from urllib.parse import urljoin + + +class TestExpansions(unittest.TestCase): + + def setUp(self): + self.maxDiff = None + self.headers = {'Content-Type': 'application/json'} + self.url = "http://127.0.0.1:6666/" + + def misp_modules_post(self, query): + return requests.post(urljoin(self.url, "query"), json=query) + + def get_values(self, response): + return response.json()['results'][0]['values'] + + def test_cve(self): + query = {"module": "cve", "vulnerability": "CVE-2010-3333"} + response = self.misp_modules_post(query) + self.assertTrue(self.get_values(response).startswith("Stack-based buffer overflow in Microsoft Office XP SP3, Office 2003 SP3")) + + def test_dns(self): + query = {"module": "dns", "hostname": "www.circl.lu", "config": {"nameserver": "8.8.8.8"}} + response = self.misp_modules_post(query) + self.assertEqual(self.get_values(response), ['149.13.33.14']) + + def test_macvendors(self): + query = {"module": "macvendors", "mac-address": "FC-A1-3E-2A-1C-33"} + response = self.misp_modules_post(query) + self.assertEqual(self.get_values(response), 'Samsung Electronics Co.,Ltd') + + def test_haveibeenpwned(self): + query = {"module": "hibp", "email-src": "info@circl.lu"} + response = self.misp_modules_post(query) + self.assertEqual(self.get_values(response), 'OK (Not Found)') + + def test_greynoise(self): + query = {"module": "greynoise", "ip-dst": "1.1.1.1"} + response = self.misp_modules_post(query) + self.assertEqual(self.get_values(response)['status'], 'ok') + + def test_ipasn(self): + query = {"module": "ipasn", "ip-dst": "1.1.1.1"} + response = self.misp_modules_post(query) + key = list(self.get_values(response)['response'].keys())[0] + entry = self.get_values(response)['response'][key]['asn'] + self.assertEqual(entry, '13335') + + def test_bgpranking(self): + query = {"module": "bgpranking", "AS": "13335"} + response = self.misp_modules_post(query) + self.assertEqual(self.get_values(response)['response']['asn_description'], 'CLOUDFLARENET - Cloudflare, Inc., US') From 9ea9816ad3911f0833d337f7b44cd580e940b35b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 2 Apr 2019 15:31:50 +0200 Subject: [PATCH 20/24] chg: Bump dependencies --- Pipfile.lock | 180 +++++++++++++++++++++++++-------------------------- 1 file changed, 88 insertions(+), 92 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index 60f1d03..b5faca1 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -253,39 +253,35 @@ }, "pillow": { "hashes": [ - "sha256:051de330a06c99d6f84bcf582960487835bcae3fc99365185dc2d4f65a390c0e", - "sha256:0ae5289948c5e0a16574750021bd8be921c27d4e3527800dc9c2c1d2abc81bf7", - "sha256:0b1efce03619cdbf8bcc61cfae81fcda59249a469f31c6735ea59badd4a6f58a", - "sha256:163136e09bd1d6c6c6026b0a662976e86c58b932b964f255ff384ecc8c3cefa3", - "sha256:18e912a6ccddf28defa196bd2021fe33600cbe5da1aa2f2e2c6df15f720b73d1", - "sha256:24ec3dea52339a610d34401d2d53d0fb3c7fd08e34b20c95d2ad3973193591f1", - "sha256:267f8e4c0a1d7e36e97c6a604f5b03ef58e2b81c1becb4fccecddcb37e063cc7", - "sha256:3273a28734175feebbe4d0a4cde04d4ed20f620b9b506d26f44379d3c72304e1", - "sha256:4c678e23006798fc8b6f4cef2eaad267d53ff4c1779bd1af8725cc11b72a63f3", - "sha256:4d4bc2e6bb6861103ea4655d6b6f67af8e5336e7216e20fff3e18ffa95d7a055", - "sha256:505738076350a337c1740a31646e1de09a164c62c07db3b996abdc0f9d2e50cf", - "sha256:5233664eadfa342c639b9b9977190d64ad7aca4edc51a966394d7e08e7f38a9f", - "sha256:5d95cb9f6cced2628f3e4de7e795e98b2659dfcc7176ab4a01a8b48c2c2f488f", - "sha256:7eda4c737637af74bac4b23aa82ea6fbb19002552be85f0b89bc27e3a762d239", - "sha256:801ddaa69659b36abf4694fed5aa9f61d1ecf2daaa6c92541bbbbb775d97b9fe", - "sha256:825aa6d222ce2c2b90d34a0ea31914e141a85edefc07e17342f1d2fdf121c07c", - "sha256:9c215442ff8249d41ff58700e91ef61d74f47dfd431a50253e1a1ca9436b0697", - "sha256:a3d90022f2202bbb14da991f26ca7a30b7e4c62bf0f8bf9825603b22d7e87494", - "sha256:a631fd36a9823638fe700d9225f9698fb59d049c942d322d4c09544dc2115356", - "sha256:a6523a23a205be0fe664b6b8747a5c86d55da960d9586db039eec9f5c269c0e6", - "sha256:a756ecf9f4b9b3ed49a680a649af45a8767ad038de39e6c030919c2f443eb000", - "sha256:b117287a5bdc81f1bac891187275ec7e829e961b8032c9e5ff38b70fd036c78f", - "sha256:ba04f57d1715ca5ff74bb7f8a818bf929a204b3b3c2c2826d1e1cc3b1c13398c", - "sha256:cd878195166723f30865e05d87cbaf9421614501a4bd48792c5ed28f90fd36ca", - "sha256:cee815cc62d136e96cf76771b9d3eb58e0777ec18ea50de5cfcede8a7c429aa8", - "sha256:d1722b7aa4b40cf93ac3c80d3edd48bf93b9208241d166a14ad8e7a20ee1d4f3", - "sha256:d7c1c06246b05529f9984435fc4fa5a545ea26606e7f450bdbe00c153f5aeaad", - "sha256:e9c8066249c040efdda84793a2a669076f92a301ceabe69202446abb4c5c5ef9", - "sha256:f227d7e574d050ff3996049e086e1f18c7bd2d067ef24131e50a1d3fe5831fbc", - "sha256:fc9a12aad714af36cf3ad0275a96a733526571e52710319855628f476dcb144e" + "sha256:15c056bfa284c30a7f265a41ac4cbbc93bdbfc0dfe0613b9cb8a8581b51a9e55", + "sha256:1a4e06ba4f74494ea0c58c24de2bb752818e9d504474ec95b0aa94f6b0a7e479", + "sha256:1c3c707c76be43c9e99cb7e3d5f1bee1c8e5be8b8a2a5eeee665efbf8ddde91a", + "sha256:1fd0b290203e3b0882d9605d807b03c0f47e3440f97824586c173eca0aadd99d", + "sha256:24114e4a6e1870c5a24b1da8f60d0ba77a0b4027907860188ea82bd3508c80eb", + "sha256:258d886a49b6b058cd7abb0ab4b2b85ce78669a857398e83e8b8e28b317b5abb", + "sha256:33c79b6dd6bc7f65079ab9ca5bebffb5f5d1141c689c9c6a7855776d1b09b7e8", + "sha256:367385fc797b2c31564c427430c7a8630db1a00bd040555dfc1d5c52e39fcd72", + "sha256:3c1884ff078fb8bf5f63d7d86921838b82ed4a7d0c027add773c2f38b3168754", + "sha256:44e5240e8f4f8861d748f2a58b3f04daadab5e22bfec896bf5434745f788f33f", + "sha256:46aa988e15f3ea72dddd81afe3839437b755fffddb5e173886f11460be909dce", + "sha256:74d90d499c9c736d52dd6d9b7221af5665b9c04f1767e35f5dd8694324bd4601", + "sha256:809c0a2ce9032cbcd7b5313f71af4bdc5c8c771cb86eb7559afd954cab82ebb5", + "sha256:85d1ef2cdafd5507c4221d201aaf62fc9276f8b0f71bd3933363e62a33abc734", + "sha256:8c3889c7681af77ecfa4431cd42a2885d093ecb811e81fbe5e203abc07e0995b", + "sha256:9218d81b9fca98d2c47d35d688a0cea0c42fd473159dfd5612dcb0483c63e40b", + "sha256:9aa4f3827992288edd37c9df345783a69ef58bd20cc02e64b36e44bcd157bbf1", + "sha256:9d80f44137a70b6f84c750d11019a3419f409c944526a95219bea0ac31f4dd91", + "sha256:b7ebd36128a2fe93991293f997e44be9286503c7530ace6a55b938b20be288d8", + "sha256:c4c78e2c71c257c136cdd43869fd3d5e34fc2162dc22e4a5406b0ebe86958239", + "sha256:c6a842537f887be1fe115d8abb5daa9bc8cc124e455ff995830cc785624a97af", + "sha256:cf0a2e040fdf5a6d95f4c286c6ef1df6b36c218b528c8a9158ec2452a804b9b8", + "sha256:cfd28aad6fc61f7a5d4ee556a997dc6e5555d9381d1390c00ecaf984d57e4232", + "sha256:dca5660e25932771460d4688ccbb515677caaf8595f3f3240ec16c117deff89a", + "sha256:de7aedc85918c2f887886442e50f52c1b93545606317956d65f342bd81cb4fc3", + "sha256:e6c0bbf8e277b74196e3140c35f9a1ae3eafd818f7f2d3a15819c49135d6c062" ], "index": "pypi", - "version": "==5.4.1" + "version": "==6.0.0" }, "psutil": { "hashes": [ @@ -304,7 +300,7 @@ "pybgpranking": { "editable": true, "git": "https://github.com/D4-project/BGP-Ranking.git/", - "ref": "4f2898af7c4e237b6497831d5acf3f4531ac14d8", + "ref": "019ef1c40aad1e5bb5c5072c9a998c6a8f0271f3", "subdirectory": "client" }, "pydnstrails": { @@ -335,13 +331,13 @@ "pyipasnhistory": { "editable": true, "git": "https://github.com/D4-project/IPASN-History.git/", - "ref": "7ef09cf761fc58aa774ea305a33ba75959e39887", + "ref": "0c4f11792061417b77ca6e22d2ece18109d74c75", "subdirectory": "client" }, "pymisp": { "editable": true, "git": "https://github.com/MISP/PyMISP.git", - "ref": "1dddfd72e08886673e57e23627064f9ea8303d4c" + "ref": "64bcaad0e578129543cdffad532a232722615f6c" }, "pyonyphe": { "editable": true, @@ -414,44 +410,44 @@ }, "redis": { "hashes": [ - "sha256:724932360d48e5407e8f82e405ab3650a36ed02c7e460d1e6fddf0f038422b54", - "sha256:9b19425a38fd074eb5795ff2b0d9a55b46a44f91f5347995f27e3ad257a7d775" + "sha256:6946b5dca72e86103edc8033019cc3814c031232d339d5f4533b02ea85685175", + "sha256:8ca418d2ddca1b1a850afa1680a7d2fd1f3322739271de4b704e0d4668449273" ], - "version": "==3.2.0" + "version": "==3.2.1" }, "reportlab": { "hashes": [ - "sha256:069f684cd0aaa518a27dc9124aed29cee8998e21ddf19604e53214ec8462bdd7", - "sha256:09b68ec01d86b4b120456b3f3202570ec96f57624e3a4fc36f3829323391daa4", - "sha256:0c32be9a406172c29ea20ff55a709ccac1e7fb09f15aba67cb7b455fd1d3dbe0", - "sha256:233196cf25e97cfe7c452524ea29d9a4909f1cb66599299233be1efaaaa7a7a3", - "sha256:2b5e4533f3e5b962835a5ce44467e66d1ecc822761d1b508077b5087a06be338", - "sha256:2e860bcdace5a558356802a92ae8658d7e5fdaa00ded82e83a3f2987c562cb66", - "sha256:3546029e63a9a9dc24ee38959eb417678c2425b96cd27b31e09e216dafc94666", - "sha256:4452b93f9c73b6b70311e7d69082d64da81b38e91bfb4766397630092e6da6fd", - "sha256:528c74a1c6527d1859c2c7a64a94a1cba485b00175162ea23699ae58a1e94939", - "sha256:6116e750f98018febc08dfee6df20446cf954adbcfa378d2c703d56c8864aff3", - "sha256:6b2b3580c647d75ef129172cb3da648cdb24566987b0b59c5ebb80ab770748d6", - "sha256:727b5f2bed08552d143fc99649b1863c773729f580a416844f9d9967bb0a1ae8", - "sha256:74c24a3ec0a3d4f8acb13a07192f45bdb54a1cc3c2286241677e7e8bcd5011fa", - "sha256:98ccd2f8b4f8636db05f3f14db0b471ad6bb4b66ae0dc9052c4822b3bd5d6a7d", - "sha256:a5905aa567946bc938b489a7249c7890c3fd3c9b7b5680dece5bc551c2ddbe0d", - "sha256:acbb7f676b8586b770719e9683eda951fdb38eb7970d46fcbf3cdda88d912a64", - "sha256:b5e30f865add48cf880f1c363eb505b97f2f7baaa88c155f87a335a76515a3e5", - "sha256:be2a7c33a2c28bbd3f453ffe4f0e5200b88c803a097f4cf52d69c6b53fad7a8f", - "sha256:c356bb600f59ac64955813d6497a08bfd5d0c451cb5829b61e3913d0ac084e26", - "sha256:c7ec4ae2393beab584921b1287a04e94fd98c28315e348362d89b85f4b464546", - "sha256:d476edc831bb3e9ebd04d1403abaf3ea57b3e4c2276c91a54fdfb6efbd3f9d97", - "sha256:db059e1a0691c872784062421ec51848539eb4f5210142682e61059a5ca7cc55", - "sha256:dd423a6753509ab14a0ac1b5be39d219c8f8d3781cce3deb4f45eda31969b5e8", - "sha256:ed9b7c0d71ce6fe2b31c6cde530ad8238632b876a5d599218739bda142a77f7c", - "sha256:f0a2465af4006f97b05e1f1546d67d3a3213d414894bf28be7f87f550a7f4a55", - "sha256:f20bfe26e57e8e1f575a9e0325be04dd3562db9f247ffdd73b5d4df6dec53bc2", - "sha256:f3463f2cb40a1b515ac0133ba859eca58f53b56760da9abb27ed684c565f853c", - "sha256:facc3c9748ab1525fb8401a1223bce4f24f0d6aa1a9db86c55db75777ccf40f9" + "sha256:0135bc54a463db5315c93bba4182fb83dc088fefaa7da18784ecd2a0c4a9c068", + "sha256:09e167e01458ea1e0cf3acff634ae9ecc1f1757e7585060d039c90b762859cfd", + "sha256:0dfcea18ba3ca1fac55cb273d056a8a43a48bd04d419299b3267e1994c72455a", + "sha256:1a61e56593ea1a8a38135eedfb40f79dcad13164fff034313ebf2a30e200ca79", + "sha256:1bdd871c2087d3853a0e9a3a573b1a7535500f3341944b1e34e68f3213cd28b8", + "sha256:26878a4b9c45f046c635b5695681188c19806f08b04129ea01c9ed51c7754039", + "sha256:27c62264c758aa30113df105da816223d149e4e87ee778ad49469725b79be2eb", + "sha256:29a9dd3954465b9e4efb129ffda9ab3e6a4f06488e8aa2efd5aff8ad332f13c2", + "sha256:5740e3218ca98c1bc86bd2d2e2a8c1d23e7c97d949d6377ac30aaf449f01c363", + "sha256:605892bb3f822a1e7342ce2b461d645ab8e4d13875127c0ae5377f76853db422", + "sha256:6dacc72552bc0dd50286e856f09a5e646a007d9345598bf6f75b117a200bfd9d", + "sha256:7021b7c8ba6d8e69e4c68c9473067482aaa40b9094270b45dbf798fcb0e09bd4", + "sha256:8acd950dad5b20a417579d1253c1065222dde48f9412e71533b052ab3dd98632", + "sha256:8b8fb3b0dd1e2124aba24544a02c95bff1fffa966b0581f30abf4fb28e414005", + "sha256:920c61c942eb1cc446e1647a04978f4afe31993ed403b74576a018c3ca526394", + "sha256:928e8d99befe064e28e9a29a4fd9afcf2066dcd758b0903280e67e221527422a", + "sha256:a04787eee401a74c80b65e539b5fe9226fdeabe25caa3d216c21dc990b2f8a01", + "sha256:a5bb6bd7753cba854425fcf7ecf04627a17de78d47ef9e8fac615887c5658da3", + "sha256:a70d970619014dc83b4406bcfed7e2f9d5aaf5f521aad808f5560d90ea896fb4", + "sha256:ae468fe82c8af3d1987113f03c1f87d01daa5b4c85c1f10da126be84423a744d", + "sha256:b278d83a7f76410bd310b368309e6e4b19664ffa686abfa9f0696130b09c17d3", + "sha256:b6623e9a96db3edc4b384e036e67c7bc87bbd7e5dc2d72ce66efa0043f9383b0", + "sha256:dc15cfa577bb25f0a598d483cf6dcc5ecad576ba723fe9bec63b6ec720dab2a3", + "sha256:dffdb4f6b34ce791e67365f3f96ab3c45b4cdd2c70d212fac98fb146dc75ac80", + "sha256:e84020e3482856da733e1359cb7b84e6bac09179bd3af860e70468a9c3cb43e3", + "sha256:edda09668e8474d5acb1a37fb64599557b43a714f1469bd49a058e95b5b410ff", + "sha256:f77e9835873931d25f836a3c107e53e0f7d3c0b4906b13063815308cf5ca1fac", + "sha256:f91d16ff07d5d3c92303f64c6864d74d3b6a491dde186bfef90c58088f932998" ], "index": "pypi", - "version": "==3.5.13" + "version": "==3.5.17" }, "requests": { "hashes": [ @@ -477,10 +473,10 @@ }, "sigmatools": { "hashes": [ - "sha256:3bdbd2ee99c32f245e948d6b882219729ab379685dd7366e4d6149c390e08170" + "sha256:ae980b6d6fd466294911efa493934d24e3c5df406da4a190b9fff0943a81cc5f" ], "index": "pypi", - "version": "==0.9" + "version": "==0.10" }, "six": { "hashes": [ @@ -491,10 +487,10 @@ }, "soupsieve": { "hashes": [ - "sha256:afa56bf14907bb09403e5d15fbed6275caa4174d36b975226e3b67a3bb6e2c4b", - "sha256:eaed742b48b1f3e2d45ba6f79401b2ed5dc33b2123dfe216adb90d4bfa0ade26" + "sha256:3aef141566afd07201b525c17bfaadd07580a8066f82b57f7c9417f26adbd0a3", + "sha256:e41a65e99bd125972d84221022beb1e4b5cfc68fa12c170c39834ce32d1b294c" ], - "version": "==1.8" + "version": "==1.9" }, "sparqlwrapper": { "hashes": [ @@ -520,15 +516,15 @@ }, "tornado": { "hashes": [ - "sha256:1a58f2d603476d5e462f7c28ca1dbb5ac7e51348b27a9cac849cdec3471101f8", - "sha256:33f93243cd46dd398e5d2bbdd75539564d1f13f25d704cfc7541db74066d6695", - "sha256:34e59401afcecf0381a28228daad8ed3275bcb726810654612d5e9c001f421b7", - "sha256:35817031611d2c296c69e5023ea1f9b5720be803e3bb119464bb2a0405d5cd70", - "sha256:666b335cef5cc2759c21b7394cff881f71559aaf7cb8c4458af5bb6cb7275b47", - "sha256:81203efb26debaaef7158187af45bc440796de9fb1df12a75b65fae11600a255", - "sha256:de274c65f45f6656c375cdf1759dbf0bc52902a1e999d12a35eb13020a641a53" + "sha256:1174dcb84d08887b55defb2cda1986faeeea715fff189ef3dc44cce99f5fca6b", + "sha256:2613fab506bd2aedb3722c8c64c17f8f74f4070afed6eea17f20b2115e445aec", + "sha256:44b82bc1146a24e5b9853d04c142576b4e8fa7a92f2e30bc364a85d1f75c4de2", + "sha256:457fcbee4df737d2defc181b9073758d73f54a6cfc1f280533ff48831b39f4a8", + "sha256:49603e1a6e24104961497ad0c07c799aec1caac7400a6762b687e74c8206677d", + "sha256:8c2f40b99a8153893793559919a355d7b74649a11e59f411b0b0a1793e160bc0", + "sha256:e1d897889c3b5a829426b7d52828fb37b28bc181cd598624e65c8be40ee3f7fa" ], - "version": "==6.0.1" + "version": "==6.0.2" }, "url-normalize": { "hashes": [ @@ -559,20 +555,20 @@ }, "vulners": { "hashes": [ - "sha256:08a7ccb2b210d45143354c6161c73fe209dc14fae8692e8b793b36b79330ad11", - "sha256:bfe2478cc11c69ba7e436d7a5df925e227565782c0bd603929fb3d612c73d78d", - "sha256:d035f6a883625878a1dc377830d17d9702ef138ca31569ac01cb8686874f89cd" + "sha256:6617d5904b5369507bc34105071d312e9e1c38d73654505e7b15b9a3f1325915", + "sha256:8b05d12a9dd7cbc07198a13281299a6e014ec348522e214b1efd097e194b7568", + "sha256:a19b02e0a112d70951e10c5abc1993f7f029234212828e1b617ab35f4e460a24" ], "index": "pypi", - "version": "==1.4.5" + "version": "==1.4.7" }, "wand": { "hashes": [ - "sha256:7d6b8dc9d4eaccc430b9c86e6b749013220c994970a3f39e902b397e2fa732c3", - "sha256:cc0b5c9cd50fecd10dc8888b739dd5984c6f8085d2954f34903b83ca39a91236" + "sha256:91810d241ab0851d40e67c946beb960b869c4f4160c397eac291ec6283ee3e3f", + "sha256:ae7c0958509a22f531b7b97e93adfd3f1208f0ac1c593af9e5f0cffa4ac06d5b" ], "index": "pypi", - "version": "==0.5.1" + "version": "==0.5.2" }, "xlsxwriter": { "hashes": [ @@ -719,11 +715,11 @@ }, "more-itertools": { "hashes": [ - "sha256:0125e8f60e9e031347105eb1682cef932f5e97d7b9a1a28d9bf00c22a5daef40", - "sha256:590044e3942351a1bdb1de960b739ff4ce277960f2425ad4509446dbace8d9d1" + "sha256:2112d2ca570bb7c3e53ea1a35cd5df42bb0fd10c45f0fb97178679c3c03d64c7", + "sha256:c3e4748ba1aad8dba30a4886b0b1a2004f9a863837b8654e7059eebf727afa5a" ], "markers": "python_version > '2.7'", - "version": "==6.0.0" + "version": "==7.0.0" }, "nose": { "hashes": [ @@ -764,11 +760,11 @@ }, "pytest": { "hashes": [ - "sha256:592eaa2c33fae68c7d75aacf042efc9f77b27c08a6224a4f59beab8d9a420523", - "sha256:ad3ad5c450284819ecde191a654c09b0ec72257a2c711b9633d677c71c9850c4" + "sha256:13c5e9fb5ec5179995e9357111ab089af350d788cbc944c628f3cde72285809b", + "sha256:f21d2f1fb8200830dcbb5d8ec466a9c9120e20d8b53c7585d180125cce1d297a" ], "index": "pypi", - "version": "==4.3.1" + "version": "==4.4.0" }, "requests": { "hashes": [ From c64f514a6fdfb21397dcddd8feddf7ee90bf5767 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 2 Apr 2019 15:39:27 +0200 Subject: [PATCH 21/24] fix: Typos in variable names --- misp_modules/modules/expansion/greynoise.py | 2 +- misp_modules/modules/expansion/hibp.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/greynoise.py b/misp_modules/modules/expansion/greynoise.py index 324423d..66f6985 100644 --- a/misp_modules/modules/expansion/greynoise.py +++ b/misp_modules/modules/expansion/greynoise.py @@ -37,4 +37,4 @@ def introspection(): def version(): moduleinfo['config'] = moduleconfig - return moduleinf + return moduleinfo diff --git a/misp_modules/modules/expansion/hibp.py b/misp_modules/modules/expansion/hibp.py index 7010aee..67af644 100644 --- a/misp_modules/modules/expansion/hibp.py +++ b/misp_modules/modules/expansion/hibp.py @@ -37,4 +37,4 @@ def introspection(): def version(): moduleinfo['config'] = moduleconfig - return moduleinf + return moduleinfo From 9cb21f98e1b6670d733940ea74d75a7a01a1b38e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 2 Apr 2019 15:46:17 +0200 Subject: [PATCH 22/24] fix: Add the new module sin the list of modules availables. --- misp_modules/modules/expansion/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py index d8fb153..42f74a3 100644 --- a/misp_modules/modules/expansion/__init__.py +++ b/misp_modules/modules/expansion/__init__.py @@ -8,4 +8,4 @@ __all__ = ['vmray_submit', 'bgpranking', 'circl_passivedns', 'circl_passivessl', 'yara_syntax_validator', 'hashdd', 'onyphe', 'onyphe_full', 'rbl', 'xforceexchange', 'sigma_syntax_validator', 'stix2_pattern_syntax_validator', 'sigma_queries', 'dbl_spamhaus', 'vulners', 'yara_query', 'macaddress_io', - 'intel471', 'backscatter_io', 'btc_scam_check'] + 'intel471', 'backscatter_io', 'btc_scam_check', 'hibp', 'greynoise', 'macvendors'] From f82933779f1567e1d4325f86a29b4459c9d624cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 2 Apr 2019 16:01:33 +0200 Subject: [PATCH 23/24] fix: pep8 foobar. --- misp_modules/modules/expansion/greynoise.py | 15 +++++++++------ misp_modules/modules/expansion/hibp.py | 17 ++++++++++------- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/misp_modules/modules/expansion/greynoise.py b/misp_modules/modules/expansion/greynoise.py index 66f6985..d26736d 100644 --- a/misp_modules/modules/expansion/greynoise.py +++ b/misp_modules/modules/expansion/greynoise.py @@ -4,11 +4,12 @@ import json misperrors = {'error': 'Error'} mispattributes = {'input': ['ip-dst', 'ip-src'], 'output': ['text']} moduleinfo = {'version': '0.1', 'author': 'Aurélien Schwab ', 'description': 'Module to access GreyNoise.io API.', 'module-type': ['hover']} -moduleconfig = ['user-agent']#TODO take this into account in the code +moduleconfig = ['user-agent'] # TODO take this into account in the code greynoise_api_url = 'http://api.greynoise.io:8888/v1/query/ip' default_user_agent = 'MISP-Module' + def handler(q=False): if q is False: return False @@ -20,21 +21,23 @@ def handler(q=False): else: misperrors['error'] = "Unsupported attributes type" return misperrors - data = {'ip':ip} - r = requests.post(greynoise_api_url, data=data, headers={'user-agent': default_user_agent})#Real request - if r.status_code == 200:#OK (record found) + data = {'ip': ip} + r = requests.post(greynoise_api_url, data=data, headers={'user-agent': default_user_agent}) # Real request + if r.status_code == 200: # OK (record found) response = json.loads(r.text) if response: return {'results': [{'types': mispattributes['output'], 'values': response}]} - elif r.status_code == 404:#Not found (not an error) + elif r.status_code == 404: # Not found (not an error) return {'results': [{'types': mispattributes['output'], 'values': 'No data'}]} - else:#Real error + else: # Real error misperrors['error'] = 'GreyNoise API not accessible (HTTP ' + str(r.status_code) + ')' return misperrors['error'] + def introspection(): return mispattributes + def version(): moduleinfo['config'] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/hibp.py b/misp_modules/modules/expansion/hibp.py index 67af644..8db3fa7 100644 --- a/misp_modules/modules/expansion/hibp.py +++ b/misp_modules/modules/expansion/hibp.py @@ -2,12 +2,13 @@ import requests import json misperrors = {'error': 'Error'} -mispattributes = {'input': ['email-dst', 'email-src'], 'output': ['text']}#All mails as input +mispattributes = {'input': ['email-dst', 'email-src'], 'output': ['text']} # All mails as input moduleinfo = {'version': '0.1', 'author': 'Aurélien Schwab', 'description': 'Module to access haveibeenpwned.com API.', 'module-type': ['hover']} -moduleconfig = ['user-agent']#TODO take this into account in the code +moduleconfig = ['user-agent'] # TODO take this into account in the code haveibeenpwned_api_url = 'https://api.haveibeenpwned.com/api/v2/breachedaccount/' -default_user_agent = 'MISP-Module'#User agent (must be set, requiered by API)) +default_user_agent = 'MISP-Module' # User agent (must be set, requiered by API)) + def handler(q=False): if q is False: @@ -21,20 +22,22 @@ def handler(q=False): misperrors['error'] = "Unsupported attributes type" return misperrors - r = requests.get(haveibeenpwned_api_url + email, headers={'user-agent': default_user_agent})#Real request - if r.status_code == 200:##OK (record found) + r = requests.get(haveibeenpwned_api_url + email, headers={'user-agent': default_user_agent}) # Real request + if r.status_code == 200: # OK (record found) breaches = json.loads(r.text) if breaches: return {'results': [{'types': mispattributes['output'], 'values': breaches}]} - elif r.status_code == 404:#Not found (not an error) + elif r.status_code == 404: # Not found (not an error) return {'results': [{'types': mispattributes['output'], 'values': 'OK (Not Found)'}]} - else:#Real error + else: # Real error misperrors['error'] = 'haveibeenpwned.com API not accessible (HTTP ' + str(r.status_code) + ')' return misperrors['error'] + def introspection(): return mispattributes + def version(): moduleinfo['config'] = moduleconfig return moduleinfo From 07a66d62b065bfe06225f5a0d1f6d95247e6d9b2 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Tue, 2 Apr 2019 20:03:11 +0200 Subject: [PATCH 24/24] chg: [doc] new modules added --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index e890433..5669b19 100644 --- a/README.md +++ b/README.md @@ -32,11 +32,14 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/ * [EUPI](misp_modules/modules/expansion/eupi.py) - a hover and expansion module to get information about an URL from the [Phishing Initiative project](https://phishing-initiative.eu/?lang=en). * [Farsight DNSDB Passive DNS](misp_modules/modules/expansion/farsight_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information. * [GeoIP](misp_modules/modules/expansion/geoip_country.py) - a hover and expansion module to get GeoIP information from geolite/maxmind. +* [Greynoise](misp_modules/modules/expansion/greynoise.py) - a hover to get information from greynoise. * [hashdd](misp_modules/modules/expansion/hashdd.py) - a hover module to check file hashes against [hashdd.com](http://www.hashdd.com) including NSLR dataset. +* [hibp](misp_modules/modules/expansion/hibp.py) - a hover module to lookup against Have I Been Pwned? * [intel471](misp_modules/modules/expansion/intel471.py) - an expansion module to get info from [Intel471](https://intel471.com). * [IPASN](misp_modules/modules/expansion/ipasn.py) - a hover and expansion to get the BGP ASN of an IP address. * [iprep](misp_modules/modules/expansion/iprep.py) - an expansion module to get IP reputation from packetmail.net. * [macaddress.io](misp_modules/modules/expansion/macaddress_io.py) - a hover module to retrieve vendor details and other information regarding a given MAC address or an OUI from [MAC address Vendor Lookup](https://macaddress.io). See [integration tutorial here](https://macaddress.io/integrations/MISP-module). +* [macvendors](misp_modules/modules/expansion/macvendors.py) - a hover module to retrieve mac vendor information. * [onyphe](misp_modules/modules/expansion/onyphe.py) - a modules to process queries on Onyphe. * [onyphe_full](misp_modules/modules/expansion/onyphe_full.py) - a modules to process full queries on Onyphe. * [OTX](misp_modules/modules/expansion/otx.py) - an expansion module for [OTX](https://otx.alienvault.com/).