chg: Bump dependencies, add update script

Pipfile.lock

@@ -333,7 +333,7 @@
         "pymisp": {
             "editable": true,
             "git": "https://github.com/MISP/PyMISP.git",
-            "ref": "634ecc3ac308d01ebf5f5fbb9aace7746a2b8707"
+            "ref": "62e047f3c1972e21aa36a8882bebf4488cdc1f84"
         },
         "pyonyphe": {
             "editable": true,

REQUIREMENTS

@@ -1,17 +1,16 @@
 -i https://pypi.org/simple
 -e .
--e git+https://github.com/D4-project/BGP-Ranking.git/@7e698f87366e6f99b4d0d11852737db28e3ddc62#egg=pybgpranking&subdirectory=client
+-e git+https://github.com/D4-project/BGP-Ranking.git/@37c97ae252ec4bf1d67733a49d4895c8cb009cf9#egg=pybgpranking&subdirectory=client
 -e git+https://github.com/D4-project/IPASN-History.git/@e846cd36fe1ed6b22f60890bba89f84e61b62e59#egg=pyipasnhistory&subdirectory=client
 -e git+https://github.com/MISP/PyIntel471.git@0df8d51f1c1425de66714b3a5a45edb69b8cc2fc#egg=pyintel471
--e git+https://github.com/MISP/PyMISP.git@634ecc3ac308d01ebf5f5fbb9aace7746a2b8707#egg=pymisp
--e git+https://github.com/Rafiot/uwhoisd.git@f6f035e52213c8abc20f2084d28cfffb399457cb#egg=uwhois&subdirectory=client
+-e git+https://github.com/MISP/PyMISP.git@62e047f3c1972e21aa36a8882bebf4488cdc1f84#egg=pymisp
+-e git+https://github.com/Rafiot/uwhoisd.git@411572840eba4c72dc321c549b36a54ed5cea9de#egg=uwhois&subdirectory=client
 -e git+https://github.com/sebdraven/pydnstrails@48c1f740025c51289f43a24863d1845ff12fd21a#egg=pydnstrails
--e git+https://github.com/sebdraven/pyonyphe@66329baeee7cab844f2203c047c2551828eaf14d#egg=pyonyphe
+-e git+https://github.com/sebdraven/pyonyphe@cbb0168d5cb28a9f71f7ab3773164a7039ccdb12#egg=pyonyphe
 aiohttp==3.4.4
 antlr4-python3-runtime==4.7.2 ; python_version >= '3'
 async-timeout==3.0.1
 attrs==18.2.0
-backscatter==0.2.3
 beautifulsoup4==4.7.1
 blockchain==1.4.4
 certifi==2018.11.29
@@ -24,42 +23,43 @@ domaintools-api==0.3.3
 enum-compat==0.0.2
 ez-setup==0.9
 future==0.17.1
-httplib2==0.12.0
+httplib2==0.12.1
 idna-ssl==1.1.0 ; python_version < '3.7'
 idna==2.8
 isodate==0.6.0
-jsonschema==2.6.0
+jsonschema==3.0.0
 maclookup==1.0.3
 multidict==4.5.2
 oauth2==1.9.0.post1
 passivetotal==1.0.30
 pillow==5.4.1
-psutil==5.5.0
+psutil==5.5.1
 pyeupi==1.0
 pygeoip==0.3.2
 pyparsing==2.3.1
 pypdns==1.3
 pypssl==2.1
+pyrsistent==0.14.11
 pytesseract==0.2.6
-python-dateutil==2.7.5
+python-dateutil==2.8.0
 pyyaml==3.13
 rdflib==4.2.2
-redis==3.1.0
+redis==3.2.0
+reportlab==3.5.13
 requests-cache==0.4.13
 requests==2.21.0
-reportlab
-shodan==1.10.4
+shodan==1.11.1
 sigmatools==0.7.1
 six==1.12.0
-soupsieve==1.7.3
+soupsieve==1.8
 sparqlwrapper==1.8.2
 stix2-patterns==1.1.0
 tornado==5.1.1
 url-normalize==1.4.1
 urlarchiver==0.2
 urllib3==1.24.1
-vulners==1.4.0
-wand==0.5.0
-xlsxwriter==1.1.2
+vulners==1.4.4
+wand==0.5.1
+xlsxwriter==1.1.5
 yara-python==3.8.1
 yarl==1.3.0

setup.py

@@ -12,6 +12,7 @@ setup(
     description='MISP modules are autonomous modules that can be used for expansion and other services in MISP',
     packages=find_packages(),
     entry_points={'console_scripts': ['misp-modules = misp_modules:main']},
+    scripts=['tools/update_misp_modules.sh'],
     test_suite="tests",
     classifiers=[
         'License :: OSI Approved :: GNU Affero General Public License v3',

tools/update_misp_modules.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+set -e
+set -x
+
+# Updates the MISP Modules while respecting the current permissions
+# It aims to support the two following installation methods:
+# * Everything is runinng on the same machine following the MISP installation guide.
+# * The modules are installed using pipenv on a different machine from the one where MISP is running.
+
+if [ -d "/var/www/MISP" ] && [ -d "/usr/local/src/misp-modules" ]
+then
+    echo "MISP is installed on the same machine, following the recommanded install script. Using MISP virtualenv."
+    PATH_TO_MISP="/var/www/MISP"
+    PATH_TO_MISP_MODULES="/usr/local/src/misp-modules"
+
+    pushd ${PATH_TO_MISP_MODULES}
+    USER=`stat -c "%U" .`
+    sudo -H -u ${USER} git pull
+    sudo -H -u ${USER} ${PATH_TO_MISP}/venv/bin/pip install -U -r REQUIREMENTS
+    sudo -H -u ${USER} ${PATH_TO_MISP}/venv/bin/pip install -U -e .
+
+    popd
+else
+    if ! [ -x "$(command -v pipenv)" ]; then
+        echo 'Error: pipenv not available, unable to automatically update.' >&2
+        exit 1
+    fi
+
+    echo "Standalone mode, use pipenv from the current directory."
+    git pull
+    pipenv install
+fi
+
+