Migrate to poetry and optimize dependencies

2024-08-12 10:43:25 +01:00 · 2024-08-12 10:43:25 +01:00 · b89a33b74f
parent 3a13af724f
commit b89a33b74f
35 changed files with 6531 additions and 4969 deletions
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@ -19,50 +19,55 @@ jobs:
    - name: Install packages
      run: |
        sudo apt-get install libpoppler-cpp-dev libzbar0 tesseract-ocr
    - name: Compile and install yara
      run: |
        sudo apt-get install automake libtool make gcc pkg-config
-        wget https://github.com/VirusTotal/yara/archive/refs/tags/v4.5.0.tar.gz
+        wget https://github.com/VirusTotal/yara/archive/refs/tags/v4.5.0.tar.gz -O yara.tar.gz
-        tar xzf *.tar.gz
+        tar xzf yara.tar.gz
        pushd yara-*
        ./bootstrap.sh
        ./configure
        make
        sudo make install
        popd
-    - uses: actions/checkout@v4
+
    - name: Checkout repository
      uses: actions/checkout@v4
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}
        cache: 'pip'
    - name: Install dependencies
      run: |
-        python -m pip install --upgrade pip
+        python -m pip install --upgrade poetry
-        pip install flake8 pytest
+        poetry install
-        pip install pipenv
+        # Tell poetry to not use a virtual environment
-        sed -i "s/python_version.*/python_version = \"${{ matrix.python-version }}\"/" Pipfile
+        # poetry config virtualenvs.create false
-        pipenv lock
+
        pipenv requirements > requirements.txt
        # pyfaul must be installed manually (?)
        pip install -r requirements.txt pyfaup
        pip install .
    - name: Lint with flake8
      run: |
        # stop the build if there are Python syntax errors or undefined names
-        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+        poetry run flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+        poetry run flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
    - name: Run server in background
      run: |
-        misp-modules -l 127.0.0.1 -s 2>error.log &
+        poetry run misp-modules -l 127.0.0.1 -s 2>error.log &
-        sleep 3
+        sleep 10
    - name: Check if server is running
      run: |
        curl -sS localhost:6666/healthcheck
    - name: Test with pytest
      run: |
-        pytest tests
+        poetry run pytest
    - name: Show error log
      if: always()
      run: |
--- a/.gitignore
+++ b/.gitignore
@ -24,3 +24,4 @@ venv*
 .vscode*
 *.sqlite
 website/conf/config.cfg
 wheels
--- a/.travis.yml
+++ b/.travis.yml
@ -1,56 +0,0 @@
 language: python
 services:
    - redis-server
 cache: pip
 python:
    - "3.6"
    - "3.6-dev"
    - "3.7-dev"
    - "3.8-dev"
 install:
    - sudo apt-get install libzbar0 libzbar-dev libpoppler-cpp-dev tesseract-ocr libfuzzy-dev libcaca-dev liblua5.3-dev
    - pip install pipenv
    - pip install -r REQUIREMENTS
    #    - pipenv install --dev
    # install gtcaca
    - git clone git://github.com/stricaud/gtcaca.git
    - mkdir -p gtcaca/build
    - pushd gtcaca/build
    - cmake .. && make
    - sudo make install
    - popd
    # install pyfaup
    - git clone https://github.com/stricaud/faup.git
    - pushd faup/build
    - cmake .. && make
    - sudo make install
    - popd
    - sudo ldconfig
    - pushd faup/src/lib/bindings/python
    - pip install .
    - popd
 script:
    - pip install coverage
    - coverage run -m --parallel-mode --source=misp_modules misp_modules.__init__  -l 127.0.0.1 &
    - pid=$!
    - sleep 5
    - nosetests --with-coverage --cover-package=misp_modules
    - kill -s KILL $pid
    - pushd ~/
    - coverage run -m --parallel-mode --source=misp_modules misp_modules.__init__ -s -l 127.0.0.1 &
    - pid=$!
    - popd
    - sleep 5
    - nosetests --with-coverage --cover-package=misp_modules
    - kill -s KILL $pid
    - pip install flake8
    - flake8 --ignore=E501,W503,E226,E126 misp_modules
 after_success:
    - coverage combine .coverage*
    - codecov
--- a/3
+++ b/3
@ -1,3 +0,0 @@
 mkdocs
 pymdown-extensions
 mkdocs-material
--- a/81
+++ b/81
@ -1,38 +1,61 @@
-# https://www.mkdocs.org/user-guide/deploying-your-docs/
+# See: https://www.mkdocs.org/user-guide/deploying-your-docs/
 # Running 'make' uses poetry-installed mkdocs
 # Running 'USE_DOCKER=true make' uses docker mkdocs
 .PHONY: prepare_docs generate_docs deploy test_docs
 MKDOCS_DOCKER_IMAGE := squidfunk/mkdocs-material
 DOCS_DIST_DIR := ./docs
 DOCS_SRC_DIR := ./documentation
 USE_DOCKER ?=
 .DEFAULT_GOAL := generate_docs
 .PHONY: prepare_docs generate_docs ci_generate_docs test_docs
 prepare_docs:
-	pip3 install .
+	@echo "Preparing documentation."
-	cd documentation; python3 generate_documentation.py
+	poetry install --only docs
-	mkdir -p docs/expansion/logos docs/export_mod/logos docs/import_mod/logos
+	poetry run python $(DOCS_SRC_DIR)/generate_documentation.py
-	mkdir -p docs/logos
+	mkdir -p $(DOCS_DIST_DIR)/logos
-	cd documentation; cp -R ./logos/*  ../docs/logos
+	mkdir -p $(DOCS_DIST_DIR)/expansion/logos
-	cd documentation; cp -R ./logos/* ../docs/expansion/logos
+	mkdir -p $(DOCS_DIST_DIR)/export_mod/logos
-	cd documentation; cp -R ./logos/* ../docs/export_mod/logos
+	mkdir -p $(DOCS_DIST_DIR)/import_mod/logos
-	cd documentation; cp -R ./logos/* ../docs/import_mod/logos
+	cp -R $(DOCS_SRC_DIR)/logos/* $(DOCS_DIST_DIR)/logos
-	cp ./documentation/mkdocs/*.md ./docs
+	cp -R $(DOCS_SRC_DIR)/logos/* $(DOCS_DIST_DIR)/expansion/logos
-	cp LICENSE ./docs/license.md
+	cp -R $(DOCS_SRC_DIR)/logos/* $(DOCS_DIST_DIR)/export_mod/logos
 	cp -R $(DOCS_SRC_DIR)/logos/* $(DOCS_DIST_DIR)/import_mod/logos
 	cp $(DOCS_SRC_DIR)/mkdocs/*.md $(DOCS_DIST_DIR)
 	cp LICENSE $(DOCS_DIST_DIR)/license.md
 install_requirements:
 	pip install -r docs/REQUIREMENTS.txt
 generate_docs: prepare_docs
-	mkdocs build
+ifeq ($(USE_DOCKER), true)
 	@echo "Generating documentation using '$(MKDOCS_DOCKER_IMAGE)'."
 	docker run --rm -it -v $(PWD):/docs $(MKDOCS_DOCKER_IMAGE) build
 else
 	@echo "Generating docunentation."
 	poetry run mkdocs build
 endif
 deploy: generate_docs
 ifeq ($(USE_DOCKER), true)
 	@echo "Deploying documentation using '$(MKDOCS_DOCKER_IMAGE)'."
 	docker run --rm -it -v $(PWD):/docs -v /home/$(whoami)/.docker:/root/.docker:ro $(MKDOCS_DOCKER_IMAGE) gh-deploy
 else
 	@echo "Deploying docunentation."
 	poetry run mkdocs gh-deploy
 endif
 deploy:
 	mkdocs gh-deploy
 test_docs: prepare_docs
-	mkdocs serve
+ifeq ($(USE_DOCKER), true)
-
+	@echo "Serving documentation using '$(MKDOCS_DOCKER_IMAGE)'."
-
+	docker run --rm -it -v $(PWD):/docs -p 8000:8000 $(MKDOCS_DOCKER_IMAGE)
-# DOCKER make commands
+else
-generate_docs_docker: prepare_docs
+	@echo "Serving docunentation."
-	docker run --rm -it -v $(PWD):/docs squidfunk/mkdocs-material build
+	poetry run mkdocs serve
-
+endif
 deploy_docker:
 	docker run --rm -it -v $(PWD):/docs -v /home/$(whoami)/.docker:/root/.docker:ro squidfunk/mkdocs-material gh-deploy
 test_docs_docker: prepare_docs
 	docker run --rm -it -p 8000:8000 -v $(PWD):/docs squidfunk/mkdocs-material
--- a/87
+++ b/87
@ -1,87 +0,0 @@
 [[source]]
 name = "pypi"
 url = "https://pypi.org/simple"
 verify_ssl = true
 [dev-packages]
 nose = "*"
 codecov = "*"
 pytest = "*"
 flake8 = "*"
 [packages]
 numpy = "<2.0.0"
 matplotlib = "*"
 sigmf = "*"
 pysafebrowsing = "*"
 dnspython = "*"
 requests = { extras = ["security"], version = "*" }
 urlarchiver = "*"
 passivetotal = "*"
 pypdns = "*"
 pypssl = "*"
 pyeupi = "*"
 pymisp = { extras = ["fileobjects,openioc,pdfexport,email,url"], version = "*" }
 pyonyphe = { git = "https://github.com/sebdraven/pyonyphe" }
 pydnstrails = { git = "https://github.com/sebdraven/pydnstrails" }
 pytesseract = "*"
 pygeoip = "*"
 beautifulsoup4 = "*"
 oauth2 = "*"
 yara-python = "==4.5.0"
 sigmatools = "*"
 stix2 = "*"
 stix2-patterns = "*"
 taxii2-client = "*"
 maclookup = "*"
 vulners = "*"
 blockchain = "*"
 reportlab = "*"
 pyintel471 = { git = "https://github.com/MISP/PyIntel471.git" }
 shodan = "*"
 Pillow = ">=8.2.0"
 Wand = "*"
 SPARQLWrapper = "*"
 domaintools_api = "*"
 misp-modules = { path = "." }
 pybgpranking = { git = "https://github.com/D4-project/BGP-Ranking.git/", subdirectory = "client", ref = "68de39f6c5196f796055c1ac34504054d688aa59" }
 pyipasnhistory = { git = "https://github.com/D4-project/IPASN-History.git/", subdirectory = "client", ref = "a2853c39265cecdd0c0d16850bd34621c0551b87" }
 backscatter = "*"
 pyzbar = "*"
 opencv-python = "*"
 np = "*"
 ODTReader = { git = "https://github.com/cartertemm/ODTReader.git/" }
 python-pptx = "*"
 python-docx = "*"
 ezodf = "*"
 pandas = "*"
 pandas_ods_reader = "*"
 pdftotext = "*"
 lxml = "*"
 xlrd = "*"
 jbxapi = "*"
 geoip2 = "*"
 apiosintDS = "*"
 assemblyline_client = "*"
 vt-graph-api = "*"
 trustar = { git = "https://github.com/SteveClement/trustar-python.git" }
 markdownify = "==0.5.3"
 socialscan = "==1.4"
 pycountry = "==22.3.5"
 dnsdb2 = "*"
 clamd = "*"
 aiohttp = ">=3.7.4"
 tau-clients = "*"
 vt-py = ">=0.7.1"
 crowdstrike-falconpy = "0.9.0"
 censys = "2.0.9"
 mwdblib = "3.4.1"
 ndjson = "0.3.1"
 Jinja2 = ">=3.1.2"
 mattermostdriver = "7.3.2"
 openpyxl = "*"
 slack-sdk = "3.27.1"
 vysion = "*"
 [requires]
 python_version = "3.12"
--- a/Pipfile.lock
+++ b/Pipfile.lock
--- a/README.md
+++ b/README.md
@ -42,7 +42,6 @@ For further Information see the [license file](https://misp.github.io/misp-modul
 * [AssemblyLine Query](https://misp.github.io/misp-modules/expansion/#assemblyline-query) - A module tu query the AssemblyLine API with a submission ID to get the submission report and parse it.
 * [AssemblyLine Submit](https://misp.github.io/misp-modules/expansion/#assemblyline-submit) - A module to submit samples and URLs to AssemblyLine for advanced analysis, and return the link of the submission.
 * [Backscatter.io](https://misp.github.io/misp-modules/expansion/#backscatter.io) - Backscatter.io module to bring mass-scanning observations into MISP.
 * [BGP Ranking](https://misp.github.io/misp-modules/expansion/#bgp-ranking) - Query BGP Ranking to get the ranking of an Autonomous System number.
 * [BTC Scam Check](https://misp.github.io/misp-modules/expansion/#btc-scam-check) - An expansion hover module to query a special dns blacklist to check if a bitcoin address has been abused.
 * [BTC Steroids](https://misp.github.io/misp-modules/expansion/#btc-steroids) - An expansion hover module to get a blockchain balance from a BTC address in MISP.
 * [Censys Enrich](https://misp.github.io/misp-modules/expansion/#censys-enrich) - An expansion module to enrich attributes in MISP by quering the censys.io API
@ -136,7 +135,6 @@ For further Information see the [license file](https://misp.github.io/misp-modul
 * [Vulnerability Lookup](https://misp.github.io/misp-modules/expansion/#vulnerability-lookup) - An expansion module to query Vulnerability Lookup
 * [Vulners Lookup](https://misp.github.io/misp-modules/expansion/#vulners-lookup) - An expansion hover module to expand information about CVE id using Vulners API.
 * [Vysion Enrich](https://misp.github.io/misp-modules/expansion/#vysion-enrich) - Module to enrich the information by making use of the Vysion API.
 * [Whois Lookup](https://misp.github.io/misp-modules/expansion/#whois-lookup) - Module to query a local instance of uwhois (https://github.com/rafiot/uwhoisd).
 * [WhoisFreaks Lookup](https://misp.github.io/misp-modules/expansion/#whoisfreaks-lookup) - An expansion module for https://whoisfreaks.com/ that will provide an enriched analysis of the provided domain, including WHOIS and DNS information.
 * [Wikidata Lookup](https://misp.github.io/misp-modules/expansion/#wikidata-lookup) - An expansion hover module to extract information from Wikidata to have additional information about particular term for analysis.
 * [IBM X-Force Exchange Lookup](https://misp.github.io/misp-modules/expansion/#ibm-x-force-exchange-lookup) - An expansion module for IBM X-Force Exchange.
--- a/188
+++ b/188
@ -1,188 +0,0 @@
 -i https://pypi.org/simple
 aiohappyeyeballs==2.3.5; python_version >= '3.8'
 aiohttp==3.10.2; python_version >= '3.8'
 aiosignal==1.3.1; python_version >= '3.7'
 annotated-types==0.7.0; python_version >= '3.8'
 antlr4-python3-runtime==4.9.3
 anyio==4.4.0; python_version >= '3.8'
 apiosintds==2.0.3; python_version >= '3.6'
 appdirs==1.4.4
 assemblyline-client==4.9.3
 attrs==24.2.0; python_version >= '3.7'
 backoff==1.11.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
 backscatter==0.2.4
 beautifulsoup4==4.12.3; python_full_version >= '3.6.0'
 bidict==0.23.1; python_version >= '3.8'
 blockchain==1.4.4
 cattrs==23.2.3; python_version >= '3.8'
 censys==2.0.9; python_version < '4.0' and python_full_version >= '3.6.2'
 certifi==2024.7.4; python_version >= '3.6'
 cffi==1.17.0; platform_python_implementation != 'PyPy'
 chardet==5.2.0; python_version >= '3.7'
 charset-normalizer==3.3.2; python_full_version >= '3.7.0'
 clamd==1.0.2
 click==8.1.7; python_version >= '3.7'
 click-plugins==1.1.1
 colorama==0.4.6; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6'
 colorclass==2.2.2; python_version >= '2.6'
 commonmark==0.9.1
 compressed-rtf==1.0.6
 configparser==7.0.0; python_version >= '3.8'
 contourpy==1.2.1; python_version >= '3.9'
 crowdstrike-falconpy==0.9.0; python_version >= '3.6'
 cryptography==43.0.0; python_version >= '3.7'
 cycler==0.12.1; python_version >= '3.8'
 deprecated==1.2.14; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 dnsdb2==1.1.4
 dnspython==2.6.1; python_version >= '3.8'
 domaintools-api==2.0.0; python_version >= '3.6'
 easygui==0.98.3
 ebcdic==1.1.1
 enum-compat==0.0.3
 et-xmlfile==1.1.0; python_version >= '3.6'
 extract-msg==0.48.7
 ezodf==0.3.2
 filelock==3.15.4; python_version >= '3.8'
 fonttools==4.53.1; python_version >= '3.8'
 frozenlist==1.4.1; python_version >= '3.8'
 future==1.0.0; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
 geoip2==4.8.0; python_version >= '3.8'
 h11==0.14.0; python_version >= '3.7'
 httpcore==1.0.5; python_version >= '3.8'
 httplib2==0.22.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 httpx==0.27.0; python_version >= '3.8'
 idna==3.7; python_version >= '3.5'
 isodate==0.6.1
 jbxapi==3.23.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
 jinja2==3.1.4; python_version >= '3.7'
 json-log-formatter==1.0; python_version >= '3.6'
 jsonschema==4.23.0; python_version >= '3.8'
 jsonschema-specifications==2023.12.1; python_version >= '3.8'
 kiwisolver==1.4.5; python_version >= '3.7'
 lark==1.1.9; python_version >= '3.6'
 lief==0.15.1
 lxml==5.2.2; python_version >= '3.6'
 maclookup==1.0.3
 markdownify==0.5.3
 markupsafe==2.1.5; python_version >= '3.7'
 matplotlib==3.9.1.post1; python_version >= '3.9'
 mattermostdriver==7.3.2; python_version >= '3.5'
 maxminddb==2.6.2; python_version >= '3.8'
 -e .
 more-itertools==10.4.0; python_version >= '3.8'
 msoffcrypto-tool==5.4.2; python_version >= '3' and platform_python_implementation != 'PyPy' or (platform_system != 'Windows' and platform_system != 'Darwin')
 multidict==6.0.5; python_version >= '3.7'
 mwdblib==3.4.1
 ndjson==0.3.1
 np==1.0.2
 numpy==1.26.4; python_version >= '3.9'
 oauth2==1.9.0.post1
 odtreader@ git+https://github.com/cartertemm/ODTReader.git/@49d6938693f6faa3ff09998f86dba551ae3a996b
 olefile==0.47; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
 oletools==0.60.2
 opencv-python==4.10.0.84; python_version >= '3.6'
 openpyxl==3.1.5; python_version >= '3.8'
 packaging==24.1; python_version >= '3.8'
 pandas==2.2.2; python_version >= '3.9'
 pandas-ods-reader==1.0.1; python_version < '3.13' and python_version >= '3.9'
 passivetotal==2.5.9
 pcodedmp==1.2.6
 pdftotext==2.2.2
 pillow==10.4.0; python_version >= '3.8'
 platformdirs==4.2.2; python_version >= '3.8'
 progressbar2==4.4.2; python_version >= '3.8'
 psutil==6.0.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
 publicsuffixlist==1.0.2.20240810; python_version >= '3.5'
 pybgpranking@ git+https://github.com/D4-project/BGP-Ranking.git/#subdirectory=client
 pycountry==22.3.5; python_version >= '3.6' and python_version < '4'
 pycparser==2.22; python_version >= '3.8'
 pycryptodome==3.20.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
 pycryptodomex==3.20.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
 pydantic==2.8.2; python_version >= '3.8'
 pydantic-core==2.20.1; python_version >= '3.8'
 pydeep2==0.5.1
 pydnstrails@ git+https://github.com/sebdraven/pydnstrails@48c1f740025c51289f43a24863d1845ff12fd21a
 pyeupi==1.3.0; python_version >= '3.8' and python_version < '4.0'
 pyfaup==1.2
 pygeoip==0.3.2
 pygments==2.18.0; python_version >= '3.8'
 pyintel471@ git+https://github.com/MISP/PyIntel471.git@917272fafa8e12102329faca52173e90c5256968
 pyipasnhistory@ git+https://github.com/D4-project/IPASN-History.git/#subdirectory=client
 pymisp[fileobjects,openioc,pdfexport,email,url]==2.4.195; python_version >= '3.8' and python_version < '4.0'
 pyonyphe@ git+https://github.com/sebdraven/pyonyphe@d1d6741f8ea4475f3bb77ff20c876f08839cabd1
 pyparsing==2.4.7; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
 pypdns==2.2.3; python_version >= '3.8' and python_version < '4.0'
 pypssl==2.2; python_version >= '3.6' and python_version < '4.0'
 pysafebrowsing==0.1.3
 pytesseract==0.3.10; python_version >= '3.7'
 python-baseconv==1.2.2
 python-dateutil==2.9.0.post0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 python-docx==1.1.2; python_version >= '3.7'
 python-engineio==4.9.1; python_version >= '3.6'
 python-magic==0.4.27
 python-pptx==1.0.2; python_version >= '3.8'
 python-socketio[client]==5.11.3; python_version >= '3.8'
 python-utils==3.8.2; python_version >= '3.9'
 pytz==2024.1
 pyyaml==6.0.2; python_version >= '3.8'
 pyzbar==0.1.9
 pyzipper==0.3.6; python_version >= '3.5'
 rdflib==7.0.0; python_full_version >= '3.8.1' and python_full_version < '4.0.0'
 red-black-tree-mod==1.20
 redis==5.0.8; python_version >= '3.7'
 referencing==0.35.1; python_version >= '3.8'
 reportlab==4.2.2; python_version >= '3.7' and python_version < '4'
 requests[security]==2.32.3; python_version >= '3.8'
 requests-cache==1.2.1; python_version >= '3.8'
 requests-file==2.1.0
 rich==10.16.2; python_full_version >= '3.6.2' and python_full_version < '4.0.0'
 rpds-py==0.20.0; python_version >= '3.8'
 rtfde==0.1.2
 ruamel.yaml==0.18.6; python_version >= '3.7'
 ruamel.yaml.clib==0.2.8; python_version < '3.13' and platform_python_implementation == 'CPython'
 setuptools==72.1.0; python_version >= '3.8'
 shellingham==1.5.4; python_version >= '3.7'
 shodan==1.31.0
 sigmatools==0.23.1; python_version ~= '3.8'
 sigmf==1.2.2; python_version >= '3.7'
 simple-websocket==1.0.0; python_version >= '3.6'
 simplejson==3.19.2; python_version >= '2.5' and python_version not in '3.0, 3.1, 3.2, 3.3'
 six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 slack-sdk==3.27.1; python_version >= '3.6'
 sniffio==1.3.1; python_version >= '3.7'
 socialscan==1.4.0; python_version >= '3.6'
 socketio-client==0.5.7.4
 softenum==1.0.1; python_full_version >= '3.7.0' and python_full_version < '4.0.0'
 soupsieve==2.5; python_version >= '3.8'
 sparqlwrapper==2.0.0; python_version >= '3.7'
 stix2==3.0.1; python_version >= '3.6'
 stix2-patterns==2.0.0; python_version >= '3.6'
 tabulate==0.9.0; python_version >= '3.7'
 tau-clients==0.3.0; python_version >= '3.6'
 taxii2-client==2.3.0
 termcolor==2.4.0; python_version >= '3.8'
 tldextract==5.1.2; python_version >= '3.8'
 tornado==6.4.1; python_version >= '3.8'
 tqdm==4.66.5; python_version >= '3.7'
 trustar@ git+https://github.com/SteveClement/trustar-python.git@6954eae38e0c77eaeef26084b6c5fd033925c1c7
 typer==0.12.3; python_version >= '3.7'
 typing-extensions==4.12.2; python_version >= '3.8'
 tzdata==2024.1; python_version >= '2'
 tzlocal==5.2; python_version >= '3.8'
 unicodecsv==0.14.1
 url-normalize==1.4.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
 urlarchiver==0.2
 urllib3==2.2.2; python_version >= '3.8'
 vt-graph-api==2.2.0
 vt-py==0.18.3; python_full_version >= '3.7.0'
 vulners==2.2.0; python_version >= '3.8'
 vysion==2.0.8; python_full_version >= '3.8.0' and python_full_version < '4.0.0'
 wand==0.6.13
 websocket-client==1.8.0; python_version >= '3.8'
 websockets==12.0; python_version >= '3.8'
 wrapt==1.16.0; python_version >= '3.6'
 wsproto==1.2.0; python_full_version >= '3.7.0'
 xlrd==2.0.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
 xlsxwriter==3.2.0; python_version >= '3.6'
 yara-python==4.5.0
 yarl==1.9.4; python_version >= '3.7'
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@ -1,130 +0,0 @@
 FROM python:3.7-buster AS build
 ENV DEBIAN_FRONTEND noninteractive
 ENV WORKDIR="/usr/local/src/misp_modules"
 ENV VENV_DIR="/misp_modules"
 # Install Packages for build
 RUN set -eu \
        ;mkdir -p ${WORKDIR} ${VENV_DIR} \
        ;apt-get update  \
        ;apt-get install -y \
                git \
                libpq5 \
                libjpeg-dev \
                tesseract-ocr \
                libpoppler-cpp-dev \
                imagemagick \
                virtualenv \
                libopencv-dev \
                zbar-tools \
                libzbar0 \
                libzbar-dev \
                libfuzzy-dev \
        ;apt-get -y autoremove \
        ;apt-get -y clean \
        ;rm -rf /var/lib/apt/lists/* \
        ;
 # Create MISP Modules
 RUN set -eu \
        ;git clone https://github.com/MISP/misp-modules.git ${WORKDIR} \
        ;virtualenv -p python3 ${VENV_DIR}/venv \
        ;cd ${WORKDIR} \
        ;${VENV_DIR}/venv/bin/pip3 install -I -r REQUIREMENTS --no-cache-dir \
        ;${VENV_DIR}/venv/bin/pip3 install . --no-cache-dir \
        ;
 #########################################
 # Start Final Docker Image
 #
 FROM python:3.7-slim-buster AS final
 ENV DEBIAN_FRONTEND noninteractive 
 ENV VENV_DIR="/misp_modules"
 # Copy all builded files from build stage
 COPY --from=build ${VENV_DIR} ${VENV_DIR}
 # Install Packages to run it
 RUN set -eu \
        ;apt-get update  \
        ;apt-get install -y \
                curl \
                libpq5 \
                # libjpeg-dev \
                tesseract-ocr \
                libpoppler-cpp-dev \
                imagemagick \
                # virtualenv \
                # libopencv-dev \
                zbar-tools \
                libzbar0 \
                # libzbar-dev \
                # libfuzzy-dev \
        ;apt-get -y autoremove \
        ;apt-get -y clean \
        ;rm -rf /var/lib/apt/lists/* \
        ;chown -R nobody ${VENV_DIR} \
        ;
 # Entrypoint
        COPY files/entrypoint.sh /entrypoint.sh
        ENTRYPOINT [ "/entrypoint.sh" ]
 # Add Healthcheck Config
        COPY files/healthcheck.sh /healthcheck.sh
        HEALTHCHECK --interval=1m --timeout=45s --retries=3 CMD ["/healthcheck.sh"]
 # Change Workdir
        WORKDIR ${VENV_DIR}
 # Change from root to www-data
        USER nobody
 # Expose Port
        EXPOSE 6666
 # Shortterm ARG Variables:
        ARG VENDOR="MISP"
        ARG COMPONENT="misp-modules"
        ARG BUILD_DATE
        ARG GIT_REPO="https://github.com/MISP/misp-modules"
        ARG VCS_REF
        ARG RELEASE_DATE
        ARG NAME="MISP-dockerized-misp-modules"
        ARG DESCRIPTION="This docker container contains MISP modules in an Debian Container."
        ARG DOCUMENTATION="https://misp.github.io/misp-modules/"
        ARG AUTHOR="MISP"
        ARG LICENSE="BSD-3-Clause"
 # Longterm Environment Variables
 ENV \
        BUILD_DATE=${BUILD_DATE} \
        NAME=${NAME} \
        PATH=$PATH:${VENV_DIR}/venv/bin
 # Labels
 LABEL org.label-schema.build-date="${BUILD_DATE}" \
        org.label-schema.name="${NAME}" \
        org.label-schema.description="${DESCRIPTION}" \
        org.label-schema.vcs-ref="${VCS_REF}" \
        org.label-schema.vcs-url="${GIT_REPO}" \
        org.label-schema.url="${GIT_REPO}" \
        org.label-schema.vendor="${VENDOR}" \
        org.label-schema.version="${VERSION}" \
        org.label-schema.usage="${DOCUMENTATION}" \
        org.label-schema.schema-version="1.0.0-rc1"
 LABEL   org.opencontainers.image.created="${BUILD_DATE}" \
        org.opencontainers.image.url="${GIT_REPO}" \
        org.opencontainers.image.source="${GIT_REPO}" \
        org.opencontainers.image.version="${VERSION}" \
        org.opencontainers.image.revision="${VCS_REF}" \
        org.opencontainers.image.vendor="${VENDOR}" \
        org.opencontainers.image.title="${NAME}" \
        org.opencontainers.image.description="${DESCRIPTION}" \
        org.opencontainers.image.documentation="${DOCUMENTATION}" \
        org.opencontainers.image.authors="${AUTHOR}" \
        org.opencontainers.image.licenses="${LICENSE}"
--- a/docker/files/entrypoint.sh
+++ b/docker/files/entrypoint.sh
@ -1,37 +0,0 @@
 #!/bin/sh
 set -eu
 # Variables
 NC='\033[0m' # No Color
 Light_Green='\033[1;32m' 
 STARTMSG="${Light_Green}[ENTRYPOINT_MISP_MODULES]${NC}"
 VENV_DIR=${VENV_DIR:-"/misp-modules"}
 MISP_MODULES_BINARY="${VENV_DIR}/venv/bin/misp-modules"
 DEBUG=""
 # Functions
 echo (){
    command echo "$STARTMSG $*"
 }
 # Environment Variables
 MISP_MODULES_DEBUG=${MISP_MODULES_DEBUG:-"false"}
 #
 #   MAIN
 #
 # Check if debugging mode should be enabled
 [ "$MISP_MODULES_DEBUG" = "true" ] && DEBUG="-d"
 # check if a command parameter exists and start misp-modules
 if [ $# = 0 ]
 then
    # If no cmd parameter is set
    echo "Start MISP Modules" && $MISP_MODULES_BINARY $DEBUG -l 0.0.0.0  > /dev/stdout 2> /dev/stderr
 else
    # If cmd parameter is set
    echo "Start MISP Modules" && $MISP_MODULES_BINARY $DEBUG -l 0.0.0.0 > /dev/stdout 2> /dev/stderr &
    exec "$@"
 fi
--- a/docker/files/healthcheck.sh
+++ b/docker/files/healthcheck.sh
@ -1,4 +0,0 @@
 #!/bin/sh
 # If no contain is there or curl get an error back: exit 1. Docker restart then the container.
 curl -fk http://0.0.0.0:6666/modules || exit 1
--- a/docs/index.md
+++ b/docs/index.md
@ -21,7 +21,6 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj
 * [AssemblyLine Query](https://misp.github.io/misp-modules/expansion/#assemblyline-query) - A module tu query the AssemblyLine API with a submission ID to get the submission report and parse it.
 * [AssemblyLine Submit](https://misp.github.io/misp-modules/expansion/#assemblyline-submit) - A module to submit samples and URLs to AssemblyLine for advanced analysis, and return the link of the submission.
 * [Backscatter.io](https://misp.github.io/misp-modules/expansion/#backscatter.io) - Backscatter.io module to bring mass-scanning observations into MISP.
 * [BGP Ranking](https://misp.github.io/misp-modules/expansion/#bgp-ranking) - Query BGP Ranking to get the ranking of an Autonomous System number.
 * [BTC Scam Check](https://misp.github.io/misp-modules/expansion/#btc-scam-check) - An expansion hover module to query a special dns blacklist to check if a bitcoin address has been abused.
 * [BTC Steroids](https://misp.github.io/misp-modules/expansion/#btc-steroids) - An expansion hover module to get a blockchain balance from a BTC address in MISP.
 * [Censys Enrich](https://misp.github.io/misp-modules/expansion/#censys-enrich) - An expansion module to enrich attributes in MISP by quering the censys.io API
@ -115,7 +114,6 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj
 * [Vulnerability Lookup](https://misp.github.io/misp-modules/expansion/#vulnerability-lookup) - An expansion module to query Vulnerability Lookup
 * [Vulners Lookup](https://misp.github.io/misp-modules/expansion/#vulners-lookup) - An expansion hover module to expand information about CVE id using Vulners API.
 * [Vysion Enrich](https://misp.github.io/misp-modules/expansion/#vysion-enrich) - Module to enrich the information by making use of the Vysion API.
 * [Whois Lookup](https://misp.github.io/misp-modules/expansion/#whois-lookup) - Module to query a local instance of uwhois (https://github.com/rafiot/uwhoisd).
 * [WhoisFreaks Lookup](https://misp.github.io/misp-modules/expansion/#whoisfreaks-lookup) - An expansion module for https://whoisfreaks.com/ that will provide an enriched analysis of the provided domain, including WHOIS and DNS information.
 * [Wikidata Lookup](https://misp.github.io/misp-modules/expansion/#wikidata-lookup) - An expansion hover module to extract information from Wikidata to have additional information about particular term for analysis.
 * [IBM X-Force Exchange Lookup](https://misp.github.io/misp-modules/expansion/#ibm-x-force-exchange-lookup) - An expansion module for IBM X-Force Exchange.
--- a/docs/install.md
+++ b/docs/install.md
@ -1,199 +1,147 @@
-## How to install and start MISP modules (in a Python virtualenv)? (recommended)
+IMPORTANT NOTE: we will soon be publishing `misp-modules` on PyPI.
-***Be sure to run the latest version of `pip`***. To install the latest version of pip, `pip install --upgrade pip` will do the job.
+
 ## Install from pip
 It is strongly recommended to use a virtual environment (see here for instructions https://docs.python.org/3/tutorial/venv.html).
 Once the virtual environment is loaded just use the command:
 ~~~~bash
-SUDO_WWW="sudo -u www-data"
+pip install misp-modules
 sudo apt-get install -y \
  python3-dev \
  python3-pip \
  git \
  libpq5 \
  libjpeg-dev \
  tesseract-ocr \
  libpoppler-cpp-dev \
  imagemagick virtualenv \
  libopencv-dev \
  zbar-tools \
  libzbar0 \
  libzbar-dev \
  libfuzzy-dev \
  libcaca-dev \
  build-essential
 # BEGIN with virtualenv:
 $SUDO_WWW virtualenv -p python3 /var/www/MISP/venv
 # END with virtualenv
 cd /usr/local/src/
 # Ideally you add your user to the staff group and make /usr/local/src group writeable, below follows an example with user misp
 sudo adduser misp staff
 sudo chmod 2775 /usr/local/src
 sudo chown root:staff /usr/local/src
 git clone https://github.com/MISP/misp-modules.git
 git clone git://github.com/stricaud/faup.git faup
 git clone git://github.com/stricaud/gtcaca.git gtcaca
 # Install gtcaca/faup
 cd gtcaca
 mkdir -p build
 cd build
 cmake .. && make
 sudo make install
 cd ../../faup
 mkdir -p build
 cd build
 cmake .. && make
 sudo make install
 sudo ldconfig
 cd ../../misp-modules
 # BEGIN with virtualenv:
 $SUDO_WWW  /var/www/MISP/venv/bin/pip install -I -r REQUIREMENTS
 $SUDO_WWW  /var/www/MISP/venv/bin/pip install .
 # END with virtualenv
 # BEGIN without virtualenv:
 sudo pip install -I -r REQUIREMENTS
 sudo pip install .
 # END without virtualenv
 # Start misp-modules as a service
 sudo cp etc/systemd/system/misp-modules.service /etc/systemd/system/
 sudo systemctl daemon-reload
 sudo systemctl enable --now misp-modules
 sudo service misp-modules start  # or
 /var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s & # to start the modules manually
 ~~~~
 ## How to install and start MISP modules on RHEL-based distributions ?
-As of this writing, the official RHEL repositories only contain Ruby 2.0.0 and Ruby 2.1 or higher is required. As such, this guide installs Ruby 2.2 from the [SCL](https://access.redhat.com/documentation/en-us/red_hat_software_collections/3/html/3.2_release_notes/chap-installation#sect-Installation-Subscribe) repository.
+## Install from cloned repository
 In this case the only requirement is to install `poetry`. Normally you just need to run `pip install poetry`, but see here for more alternatives https://python-poetry.org/docs/#installation.
 Once `poetry` is installed, you can clone the repository and install `misp-modules` as follows:
 ~~~~bash
-SUDO_WWW="sudo -u apache"
+git clone https://github.com/MISP/misp-modules.git && cd misp-modules
-sudo yum install \
+git submodule update --init
-  rh-python36 \
+poetry install
  rh-ruby22 \
  openjpeg-devel \
  rubygem-rouge \
  rubygem-asciidoctor \
  zbar-devel \
  opencv-devel \
  gcc-c++ \
  pkgconfig \
  poppler-cpp-devel \
  python-devel \
  redhat-rpm-config
 cd /var/www/MISP
 $SUDO_WWW git clone https://github.com/MISP/misp-modules.git
 cd misp-modules
 $SUDO_WWW /usr/bin/scl enable rh-python36 "virtualenv -p python3 /var/www/MISP/venv"
 $SUDO_WWW /var/www/MISP/venv/bin/pip install -U -I -r REQUIREMENTS
 $SUDO_WWW /var/www/MISP/venv/bin/pip install -U .
 ~~~~
-Create the service file /etc/systemd/system/misp-modules.service :
+Note that the dependencies will require a number of system packages installed. On Ubuntu these packages are `libpoppler-cpp-dev`, `libzbar0`, and `tesseract-ocr`. For an updated list, check the github action used to test the build inside `.github/workflows`.
 ## Install the systemd unit
 To run `misp-modules` as a service on a distribution based on systemd, you need to create the unit as follows and store it in a file `/etc/systemd/system/misp-modules.service`:
 ~~~~bash
-echo "[Unit]
+[Unit]
-Description=MISP's modules
+Description=MISP modules
 After=misp-workers.service
 [Service]
 Type=simple
 User=apache
 Group=apache
-ExecStart=/usr/bin/scl enable rh-python36 rh-ruby22  '/var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s'
+ExecStart='/path/to/venv/bin/misp-modules -l 127.0.0.1 -s'
 Restart=always
 RestartSec=10
 [Install]
-WantedBy=multi-user.target" | sudo tee /etc/systemd/system/misp-modules.service
+WantedBy=multi-user.target
 ~~~~
 The `After=misp-workers.service` must be changed or removed if you have not created a misp-workers service.
 Then, enable the misp-modules service and start it:
 ~~~~bash
 systemctl daemon-reload
 systemctl enable --now misp-modules
 ~~~~
 ## How to use an MISP modules Docker container
-### Docker build
+## Run the tests
 To run tests you need to install misp-modules from the cloned repository, run the server, and then run the tests. You can do all these step with `poetry`.
 ~~~~bash
-docker build -t misp-modules \
+poetry install
-    --build-arg BUILD_DATE=$(date -u +"%Y-%m-%d") \
+poetry run misp-modules
  docker/
 ~~~~
-### Docker run
+And in another terminal:
 ~~~~bash
-# Start Redis
+poetry run pytest
 docker run --rm -d --name=misp-redis redis:alpine
 # Start MISP-modules
 docker run \
    --rm -d --name=misp-modules \
    -e REDIS_BACKEND=misp-redis \
    -e REDIS_PORT="6379" \
    -e REDIS_PW="" \
    -e REDIS_DATABASE="245" \
    -e MISP_MODULES_DEBUG="false" \
    dcso/misp-dockerized-misp-modules
 ~~~~
 ### Docker-compose
-~~~~yml
+## Build the documentation
 services:
  misp-modules:
    # https://hub.docker.com/r/dcso/misp-dockerized-misp-modules
    image: dcso/misp-dockerized-misp-modules:3
-    # Local image:
+To build the documentation you can use the provided `Makefile`.
-    #image: misp-modules
+Inside you will find three targets:
    #build:
    #  context: docker/
-    environment:
+- `generate_docs`: install the depdendency and generate the documentation.
-      # Redis
+
-      REDIS_BACKEND: misp-redis
+- `generate_docs`: build the documentation using `mkdocs`.
-      REDIS_PORT: "6379"
+
-      REDIS_DATABASE: "245"
+- `deploy`: deploy the documentation using `mkdocs gh-deploy`.
-      # System PROXY (OPTIONAL)
+
-      http_proxy:
+- `test-docs`: run a local server exposing the newly built documentation.
-      https_proxy:
+
-      no_proxy: 0.0.0.0
+Note that you can either run the targets using `poetry` (default), or using the `squidfunk/mkdocs-material` by setting the environment variable `USE_DOCKER=true`.
-      # Timezone (OPTIONAL)
+
-      TZ: Europe/Berlin
+
-      # MISP-Modules (OPTIONAL)
+## Run MISP modules
-      MISP_MODULES_DEBUG: "false"
+
-      # Logging options (OPTIONAL)
+If you installed it using pip, you just need to execute the command `misp-modules` (source the virtual environment a second time to update the search paths). If you installed it from the cloned repository, just use poetry, i.e., `poetry run misp-modules`.
-      LOG_SYSLOG_ENABLED: "no"
+
-  misp-redis:
+
-    # https://hub.docker.com/_/redis or alternative https://hub.docker.com/r/dcso/misp-dockerized-redis/
+## Run MISP modules in Docker
-    image: redis:alpine
+
 You can find an up-to-date container image and related documentation at the following repository:m https://github.com/MISP/misp-docker 
 ## Install misp-module on an offline instance
 ### If `misp-modules` is available on PyPI
 Once `misp-modules` is available on PyPI, you can just download all the necessary packages:
 ~~~~bash
 mkdir wheels
 pip wheel misp-modules --no-cache-dir -w ./wheels
 ~~~~
-## Install misp-module on an offline instance.
+Move the `wheels` directory to the target system, and install them there:
 First, you need to grab all necessary packages for example like this :
-Use pip wheel to create an archive
+~~~~bash
-~~~
+pip install --no-cache-dir --use-deprecated=legacy-resolver /wheels/*.whl
-mkdir misp-modules-offline
+~~~~
-pip3 wheel -r REQUIREMENTS shodan --wheel-dir=./misp-modules-offline
+
-tar -cjvf misp-module-bundeled.tar.bz2 ./misp-modules-offline/*
+Once again, using a virtual environment is recommended.
-~~~
+
-On offline machine :
+### If `misp-modules` is not available on PyPI
-~~~
+
-mkdir misp-modules-bundle
+You have two choices, the first approach uses `poetry export` to export the entire virtual environment so you can copy and run it on the target system; the second one uses `poetry bundle` to export a `requirements.txt` file.
-tar xvf misp-module-bundeled.tar.bz2 -C misp-modules-bundle
+
-cd misp-modules-bundle
+#### Using `poetry bundle`
-ls -1|while read line; do sudo pip3 install --force-reinstall --ignore-installed --upgrade --no-index --no-deps ${line};done
+
-~~~
+This is quite straightforward but it assumes your target system is relatively similar (same distribution, architecture, libaries).
-Next you can follow standard install procedure.
+
 ~~~~bash
 poetry install
 poetry self add poetry-plugin-bundle
 poetry bundle venv /destination/path/
 ~~~~
 #### Using `poetry export`
 This is a bit more convoluted and it is similar to how you would install `misp-modules` on an offline instance.
 Just follow those instructions but replace the package `misp-modules` with `-r requirements.txt`.
 Before doing so you need to generate the `requirements.txt` file. Due to the fact we are still supporting Python 3.8 and that Poetry still has some limitations (soon to be resolved) you need to need to replace the line `python = ">=3.8.*,<3.13"` inside `pyproject.toml` with your exact version (just run `python --version`).
 Once you have done that, run the following commands to generate your very own `requirements.txt`.
 ~~~~bash
 poetry lock
 poetry install
 poetry self add poetry-plugin-export
 poetry export --without-hashes -f requirements.txt -o requirements.txt
 ~~~~
--- a/documentation/README.md
+++ b/documentation/README.md
@ -175,28 +175,6 @@ Backscatter.io module to bring mass-scanning observations into MISP.
 -----
 #### [BGP Ranking](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/bgpranking.py)
 Query BGP Ranking to get the ranking of an Autonomous System number.
 [[source code](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/bgpranking.py)]
 - **features**:
 >The module takes an AS number attribute as input and displays its description as well as its ranking position in BGP Ranking for a given day.
 - **input**:
 >Autonomous system number.
 - **output**:
 >An asn object with its related bgp-ranking object.
 - **references**:
 >https://github.com/D4-project/BGP-Ranking/
 - **requirements**:
 >pybgpranking python library
 -----
 #### [BTC Scam Check](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/btc_scam_check.py)
 <img src=logos/bitcoin.png height=60>
@ -2843,32 +2821,6 @@ Module to enrich the information by making use of the Vysion API.
 -----
 #### [Whois Lookup](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/whois.py)
 Module to query a local instance of uwhois (https://github.com/rafiot/uwhoisd).
 [[source code](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/whois.py)]
 - **features**:
 >This module takes a domain or IP address attribute as input and queries a 'Univseral Whois proxy server' to get the correct details of the Whois query on the input value (check the references for more details about this whois server).
 - **config**:
 > - server
 > - port
 - **input**:
 >A domain or IP address attribute.
 - **output**:
 >Text describing the result of a whois request for the input value.
 - **references**:
 >https://github.com/rafiot/uwhoisd
 - **requirements**:
 >uwhois: A whois python library
 -----
 #### [WhoisFreaks Lookup](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/whoisfreaks.py)
 <img src=logos/whoisfreaks.png height=60>
--- a/documentation/generate_documentation.py
+++ b/documentation/generate_documentation.py
--- a/documentation/migrate_web_json_to_module.py
+++ b/documentation/migrate_web_json_to_module.py
@ -1,96 +0,0 @@
 import json
 import re
 from pathlib import Path
 import os
 module_types = ['expansion', 'export_mod', 'import_mod']
 moduleinfo_template = {
    'version': '1.0',
    'author': '',
    'module-type': [],
    'description': '',
    'logo': '',
    'requirements': [],
    'features': '',
    'references': [],
    'input': '',
    'output': ''
 }
 if __name__ == '__main__':
    exit("This code was temporary and should not be run again. It was used to migrate the JSON documentation to the module files.")
    root_path = Path(__file__).resolve().parent.parent
    modules_path = root_path / 'misp_modules' / 'modules'
    for module_type in module_types:
        files = sorted(os.listdir(modules_path / module_type))
        for python_filename in files:
            if not python_filename.endswith('.py') or '__init__' in python_filename:
                continue
            modulename = python_filename.split('.py')[0]
            json_filename = root_path / 'documentation' / 'website' / module_type / f'{modulename}.json'
            print(f"Processing type {module_type}:{modulename} in {python_filename} and {json_filename}")
            json_exists = json_filename.exists()
            if json_exists:
                print(" Found JSON file")
                with open(json_filename, 'rt') as f:
                    json_content = json.loads(f.read())
            else:
                json_content = {}
            # if json does not exist, then still edit the python file and add the stub structure
            with open(modules_path / module_type / python_filename, 'r+t') as python_f:
                # read from python file, find moduleinfo and load it as python variable
                python_content = python_f.read()
                re_pattern = r'moduleinfo\s=\s{[^}]*}'
                m = re.search(re_pattern, python_content, re.MULTILINE | re.DOTALL)
                if not m:
                    print(f" Moduleinfo not found in {python_filename}")
                    continue
                s = m.group(0)
                moduleinfo = {}
                exec(s)  # we now have a moduleinfo dict
                print(f" Moduleinfo found in {python_filename}: {moduleinfo}")
                # populate from template
                for k, v in moduleinfo_template.items():
                    if k not in moduleinfo or moduleinfo.get(k) == '' or moduleinfo.get(k) == []:
                        # print(f" Adding {k} = {v} to {python_filename}")
                        moduleinfo[k] = v
                # populate from json
                for k, v in json_content.items():
                    if k not in moduleinfo or moduleinfo.get(k) == '' or moduleinfo.get(k) == []:
                        # print(f" Adding {k} = {v} to {python_filename}")
                        moduleinfo[k] = v
                if json_content and json_content.get('description') != moduleinfo.get('description'):
                    print(" WARNING: Description in JSON and Python file do not match:")
                    print("")
                    print(f" JSON: {json_content.get('description')}")
                    print("")
                    print(f" Python: {moduleinfo.get('description')}")
                    print("")
                    user_input = input("Which version do you want to use? Enter '[j]son' for JSON version or '[p]ython' for Python version, or any other text for a new description: ")
                    if user_input in ['json', 'j', 'JSON']:
                        moduleinfo['description'] = json_content['description']
                    elif user_input in ['python', 'p', 'PYTHON']:
                        pass
                    else:
                        moduleinfo['description'] = user_input.strip()
                # write back to python file
                new_moduleinfo_text = ['moduleinfo = {']
                for k, v in moduleinfo.items():
                    v_updated = repr(v).replace('\\', '\\\\')
                    new_moduleinfo_text.append(f"    '{k}': {v_updated},")
                new_moduleinfo_text.append('}')
                python_content_new, cnt = re.subn(re_pattern, '\n'.join(new_moduleinfo_text), python_content, re.MULTILINE | re.DOTALL)
                if cnt == 0:
                    print(f" WARNING: Moduleinfo not replaced in {python_filename}")
                    continue
                python_f.seek(0)
                python_f.write(python_content_new)
                python_f.truncate()  # remove the rest of the file
                pass
    pass
--- a/documentation/mkdocs/REQUIREMENTS.txt
+++ b/documentation/mkdocs/REQUIREMENTS.txt
@ -1,3 +0,0 @@
 mkdocs
 mkdocs-material
 markdown_include
--- a/documentation/mkdocs/expansion.md
+++ b/documentation/mkdocs/expansion.md
@ -172,28 +172,6 @@ Backscatter.io module to bring mass-scanning observations into MISP.
 -----
 #### [BGP Ranking](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/bgpranking.py)
 Query BGP Ranking to get the ranking of an Autonomous System number.
 [[source code](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/bgpranking.py)]
 - **features**:
 >The module takes an AS number attribute as input and displays its description as well as its ranking position in BGP Ranking for a given day.
 - **input**:
 >Autonomous system number.
 - **output**:
 >An asn object with its related bgp-ranking object.
 - **references**:
 >https://github.com/D4-project/BGP-Ranking/
 - **requirements**:
 >pybgpranking python library
 -----
 #### [BTC Scam Check](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/btc_scam_check.py)
 <img src=../logos/bitcoin.png height=60>
@ -2840,32 +2818,6 @@ Module to enrich the information by making use of the Vysion API.
 -----
 #### [Whois Lookup](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/whois.py)
 Module to query a local instance of uwhois (https://github.com/rafiot/uwhoisd).
 [[source code](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/whois.py)]
 - **features**:
 >This module takes a domain or IP address attribute as input and queries a 'Univseral Whois proxy server' to get the correct details of the Whois query on the input value (check the references for more details about this whois server).
 - **config**:
 > - server
 > - port
 - **input**:
 >A domain or IP address attribute.
 - **output**:
 >Text describing the result of a whois request for the input value.
 - **references**:
 >https://github.com/rafiot/uwhoisd
 - **requirements**:
 >uwhois: A whois python library
 -----
 #### [WhoisFreaks Lookup](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/whoisfreaks.py)
 <img src=../logos/whoisfreaks.png height=60>
--- a/documentation/mkdocs/index.md
+++ b/documentation/mkdocs/index.md
@ -21,7 +21,6 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj
 * [AssemblyLine Query](https://misp.github.io/misp-modules/expansion/#assemblyline-query) - A module tu query the AssemblyLine API with a submission ID to get the submission report and parse it.
 * [AssemblyLine Submit](https://misp.github.io/misp-modules/expansion/#assemblyline-submit) - A module to submit samples and URLs to AssemblyLine for advanced analysis, and return the link of the submission.
 * [Backscatter.io](https://misp.github.io/misp-modules/expansion/#backscatter.io) - Backscatter.io module to bring mass-scanning observations into MISP.
 * [BGP Ranking](https://misp.github.io/misp-modules/expansion/#bgp-ranking) - Query BGP Ranking to get the ranking of an Autonomous System number.
 * [BTC Scam Check](https://misp.github.io/misp-modules/expansion/#btc-scam-check) - An expansion hover module to query a special dns blacklist to check if a bitcoin address has been abused.
 * [BTC Steroids](https://misp.github.io/misp-modules/expansion/#btc-steroids) - An expansion hover module to get a blockchain balance from a BTC address in MISP.
 * [Censys Enrich](https://misp.github.io/misp-modules/expansion/#censys-enrich) - An expansion module to enrich attributes in MISP by quering the censys.io API
@ -115,7 +114,6 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj
 * [Vulnerability Lookup](https://misp.github.io/misp-modules/expansion/#vulnerability-lookup) - An expansion module to query Vulnerability Lookup
 * [Vulners Lookup](https://misp.github.io/misp-modules/expansion/#vulners-lookup) - An expansion hover module to expand information about CVE id using Vulners API.
 * [Vysion Enrich](https://misp.github.io/misp-modules/expansion/#vysion-enrich) - Module to enrich the information by making use of the Vysion API.
 * [Whois Lookup](https://misp.github.io/misp-modules/expansion/#whois-lookup) - Module to query a local instance of uwhois (https://github.com/rafiot/uwhoisd).
 * [WhoisFreaks Lookup](https://misp.github.io/misp-modules/expansion/#whoisfreaks-lookup) - An expansion module for https://whoisfreaks.com/ that will provide an enriched analysis of the provided domain, including WHOIS and DNS information.
 * [Wikidata Lookup](https://misp.github.io/misp-modules/expansion/#wikidata-lookup) - An expansion hover module to extract information from Wikidata to have additional information about particular term for analysis.
 * [IBM X-Force Exchange Lookup](https://misp.github.io/misp-modules/expansion/#ibm-x-force-exchange-lookup) - An expansion module for IBM X-Force Exchange.
--- a/documentation/mkdocs/install.md
+++ b/documentation/mkdocs/install.md
@ -1,199 +1,159 @@
-## How to install and start MISP modules (in a Python virtualenv)? (recommended)
+## Install from pip
-***Be sure to run the latest version of `pip`***. To install the latest version of pip, `pip install --upgrade pip` will do the job.
+It is strongly recommended to use a virtual environment (see here for instructions https://docs.python.org/3/tutorial/venv.html).
 Once the virtual environment is loaded just use the command:
 ~~~~bash
-SUDO_WWW="sudo -u www-data"
+pip install misp-modules
 sudo apt-get install -y \
  python3-dev \
  python3-pip \
  git \
  libpq5 \
  libjpeg-dev \
  tesseract-ocr \
  libpoppler-cpp-dev \
  imagemagick virtualenv \
  libopencv-dev \
  zbar-tools \
  libzbar0 \
  libzbar-dev \
  libfuzzy-dev \
  libcaca-dev \
  build-essential
 # BEGIN with virtualenv:
 $SUDO_WWW virtualenv -p python3 /var/www/MISP/venv
 # END with virtualenv
 cd /usr/local/src/
 # Ideally you add your user to the staff group and make /usr/local/src group writeable, below follows an example with user misp
 sudo adduser misp staff
 sudo chmod 2775 /usr/local/src
 sudo chown root:staff /usr/local/src
 git clone https://github.com/MISP/misp-modules.git
 git clone git://github.com/stricaud/faup.git faup
 git clone git://github.com/stricaud/gtcaca.git gtcaca
 # Install gtcaca/faup
 cd gtcaca
 mkdir -p build
 cd build
 cmake .. && make
 sudo make install
 cd ../../faup
 mkdir -p build
 cd build
 cmake .. && make
 sudo make install
 sudo ldconfig
 cd ../../misp-modules
 # BEGIN with virtualenv:
 $SUDO_WWW  /var/www/MISP/venv/bin/pip install -I -r REQUIREMENTS
 $SUDO_WWW  /var/www/MISP/venv/bin/pip install .
 # END with virtualenv
 # BEGIN without virtualenv:
 sudo pip install -I -r REQUIREMENTS
 sudo pip install .
 # END without virtualenv
 # Start misp-modules as a service
 sudo cp etc/systemd/system/misp-modules.service /etc/systemd/system/
 sudo systemctl daemon-reload
 sudo systemctl enable --now misp-modules
 sudo service misp-modules start  # or
 /var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s & # to start the modules manually
 ~~~~
-## How to install and start MISP modules on RHEL-based distributions ?
+Note: this install method might not yet be available.
-As of this writing, the official RHEL repositories only contain Ruby 2.0.0 and Ruby 2.1 or higher is required. As such, this guide installs Ruby 2.2 from the [SCL](https://access.redhat.com/documentation/en-us/red_hat_software_collections/3/html/3.2_release_notes/chap-installation#sect-Installation-Subscribe) repository.
+
 ## Install from cloned repository
 In this case the only requirement is to install `poetry`. Normally you just need to run `pip install poetry`, but see here for more alternatives https://python-poetry.org/docs/#installation.
 Once `poetry` is installed, you can clone the repository and install `misp-modules` as follows:
 ~~~~bash
-SUDO_WWW="sudo -u apache"
+git clone https://github.com/MISP/misp-modules.git && cd misp-modules
-sudo yum install \
+git submodule update --init
-  rh-python36 \
+poetry install
  rh-ruby22 \
  openjpeg-devel \
  rubygem-rouge \
  rubygem-asciidoctor \
  zbar-devel \
  opencv-devel \
  gcc-c++ \
  pkgconfig \
  poppler-cpp-devel \
  python-devel \
  redhat-rpm-config
 cd /var/www/MISP
 $SUDO_WWW git clone https://github.com/MISP/misp-modules.git
 cd misp-modules
 $SUDO_WWW /usr/bin/scl enable rh-python36 "virtualenv -p python3 /var/www/MISP/venv"
 $SUDO_WWW /var/www/MISP/venv/bin/pip install -U -I -r REQUIREMENTS
 $SUDO_WWW /var/www/MISP/venv/bin/pip install -U .
 ~~~~
-Create the service file /etc/systemd/system/misp-modules.service :
+Note that the dependencies will require a number of system packages installed. On Ubuntu these packages are `libpoppler-cpp-dev`, `libzbar0`, and `tesseract-ocr`. For an updated list, check the github action used to test the build inside `.github/workflows`.
 ## Install the systemd unit
 To run `misp-modules` as a service on a distribution based on systemd, you need to create the unit as follows and store it in a file `/etc/systemd/system/misp-modules.service`:
 ~~~~bash
-echo "[Unit]
+[Unit]
-Description=MISP's modules
+Description=MISP modules
 After=misp-workers.service
 [Service]
 Type=simple
 User=apache
 Group=apache
-ExecStart=/usr/bin/scl enable rh-python36 rh-ruby22  '/var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s'
+ExecStart='/path/to/venv/bin/misp-modules -l 127.0.0.1 -s'
 Restart=always
 RestartSec=10
 [Install]
-WantedBy=multi-user.target" | sudo tee /etc/systemd/system/misp-modules.service
+WantedBy=multi-user.target
 ~~~~
 The `After=misp-workers.service` must be changed or removed if you have not created a misp-workers service.
 Then, enable the misp-modules service and start it:
 ~~~~bash
 systemctl daemon-reload
 systemctl enable --now misp-modules
 ~~~~
 ## How to use an MISP modules Docker container
-### Docker build
+## Run the tests
 To run tests you need to install misp-modules from the cloned repository, run the server, and then run the tests. You can do all these step with `poetry`.
 ~~~~bash
-docker build -t misp-modules \
+poetry install
-    --build-arg BUILD_DATE=$(date -u +"%Y-%m-%d") \
+poetry run misp-modules
  docker/
 ~~~~
-### Docker run
+And in another terminal:
 ~~~~bash
-# Start Redis
+poetry run pytest
 docker run --rm -d --name=misp-redis redis:alpine
 # Start MISP-modules
 docker run \
    --rm -d --name=misp-modules \
    -e REDIS_BACKEND=misp-redis \
    -e REDIS_PORT="6379" \
    -e REDIS_PW="" \
    -e REDIS_DATABASE="245" \
    -e MISP_MODULES_DEBUG="false" \
    dcso/misp-dockerized-misp-modules
 ~~~~
 ### Docker-compose
-~~~~yml
+## Build the documentation
 services:
  misp-modules:
    # https://hub.docker.com/r/dcso/misp-dockerized-misp-modules
    image: dcso/misp-dockerized-misp-modules:3
-    # Local image:
+To build the documentation you can use the provided `Makefile`.
-    #image: misp-modules
+Inside you will find three targets:
    #build:
    #  context: docker/
-    environment:
+- `generate_docs`: install the depdendency and generate the documentation.
-      # Redis
+
-      REDIS_BACKEND: misp-redis
+- `generate_docs`: build the documentation using `mkdocs`.
-      REDIS_PORT: "6379"
+
-      REDIS_DATABASE: "245"
+- `deploy`: deploy the documentation using `mkdocs gh-deploy`.
-      # System PROXY (OPTIONAL)
+
-      http_proxy:
+- `test-docs`: run a local server exposing the newly built documentation.
-      https_proxy:
+
-      no_proxy: 0.0.0.0
+Note: you can either run the targets using `poetry` (default), or using the Docker image `squidfunk/mkdocs-material` by setting the environment variable `USE_DOCKER=true`.
-      # Timezone (OPTIONAL)
+
-      TZ: Europe/Berlin
+
-      # MISP-Modules (OPTIONAL)
+## Run MISP modules
-      MISP_MODULES_DEBUG: "false"
+
-      # Logging options (OPTIONAL)
+If you installed it using pip, you just need to execute the command `misp-modules` (source the virtual environment a second time to update the search paths). If you installed it from the cloned repository, just use poetry, i.e., `poetry run misp-modules`.
-      LOG_SYSLOG_ENABLED: "no"
+
-  misp-redis:
+
-    # https://hub.docker.com/_/redis or alternative https://hub.docker.com/r/dcso/misp-dockerized-redis/
+## Run MISP modules in Docker
-    image: redis:alpine
+
 You can find an up-to-date container image and related documentation at the following repository: https://github.com/MISP/misp-docker .
 ## Install misp-module on an offline instance
 ### If `misp-modules` is available on PyPI
 Once `misp-modules` is available on PyPI, you can just download all the necessary packages:
 ~~~~bash
 mkdir wheels
 pip wheel misp-modules --no-cache-dir -w ./wheels
 ~~~~
-## Install misp-module on an offline instance.
+Move the `wheels` directory to the target system, and install them there:
-First, you need to grab all necessary packages for example like this :
+
 ~~~~bash
 pip install --no-cache-dir --use-deprecated=legacy-resolver /wheels/*.whl
 ~~~~
 Once again, using a virtual environment is recommended.
 ### If `misp-modules` is not available on PyPI
 You have two choices, the first approach uses `poetry export` to export the entire virtual environment so you can copy and run it on the target system; the second one uses `poetry bundle` to export a `requirements.txt` file.
 #### Using `poetry bundle`
 This is quite straightforward but it assumes your target system is relatively similar (same distribution, architecture, libaries).
 ~~~~bash
 poetry install
 poetry self add poetry-plugin-bundle
 poetry bundle venv /destination/path/
 ~~~~
 #### Using `poetry export`
 This is a bit more convoluted and it is similar to how you would install `misp-modules` on an offline instance.
 Just follow those instructions but replace the package `misp-modules` with `-r requirements.txt`.
 Before doing so you need to generate the `requirements.txt` file. Due to the fact we are still supporting Python 3.8 and that Poetry still has some limitations (soon to be resolved) you need to need to replace the line `python = ">=3.8.*,<3.13"` inside `pyproject.toml` with your exact version (just run `python --version`).
 The following `sed` command does everything for you.
 ~~~~bash
 sed -i "s/^python = .*/python = \"$(python -c 'import platform; print(platform.python_version())')\"/" pyproject.toml
 ~~~~
 Then, run the following commands to generate your very own `requirements.txt`.
 ~~~~bash
 poetry lock
 poetry install
 poetry self add poetry-plugin-export
 poetry export --without-hashes -f requirements.txt -o requirements.txt
 ~~~~
 Note that `misp-modules` will not be part of the `requirements.txt` file and you will need to create the wheel yourself:
 ~~~~bash
 poetry build --output ./wheels
 ~~~~
 Use pip wheel to create an archive
 ~~~
 mkdir misp-modules-offline
 pip3 wheel -r REQUIREMENTS shodan --wheel-dir=./misp-modules-offline
 tar -cjvf misp-module-bundeled.tar.bz2 ./misp-modules-offline/*
 ~~~
 On offline machine :
 ~~~
 mkdir misp-modules-bundle
 tar xvf misp-module-bundeled.tar.bz2 -C misp-modules-bundle
 cd misp-modules-bundle
 ls -1|while read line; do sudo pip3 install --force-reinstall --ignore-installed --upgrade --no-index --no-deps ${line};done
 ~~~
 Next you can follow standard install procedure.
--- a/etc/systemd/system/misp-modules.service
+++ b/etc/systemd/system/misp-modules.service
@ -1,14 +0,0 @@
 [Unit]
 Description=System-wide instance of the MISP Modules
 After=network.target
 [Service]
 User=www-data
 Group=www-data
 WorkingDirectory=/usr/local/src/misp-modules
 Environment="PATH=/var/www/MISP/venv/bin"
 ExecStart=/var/www/MISP/venv/bin/misp-modules -l 127.0.0.1
 [Install]
 WantedBy=multi-user.target
--- a/misp_modules/init.py
+++ b/misp_modules/init.py
@ -28,6 +28,9 @@ import argparse
 import re
 import datetime
 import psutil
 import pkgutil
 import platform
 import typing
 try:
    import orjson as json
@ -41,6 +44,52 @@ from tornado.concurrent import run_on_executor
 from concurrent.futures import ThreadPoolExecutor
 from pymisp import pymisp_json_default
 import warnings
 warnings.filterwarnings("ignore", category=SyntaxWarning)
 LIBFAUP_PATHS = [
    "/usr/local/lib/",
    "/usr/lib/",
    "/opt/local/lib/",
 ]
 ARCH_TO_EXTENSION = {
    "linux": "so",
    "darwin": "dylib",
 }
 def _get_libfaup_path(lib_path: str) -> str:
    extension = ARCH_TO_EXTENSION.get(platform.system().lower(), "lib")
    return f"{lib_path.rstrip('/')}/libfaupl.{extension}"
 def _replace_libfaup_path(module_path: str, libfaup_path: str) -> None:
    with open(module_path, "r") as f:
        file_data = f.read()
    file_data = re.sub(r"cdll.LoadLibrary\(.*\)", f"cdll.LoadLibrary(\"{libfaup_path}\")", file_data)
    with open(module_path, "w") as f:
        f.write(file_data)
 def _try_pyfaup_import(lib_path: typing.Optional[str]) -> None:
    package = pkgutil.get_loader("pyfaup")
    if not package:
        return
    if lib_path:
        _replace_libfaup_path(package.path, _get_libfaup_path(lib_path))
    importlib.import_module("pyfaup")
 for lib_path in [None, *LIBFAUP_PATHS]:
    try:
        _try_pyfaup_import(lib_path)
        break
    except OSError:
        continue
 try:
    from .modules import *  # noqa
    HAS_PACKAGE_MODULES = True
--- a/misp_modules/helpers/cache.py
+++ b/misp_modules/helpers/cache.py
@ -27,12 +27,13 @@ import hashlib
 port = int(os.getenv("REDIS_PORT")) if os.getenv("REDIS_PORT") else 6379
 hostname = os.getenv("REDIS_BACKEND") or '127.0.0.1'
 db = int(os.getenv("REDIS_DATABASE")) if os.getenv("REDIS_DATABASE") else 0
 password = os.getenv("REDIS_PW") or None
 def selftest(enable=True):
    if not enable:
        return False
-    r = redis.Redis(host=hostname, port=port, db=db)
+    r = redis.Redis(host=hostname, password=password, port=port, db=db)
    try:
        r.ping()
    except Exception:
@ -42,7 +43,7 @@ def selftest(enable=True):
 def get(modulename=None, query=None, value=None, debug=False):
    if (modulename is None or query is None):
        return False
-    r = redis.Redis(host=hostname, port=port, db=db, decode_responses=True)
+    r = redis.Redis(host=hostname, password=password, port=port, db=db, decode_responses=True)
    h = hashlib.sha1()
    h.update(query.encode('UTF-8'))
    hv = h.hexdigest()
@ -60,7 +61,7 @@ def get(modulename=None, query=None, value=None, debug=False):
 def flush():
-    r = redis.StrictRedis(host=hostname, port=port, db=db, decode_responses=True)
+    r = redis.StrictRedis(host=hostname, password=password, port=port, db=db, decode_responses=True)
    returncode = r.flushdb()
    return returncode
--- a/misp_modules/modules/expansion/init.py
+++ b/misp_modules/modules/expansion/init.py
@ -3,10 +3,10 @@ import sys
 sys.path.append('{}/lib'.format('/'.join((os.path.realpath(__file__)).split('/')[:-3])))
-__all__ = ['cuckoo_submit', 'vmray_submit', 'bgpranking', 'circl_passivedns', 'circl_passivessl',
+__all__ = ['cuckoo_submit', 'vmray_submit', 'circl_passivedns', 'circl_passivessl',
           'cluster25_expand', 'countrycode', 'cve', 'cve_advanced', 'cpe', 'dns', 'btc_steroids', 'domaintools',
           'eupi', 'eql', 'farsight_passivedns', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal',
-           'whois', 'shodan', 'reversedns', 'geoip_asn', 'geoip_city', 'geoip_country', 'wiki', 'iprep',
+           'shodan', 'reversedns', 'geoip_asn', 'geoip_city', 'geoip_country', 'wiki', 'iprep',
           'threatminer', 'otx', 'threatcrowd', 'vulndb', 'crowdstrike_falcon',
           'yara_syntax_validator', 'hashdd', 'onyphe', 'onyphe_full', 'rbl',
           'xforceexchange', 'sigma_syntax_validator', 'stix2_pattern_syntax_validator',
--- a/misp_modules/modules/expansion/bgpranking.py
+++ b/misp_modules/modules/expansion/bgpranking.py
@ -1,96 +0,0 @@
 # -*- coding: utf-8 -*-
 import json
 from . import check_input_attribute, standard_error_message
 from datetime import date, datetime, timedelta
 from pybgpranking import BGPRanking
 from pymisp import MISPAttribute, MISPEvent, MISPObject
 misperrors = {'error': 'Error'}
 mispattributes = {'input': ['AS'], 'format': 'misp_standard'}
 moduleinfo = {
    'version': '0.1',
    'author': 'Raphaël Vinot',
    'description': 'Query BGP Ranking to get the ranking of an Autonomous System number.',
    'module-type': ['expansion', 'hover'],
    'name': 'BGP Ranking',
    'logo': '',
    'requirements': ['pybgpranking python library'],
    'features': 'The module takes an AS number attribute as input and displays its description as well as its ranking position in BGP Ranking for a given day.',
    'references': ['https://github.com/D4-project/BGP-Ranking/'],
    'input': 'Autonomous system number.',
    'output': 'An asn object with its related bgp-ranking object.',
 }
 def handler(q=False):
    if q is False:
        return False
    request = json.loads(q)
    if not request.get('attribute') or not check_input_attribute(request['attribute']):
        return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'}
    toquery = request['attribute']
    if toquery['type'] not in mispattributes['input']:
        return {'error': 'Unsupported attribute type.'}
    bgpranking = BGPRanking()
    value_toquery = int(toquery['value'][2:]) if toquery['value'].startswith('AS') else int(toquery['value'])
    values = bgpranking.query(value_toquery, date=(date.today() - timedelta(1)).isoformat())
    if not values['response'] or not values['response']['asn_description']:
        misperrors['error'] = 'There is no result about this ASN in BGP Ranking'
        return misperrors
    event = MISPEvent()
    attribute = MISPAttribute()
    attribute.from_dict(**toquery)
    event.add_attribute(**attribute)
    asn_object = MISPObject('asn')
    asn_object.add_attribute(**{
        'type': 'AS',
        'object_relation': 'asn',
        'value': values['meta']['asn']
    })
    description, country = values['response']['asn_description'].split(', ')
    for relation, value in zip(('description', 'country'), (description, country)):
        asn_object.add_attribute(**{
            'type': 'text',
            'object_relation': relation,
            'value': value
        })
    mapping = {
        'address_family': {'type': 'text', 'object_relation': 'address-family'},
        'date': {'type': 'datetime', 'object_relation': 'date'},
        'position': {'type': 'float', 'object_relation': 'position'},
        'rank': {'type': 'float', 'object_relation': 'ranking'}
    }
    bgp_object = MISPObject('bgp-ranking')
    for feature in ('rank', 'position'):
        bgp_attribute = {'value': values['response']['ranking'][feature]}
        bgp_attribute.update(mapping[feature])
        bgp_object.add_attribute(**bgp_attribute)
    date_attribute = {'value': datetime.strptime(values['meta']['date'], '%Y-%m-%d')}
    date_attribute.update(mapping['date'])
    bgp_object.add_attribute(**date_attribute)
    address_attribute = {'value': values['meta']['address_family']}
    address_attribute.update(mapping['address_family'])
    bgp_object.add_attribute(**address_attribute)
    asn_object.add_reference(attribute.uuid, 'describes')
    asn_object.add_reference(bgp_object.uuid, 'ranked-with')
    event.add_object(asn_object)
    event.add_object(bgp_object)
    event = json.loads(event.to_json())
    results = {key: event[key] for key in ('Attribute', 'Object')}
    return {'results': results}
 def introspection():
    return mispattributes
 def version():
    return moduleinfo
--- a/misp_modules/modules/expansion/ods_enrich.py
+++ b/misp_modules/modules/expansion/ods_enrich.py
@ -47,8 +47,11 @@ def handler(q=False):
    try:
        for i in range(0, num_sheets):
            rows = pandas_ods_reader.parsers.ods.get_rows(doc, i)
-            ods = pandas_ods_reader.algo.parse_data(pandas_ods_reader.parsers.ods, rows, headers=False, columns=[], skiprows=0)
+            try:
-            ods = pandas_ods_reader.utils.sanitize_df(ods)
+                ods = pandas_ods_reader.algo.parse_data(pandas_ods_reader.parsers.ods, rows, headers=False, columns=[], skiprows=0)
                ods = pandas_ods_reader.utils.sanitize_df(ods)
            except TypeError:
                ods = pandas_ods_reader.algo.read_data(pandas_ods_reader.parsers.ods, ods_file, i, headers=False)
            ods_content = ods_content + "\n" + ods.to_string(max_rows=None)
        return {'results': [{'types': ['freetext'], 'values': ods_content, 'comment': ".ods-to-text from file " + filename},
                            {'types': ['text'], 'values': ods_content, 'comment': ".ods-to-text from file " + filename}]}
--- a/misp_modules/modules/expansion/vysion.py
+++ b/misp_modules/modules/expansion/vysion.py
@ -54,7 +54,7 @@ moduleconfig = [
 LOGGER = logging.getLogger("vysion")
 LOGGER.setLevel(logging.INFO)
-LOGGER.info("Starting Vysion")
+LOGGER.debug("Starting Vysion")
 DEFAULT_RESULTS_LIMIT = 10
--- a/misp_modules/modules/expansion/whois.py
+++ b/misp_modules/modules/expansion/whois.py
@ -1,62 +0,0 @@
 # -*- coding: utf-8 -*-
 import json
 try:
    from uwhois import Uwhois
 except ImportError:
    print("uwhois module not installed.")
 misperrors = {'error': 'Error'}
 mispattributes = {'input': ['domain', 'ip-src', 'ip-dst'], 'output': ['freetext']}
 moduleinfo = {
    'version': '0.1',
    'author': 'Raphaël Vinot',
    'description': 'Module to query a local instance of uwhois (https://github.com/rafiot/uwhoisd).',
    'module-type': ['expansion'],
    'name': 'Whois Lookup',
    'logo': '',
    'requirements': ['uwhois: A whois python library'],
    'features': "This module takes a domain or IP address attribute as input and queries a 'Univseral Whois proxy server' to get the correct details of the Whois query on the input value (check the references for more details about this whois server).",
    'references': ['https://github.com/rafiot/uwhoisd'],
    'input': 'A domain or IP address attribute.',
    'output': 'Text describing the result of a whois request for the input value.',
 }
 moduleconfig = ['server', 'port']
 def handler(q=False):
    if q is False:
        return False
    request = json.loads(q)
    if request.get('domain'):
        toquery = request['domain']
    elif request.get('ip-src'):
        toquery = request['ip-src']
    elif request.get('ip-dst'):
        toquery = request['ip-dst']
    else:
        misperrors['error'] = "Unsupported attributes type"
        return misperrors
    if not request.get('config') or (not request['config'].get('server') and not request['config'].get('port')):
        misperrors['error'] = 'Whois local instance address is missing'
        return misperrors
    uwhois = Uwhois(request['config']['server'], int(request['config']['port']))
    if 'event_id' in request:
        return handle_expansion(uwhois, toquery)
 def handle_expansion(w, domain):
    return {'results': [{'types': mispattributes['output'], 'values': w.query(domain)}]}
 def introspection():
    return mispattributes
 def version():
    moduleinfo['config'] = moduleconfig
    return moduleinfo
--- a/poetry.lock
+++ b/poetry.lock
--- a/pyproject.toml
+++ b/pyproject.toml
@ -1,3 +1,120 @@
 [build-system]
-requires = ["setuptools", "wheel"]
+requires = ["poetry-core>=1.0.0"]
-build-backend = "setuptools.build_meta:__legacy__"
+build-backend = "poetry.core.masonry.api"
 [tool.poetry]
 name = "misp-modules"
 version = "2.4.195"
 description = "MISP modules are autonomous modules that can be used for expansion and other services in MISP"
 authors = ["Alexandre Dulaunoy <alexandre.dulaunoy@circl.lu>"]
 license = "AGPL-3.0-only"
 repository = "https://github.com/MISP/misp-modules"
 documentation = "https://misp.github.io/misp-modules"
 readme = "README.md"
 classifiers=[
    'License :: OSI Approved :: GNU Affero General Public License v3',
    'Development Status :: 5 - Production/Stable',
    'Environment :: Console',
    'Intended Audience :: Science/Research',
    'Programming Language :: Python :: 3',
    'Topic :: Security',
 ]
 packages = [{include = "misp_modules"}]
 [tool.poetry.urls]
 "Bug Tracker" = "https://github.com/MISP/misp-modules/issues"
 "Source" = "https://github.com/MISP/misp-modules"
 [tool.poetry.scripts]
 misp-modules = "misp_modules:main"
 [tool.poetry.dependencies]
 ## platform (pin this to your python version, for 'poetry export' to work)
 python = ">=3.8.*,<3.13"
 ## core dependencies
 psutil = "*"
 pyparsing = "*"
 redis = "*"
 tornado = "*"
 ## module dependencies (if a dependency fails loading with '*', pin it here)
 censys = "2.0.9"
 socialscan = "<2.0.0"
 yara-python = "4.5.0"
 # required to support both python 3.8 and wheel builds on python 3.12
 numpy = [{version = "1.24.4", python = "3.8.*"}, {version = ">=1.26.4,<2.0.0", python = ">=3.9"}]
 pandas = [{version = "1.5.3", python = "3.8.*"}, {version = ">=2.0.0", python = ">=3.9"}]
 pandas_ods_reader = [{version = "0.1.4", python = "3.8.*"}, {version = ">=1.0.0", python = ">=3.9"}]
 ## module dependencies
 apiosintds = "*"
 assemblyline_client = "*"
 backscatter = "*"
 blockchain = "*"
 clamd = "*"
 crowdstrike-falconpy = "*"
 dnsdb2 = "*"
 domaintools_api = "*"
 geoip2 = "*"
 google-search-api = { git = "https://github.com/abenassi/Google-Search-API" }
 greynoise = "*"
 jbxapi = "*"
 maclookup = "*"
 markdownify = "*"
 matplotlib = "*"
 mattermostdriver = "*"
 mwdblib = "*"
 ndjson = "*"
 np = "*"
 oauth2 = "*"
 odtreader = { git = "https://github.com/cartertemm/ODTReader.git" }
 opencv-python = "*"
 openpyxl = "*"
 passivetotal = "*"
 pdftotext = "*"
 pycountry = "*"
 pydnstrails = { git = "https://github.com/sebdraven/pydnstrails.git" }
 pyeti-python3 = "*"
 pyeupi = "*"
 pyfaup = "*"
 pygeoip = "*"
 pyintel471 = { git = "https://github.com/MISP/PyIntel471.git" }
 pyipasnhistory = "*"
 pymisp = { version = "*", extras = ["fileobjects", "openioc", "pdfexport", "email", "url"] }
 pyonyphe = { git = "https://github.com/sebdraven/pyonyphe.git" }
 pypdns = "*"
 pypssl = "*"
 pysafebrowsing = "*"
 pytesseract = "*"
 python-docx = "*"
 python-pptx = "*"
 pyzbar = "*"
 requests = { version = "*", extras = ["security"] }
 shodan = "*"
 sigmatools = "*"
 sigmf = "*"
 slack-sdk = "*"
 sparqlwrapper = "*"
 stix2 = "*"
 tau-clients = "*"
 taxii2-client = "*"
 trustar = { git = "https://github.com/SteveClement/trustar-python.git" }
 urlarchiver = "*"
 vt-graph-api = "*"
 vt-py = "*"
 vulners = "*"
 vysion = "*"
 wand = "*"
 xlrd = "*"
 [tool.poetry.group.dev.dependencies]
 codecov = "*"
 flake8 = "*"
 nose = "*"
 pytest = "*"
 [tool.poetry.group.docs]
 optional = true
 [tool.poetry.group.docs.dependencies]
 mkdocs = "*"
 mkdocs-material = "*"
 markdown_include = "*"
--- a/setup.py
+++ b/setup.py
@ -1,31 +0,0 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 from setuptools import setup, find_packages
 setup(
    name='misp-modules',
    version='1.0',
    author='Alexandre Dulaunoy',
    author_email='alexandre.dulaunoy@circl.lu',
    maintainer='Alexandre Dulaunoy',
    url='https://github.com/MISP/misp-modules',
    description='MISP modules are autonomous modules that can be used for expansion and other services in MISP',
    packages=find_packages(),
    entry_points={'console_scripts': ['misp-modules = misp_modules:main']},
    scripts=['tools/update_misp_modules.sh'],
    test_suite="tests",
    classifiers=[
        'License :: OSI Approved :: GNU Affero General Public License v3',
        'Development Status :: 5 - Production/Stable',
        'Environment :: Console',
        'Intended Audience :: Science/Research',
        'Programming Language :: Python :: 3',
        'Topic :: Security',
    ],
    install_requires=[
        'tornado',
        'psutil',
        'redis>=3',
        'pyparsing==2.4.7'
    ],
 )
--- a/tests/test_expansions.py
+++ b/tests/test_expansions.py
@ -133,18 +133,6 @@ class TestExpansions(unittest.TestCase):
            response = self.misp_modules_post(query)
            self.assertEqual(self.get_errors(response), 'An API key for APIVoid is required.')
    def test_bgpranking(self):
        query = {
            "module": "bgpranking",
            "attribute": {
                "type": "AS",
                "value": "13335",
                "uuid": "ea89a33b-4ab7-4515-9f02-922a0bee333d"
            }
        }
        response = self.misp_modules_post(query)
        self.assertEqual(self.get_first_object_type(response), 'asn')
    def test_btc_steroids(self):
        if LiveCI:
            return True
--- a/tools/update_misp_modules.sh
+++ b/tools/update_misp_modules.sh
@ -1,37 +0,0 @@
 #!/usr/bin/env bash
 set -e
 set -x
 # Updates the MISP Modules while respecting the current permissions
 # It aims to support the two following installation methods:
 # * Everything is runinng on the same machine following the MISP installation guide.
 # * The modules are installed using pipenv on a different machine from the one where MISP is running.
 if [ -d "/var/www/MISP" ] && [ -d "/usr/local/src/misp-modules" ]
 then
    echo "MISP is installed on the same machine, following the recommanded install script. Using MISP virtualenv."
    PATH_TO_MISP="/var/www/MISP"
    PATH_TO_MISP_MODULES="/usr/local/src/misp-modules"
    pushd ${PATH_TO_MISP_MODULES}
    USER=`stat -c "%U" .`
    sudo -H -u ${USER} git pull
    sudo -H -u ${USER} ${PATH_TO_MISP}/venv/bin/pip install -U -r REQUIREMENTS
    sudo -H -u ${USER} ${PATH_TO_MISP}/venv/bin/pip install -U -e .
    service misp-modules restart
    popd
 else
    if ! [ -x "$(command -v pipenv)" ]; then
        echo 'Error: pipenv not available, unable to automatically update.' >&2
        exit 1
    fi
    echo "Standalone mode, use pipenv from the current directory."
    git pull
    pipenv install
 fi
--- a/var/.gitkeep
+++ b/var/.gitkeep