Improve compatibility and upgrade python to 3.12

Changes:
* Remove vysion (not compatible with python 3.12 and no public repository)
* Remove stiximport (requires archaic version of pymisp)
* Update Python to 3.12
* Pin Numpy to 1.X
* Add missing dependencies
* Commit lock file
* Update requirements file

.github/workflows/python-package.yml

@@ -13,7 +13,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
 
     steps:
     - name: Install packages
@@ -28,9 +28,13 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        python -m pip install flake8 pytest
+        pip install flake8 pytest
+        pip install pipenv
+        sed -i "s/python_version.*/python_version = \"${{ matrix.python-version }}\"/" Pipfile
+        pipenv lock
+        pipenv requirements > requirements.txt
         # pyfaul must be installed manually (?)
-        pip install -r REQUIREMENTS pyfaup
+        pip install -r requirements.txt pyfaup
         pip install .
     - name: Lint with flake8
       run: |

Pipfile

@@ -10,6 +10,10 @@ pytest = "*"
 flake8 = "*"
 
 [packages]
+numpy = "<2.0.0"
+matplotlib = "*"
+sigmf = "*"
+pysafebrowsing = "*"
 dnspython = "*"
 requests = { extras = ["security"], version = "*" }
 urlarchiver = "*"
@@ -79,4 +83,4 @@ openpyxl = "*"
 slack-sdk = "3.27.1"
 
 [requires]
-python_version = "3.7"
+python_version = "3.12"

README.md

@@ -100,7 +100,6 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj
 * [VMware NSX](misp_modules/modules/expansion/vmware_nsx.py) - a module to enrich a file or URL with VMware NSX Defender.
 * [VulnDB](misp_modules/modules/expansion/vulndb.py) - a module to query [VulnDB](https://www.riskbasedsecurity.com/).
 * [Vulners](misp_modules/modules/expansion/vulners.py) - an expansion module to expand information about CVEs using Vulners API.
-* [Vysion](misp_modules/modules/expansion/vysion.py) - an expansion module to add dark web intelligence using Vysion API.
 * [whois](misp_modules/modules/expansion/whois.py) - a module to query a local instance of [uwhois](https://github.com/rafiot/uwhoisd).
 * [whoisfreaks](misp_modules/modules/expansion/whoisfreaks.py) - An expansion module for [whoisfreaks](https://whoisfreaks.com/) that will provide an enriched analysis of the provided domain, including WHOIS and DNS information.
 * [wikidata](misp_modules/modules/expansion/wiki.py) - a [wikidata](https://www.wikidata.org) expansion module.

REQUIREMENTS

@@ -1,188 +1,181 @@
-aiohttp>=3.9.0
+-i https://pypi.org/simple
-aiosignal==1.3.1 ; python_version >= '3.7'
+aiohttp==3.9.5; python_version >= '3.8'
+aiosignal==1.3.1; python_version >= '3.7'
 antlr4-python3-runtime==4.9.3
-anyio==3.6.2 ; python_full_version >= '3.6.2'
+anyio==4.4.0; python_version >= '3.8'
-apiosintDS==2.0.3
+apiosintds==2.0.3; python_version >= '3.6'
 appdirs==1.4.4
-argcomplete==3.0.8 ; python_version >= '3.6'
+assemblyline-client==4.9.3
-argparse==1.4.0
+attrs==23.2.0; python_version >= '3.7'
-assemblyline-client==4.5.1
+backoff==1.11.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
-async-timeout==4.0.2 ; python_version >= '3.6'
-asynctest==0.13.0 ; python_version < '3.8'
-attrs==23.1.0 ; python_version >= '3.7'
-backoff==2.2.1 ; python_version >= '3.7' and python_version < '4.0'
-backports.zoneinfo==0.2.1 ; python_version < '3.9'
 backscatter==0.2.4
-beautifulsoup4==4.12.2
+beautifulsoup4==4.12.3; python_full_version >= '3.6.0'
-bidict==0.22.1 ; python_version >= '3.7'
+bidict==0.23.1; python_version >= '3.8'
 blockchain==1.4.4
-censys==2.2.2
+cattrs==23.2.3; python_version >= '3.8'
-certifi>=2023.7.22 ; python_version >= '3.6'
+censys==2.0.9; python_full_version >= '3.6.2' and python_version < '4.0'
-cffi==1.15.1
+certifi==2024.7.4; python_version >= '3.6'
-chardet==5.1.0
+cffi==1.16.0; platform_python_implementation != 'PyPy'
-charset-normalizer==3.1.0 ; python_full_version >= '3.7.0'
+chardet==5.2.0; python_version >= '3.7'
+charset-normalizer==3.3.2; python_full_version >= '3.7.0'
 clamd==1.0.2
-click==8.1.3 ; python_version >= '3.7'
+click==8.1.7; python_version >= '3.7'
 click-plugins==1.1.1
-colorama==0.4.6 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6'
+colorama==0.4.6; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6'
-colorclass==2.2.2 ; python_version >= '2.6'
+colorclass==2.2.2; python_version >= '2.6'
+commonmark==0.9.1
 compressed-rtf==1.0.6
-configparser==5.3.0 ; python_version >= '3.7'
+configparser==7.0.0; python_version >= '3.8'
-crowdstrike-falconpy==1.2.15
+contourpy==1.2.1; python_version >= '3.9'
-cryptography>=41.0.2 ; python_version >= '3.6'
+crowdstrike-falconpy==0.9.0; python_version >= '3.6'
-dateparser==1.1.8 ; python_version >= '3.7'
+cryptography==42.0.8; python_version >= '3.7'
-decorator==5.1.1 ; python_version >= '3.5'
+cycler==0.12.1; python_version >= '3.8'
-deprecated==1.2.14 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+deprecated==1.2.14; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 dnsdb2==1.1.4
-dnspython==2.3.0
+dnspython==2.6.1; python_version >= '3.8'
-domaintools-api==1.0.1
+domaintools-api==2.0.0; python_version >= '3.6'
 easygui==0.98.3
 ebcdic==1.1.1
 enum-compat==0.0.3
-et-xmlfile==1.1.0 ; python_version >= '3.6'
+et-xmlfile==1.1.0; python_version >= '3.6'
-extract-msg==0.45.0
+extract-msg==0.48.7
 ezodf==0.3.2
-filelock==3.12.0 ; python_version >= '3.7'
+filelock==3.15.4; python_version >= '3.8'
-frozenlist==1.3.3 ; python_version >= '3.7'
+fonttools==4.53.1; python_version >= '3.8'
-future==0.18.3 ; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
+frozenlist==1.4.1; python_version >= '3.8'
-geoip2==4.7.0
+future==1.0.0; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'
-h11==0.14.0 ; python_version >= '3.7'
+geoip2==4.8.0; python_version >= '3.8'
-httpcore==0.17.1 ; python_version >= '3.7'
+h11==0.14.0; python_version >= '3.7'
-httplib2==0.22.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+httpcore==1.0.5; python_version >= '3.8'
-httpx==0.24.1 ; python_version >= '3.7'
+httplib2==0.22.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-idna==3.4 ; python_version >= '3.5'
+httpx==0.27.0; python_version >= '3.8'
-imapclient==2.3.1
+idna==3.7; python_version >= '3.5'
-importlib-metadata==4.13.0 ; python_version < '3.8'
-importlib-resources==5.12.0 ; python_version < '3.9'
 isodate==0.6.1
-itsdangerous==2.1.2 ; python_version >= '3.7'
+jbxapi==3.23.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
-jaraco.classes==3.2.3 ; python_version >= '3.7'
+jinja2==3.1.2; python_version >= '3.7'
-jbxapi==3.21.0
+json-log-formatter==1.0; python_version >= '3.6'
-jeepney==0.8.0 ; sys_platform == 'linux'
+jsonschema==4.23.0; python_version >= '3.8'
-jinja2==3.1.2
+jsonschema-specifications==2023.12.1; python_version >= '3.8'
-json-log-formatter==0.5.2 ; python_version >= '2.7'
+kiwisolver==1.4.5; python_version >= '3.7'
-jsonschema==4.19.0 ; python_version >= '3.7'
+lark==1.1.9; python_version >= '3.6'
-keyring==23.13.1 ; python_version >= '3.7'
+lief==0.14.1
-lark-parser==0.12.0
+lxml==5.2.2; python_version >= '3.6'
-lief==0.13.2
-lxml==4.9.2
 maclookup==1.0.3
-markdown-it-py==2.2.0 ; python_version >= '3.7'
 markdownify==0.5.3
-markupsafe==2.1.2 ; python_version >= '3.7'
+markupsafe==2.1.5; python_version >= '3.7'
-matplotlib==3.7.2 ; python_version >= '3.8'
+matplotlib==3.9.1; python_version >= '3.9'
-matplotlib==3.5.3 ; python_version == '3.7'
+mattermostdriver==7.3.2; python_version >= '3.5'
-mattermostdriver==7.3.2
+maxminddb==2.6.2; python_version >= '3.8'
-maxminddb==2.3.0 ; python_version >= '3.7'
+-e .
-mdurl==0.1.2 ; python_version >= '3.7'
+more-itertools==10.3.0; python_version >= '3.8'
-.
+msoffcrypto-tool==5.4.1; platform_python_implementation != 'PyPy' or (python_version >= '3' and platform_system != 'Windows' and platform_system != 'Darwin')
-more-itertools==9.1.0 ; python_version >= '3.7'
+multidict==6.0.5; python_version >= '3.7'
-msoffcrypto-tool==5.0.1 ; python_version >= '3' and platform_python_implementation != 'PyPy' or (platform_system != 'Windows' and platform_system != 'Darwin')
+mwdblib==3.4.1
-multidict==6.0.4 ; python_version >= '3.7'
-mwdblib==4.4.0
 ndjson==0.3.1
 np==1.0.2
-numpy==1.21.6 ; python_version < '3.10' and platform_machine == 'aarch64'
+numpy==1.26.4; python_version >= '3.9'
 oauth2==1.9.0.post1
-git+https://github.com/cartertemm/ODTReader.git/@49d6938693f6faa3ff09998f86dba551ae3a996b#egg=odtreader
+odtreader@ git+https://github.com/cartertemm/ODTReader.git/@49d6938693f6faa3ff09998f86dba551ae3a996b
-olefile==0.46 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+olefile==0.47; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
-oletools==0.60.1
+oletools==0.60.2
-opencv-python==4.7.0.72
+opencv-python==4.10.0.84; python_version >= '3.6'
-openpyxl==3.1.2
+openpyxl==3.1.5; python_version >= '3.8'
-packaging==23.1 ; python_version >= '3.7'
+packaging==24.1; python_version >= '3.8'
-pandas==1.5.3
+pandas==1.3.5; python_full_version >= '3.7.1'
-pandas-ods-reader==0.1.4
+pandas-ods-reader==0.1.2
 passivetotal==2.5.9
 pcodedmp==1.2.6
 pdftotext==2.2.2
-pillow>=10.2.0
+pillow==10.4.0; python_version >= '3.8'
-pkgutil-resolve-name==1.3.10 ; python_version < '3.9'
+platformdirs==4.2.2; python_version >= '3.8'
-progressbar2==4.2.0 ; python_full_version >= '3.7.0'
+progressbar2==4.4.2; python_version >= '3.8'
-psutil==5.9.5 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+psutil==6.0.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
-publicsuffixlist==0.10.0.20230828 ; python_version >= '2.6'
+publicsuffixlist==1.0.1.20240702; python_version >= '3.5'
-git+https://github.com/D4-project/BGP-Ranking.git/@68de39f6c5196f796055c1ac34504054d688aa59#egg=pybgpranking&subdirectory=client
+pybgpranking@ git+https://github.com/D4-project/BGP-Ranking.git/@68de39f6c5196f796055c1ac34504054d688aa59#subdirectory=client
-pycountry==22.3.5
+pycountry==22.3.5; python_version >= '3.6' and python_version < '4'
-pycparser==2.21
+pycparser==2.22; python_version >= '3.8'
-pycryptodome==3.19.1 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+pycryptodome==3.20.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
-pycryptodomex==3.19.1 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+pycryptodomex==3.20.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
 pydeep2==0.5.1
-git+https://github.com/sebdraven/pydnstrails@48c1f740025c51289f43a24863d1845ff12fd21a#egg=pydnstrails
+pydnstrails@ git+https://github.com/sebdraven/pydnstrails@48c1f740025c51289f43a24863d1845ff12fd21a
-pyeupi==1.1
+pyeupi==1.3.0; python_version >= '3.8' and python_version < '4.0'
 pyfaup==1.2
 pygeoip==0.3.2
-pygments==2.15.1 ; python_version >= '3.7'
+pygments==2.18.0; python_version >= '3.8'
-git+https://github.com/MISP/PyIntel471.git@917272fafa8e12102329faca52173e90c5256968#egg=pyintel471
+pyintel471@ git+https://github.com/MISP/PyIntel471.git@917272fafa8e12102329faca52173e90c5256968
-git+https://github.com/D4-project/IPASN-History.git/@a2853c39265cecdd0c0d16850bd34621c0551b87#egg=pyipasnhistory&subdirectory=client
+pyipasnhistory@ git+https://github.com/D4-project/IPASN-History.git/@a2853c39265cecdd0c0d16850bd34621c0551b87#subdirectory=client
-pymisp[email,fileobjects,openioc,pdfexport,url]==2.4.175
+pymisp[fileobjects,openioc,pdfexport,email,url]==2.4.194; python_version >= '3.8' and python_version < '4.0'
-git+https://github.com/sebdraven/pyonyphe@d1d6741f8ea4475f3bb77ff20c876f08839cabd1#egg=pyonyphe
+pyonyphe@ git+https://github.com/sebdraven/pyonyphe@d1d6741f8ea4475f3bb77ff20c876f08839cabd1
-pyparsing==2.4.7 ; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
+pyparsing==2.4.7; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'
-pypdns==1.5.2
+pypdns==2.2.3; python_version >= '3.8' and python_version < '4.0'
-pypssl==2.2
+pypssl==2.2; python_version >= '3.6' and python_version < '4.0'
-pyrsistent==0.19.3 ; python_version >= '3.7'
+pysafebrowsing==0.1.3
-pysafebrowsing==0.1.2
+pytesseract==0.3.10; python_version >= '3.7'
-pytesseract==0.3.10
 python-baseconv==1.2.2
-python-dateutil==2.8.2 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+python-dateutil==2.9.0.post0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'
-python-docx==0.8.11
+python-docx==1.1.2; python_version >= '3.7'
-python-engineio==4.4.1 ; python_version >= '3.6'
+python-engineio==4.9.1; python_version >= '3.6'
 python-magic==0.4.27
-python-pptx==0.6.21
+python-pptx==0.6.23
-python-socketio[client]==5.8.0 ; python_version >= '3.6'
+python-socketio[client]==5.11.3; python_version >= '3.8'
-python-utils==3.5.2 ; python_version >= '3.7'
+python-utils==3.8.2; python_version >= '3.9'
-pytz==2023.3
+pytz==2024.1
-pytz-deprecation-shim==0.1.0.post0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
+pyyaml==6.0.1; python_version >= '3.6'
-pyyaml==6.0 ; python_version >= '3.6'
 pyzbar==0.1.9
-pyzipper==0.3.6 ; python_version >= '3.5'
+pyzipper==0.3.6; python_version >= '3.5'
-rdflib==6.3.2 ; python_version >= '3.7' and python_version < '4.0'
+rdflib==7.0.0; python_full_version >= '3.8.1' and python_full_version < '4.0.0'
 red-black-tree-mod==1.20
-redis==4.5.5 ; python_version >= '3.7'
+redis==5.0.7; python_version >= '3.7'
-regex==2023.5.5 ; python_version >= '3.6'
+referencing==0.35.1; python_version >= '3.8'
-reportlab==4.0.4
+reportlab==4.2.2; python_version >= '3.7' and python_version < '4'
-requests[security]==2.31.0
+requests[security]==2.32.3; python_version >= '3.8'
-requests-cache==0.6.4 ; python_version >= '3.6'
+requests-cache==1.2.1; python_version >= '3.8'
-requests-file==1.5.1
+requests-file==2.1.0
-rich==13.3.5 ; python_full_version >= '3.7.0'
+rich==10.16.2; python_full_version >= '3.6.2' and python_full_version < '4.0.0'
-rtfde==0.1.0
+rpds-py==0.19.0; python_version >= '3.8'
-secretstorage==3.3.3 ; sys_platform == 'linux'
+rtfde==0.1.2
-setuptools==67.7.2 ; python_version >= '3.7'
+ruamel.yaml==0.18.6; python_version >= '3.7'
-shodan==1.29.1
+ruamel.yaml.clib==0.2.8; platform_python_implementation == 'CPython' and python_version < '3.13'
-sigmatools==0.19.1
+setuptools==70.3.0; python_version >= '3.8'
-sigmf==1.1.1
+shellingham==1.5.4; python_version >= '3.7'
-simplejson==3.19.1 ; python_version >= '2.5' and python_version not in '3.0, 3.1, 3.2, 3.3'
+shodan==1.31.0
-six==1.16.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+sigmatools==0.23.1; python_version ~= '3.8'
-slack-sdk==3.27.1
+sigmf==1.2.2; python_version >= '3.7'
-sniffio==1.3.0 ; python_version >= '3.7'
+simple-websocket==1.0.0; python_version >= '3.6'
-socialscan==1.4
+simplejson==3.19.2; python_version >= '2.5' and python_version not in '3.0, 3.1, 3.2'
+six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'
+slack-sdk==3.27.1; python_version >= '3.6'
+sniffio==1.3.1; python_version >= '3.7'
+socialscan==1.4.0; python_version >= '3.6'
 socketio-client==0.5.7.4
-soupsieve==2.4.1 ; python_version >= '3.7'
+soupsieve==2.5; python_version >= '3.8'
-sparqlwrapper==2.0.0
+sparqlwrapper==2.0.0; python_version >= '3.7'
-stix2==3.0.1
+stix2==3.0.1; python_version >= '3.6'
-stix2-patterns==2.0.0
+stix2-patterns==2.0.0; python_version >= '3.6'
-tabulate==0.9.0 ; python_version >= '3.7'
+tabulate==0.9.0; python_version >= '3.7'
-tau-clients==0.3.0
+tau-clients==0.3.0; python_version >= '3.6'
 taxii2-client==2.3.0
-tldextract==3.4.3 ; python_version >= '3.7'
+termcolor==2.4.0; python_version >= '3.8'
-tornado==6.2 ; python_version >= '3.7'
+tldextract==5.1.2; python_version >= '3.8'
-tqdm==4.65.0 ; python_version >= '3.7'
+tornado==6.4.1; python_version >= '3.8'
-git+https://github.com/SteveClement/trustar-python.git@6954eae38e0c77eaeef26084b6c5fd033925c1c7#egg=trustar
+tqdm==4.66.4; python_version >= '3.7'
-typing-extensions==4.5.0 ; python_version < '3.8'
+trustar@ git+https://github.com/SteveClement/trustar-python.git@6954eae38e0c77eaeef26084b6c5fd033925c1c7
-tzdata==2023.3 ; python_version >= '3.6'
+typer==0.12.3; python_version >= '3.7'
-tzlocal==4.2 ; python_version >= '3.6'
+typing-extensions==4.12.2; python_version >= '3.8'
+tzlocal==5.2; python_version >= '3.8'
 unicodecsv==0.14.1
-url-normalize==1.4.3 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
+url-normalize==1.4.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
 urlarchiver==0.2
-urllib3==1.26.15 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
+urllib3==2.2.2; python_version >= '3.8'
 vt-graph-api==2.2.0
-vt-py==0.17.5
+vt-py==0.18.2; python_full_version >= '3.7.0'
-vulners==2.0.10
+vulners==2.1.7; python_version >= '3.8'
-vysion==1.0.10
+wand==0.6.13
-wand==0.6.11
+websocket-client==1.8.0; python_version >= '3.8'
-websocket-client==1.5.1 ; python_version >= '3.7'
+websockets==12.0; python_version >= '3.8'
-websockets==11.0.3 ; python_version >= '3.7'
+wrapt==1.16.0; python_version >= '3.6'
-wrapt==1.15.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+wsproto==1.2.0; python_full_version >= '3.7.0'
-xlrd==2.0.1
+xlrd==2.0.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
-xlsxwriter==3.1.0 ; python_version >= '3.6'
+xlsxwriter==3.2.0; python_version >= '3.6'
 yara-python==3.8.1
-yarl==1.9.2 ; python_version >= '3.7'
+yarl==1.9.4; python_version >= '3.7'
-zipp==3.15.0 ; python_version >= '3.7'

documentation/README.md

@@ -1944,27 +1944,6 @@ An expansion hover module to expand information about CVE id using Vulners API.
 
 -----
 
-#### [vysion](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/vysion.py)
-
-<img src=logos/vysion.png height=60>
-
-Module to enrich the information by making use of the Vysion API.
-- **features**:
->This module gets correlated information from our dark web intelligence database. With this you will get several objects containing information related to, for example, an organization victim of a ransomware attack.
-- **input**:
->MISP Attribute which include: company(target-org), country, info.
-- **output**:
->MISP objects containing title, link to our webapp and TOR, i2p or clearnet URLs.
-- **references**:
-> - https://vysion.ai/
-> - https://developers.vysion.ai/
-> - https://github.com/ByronLabs/vysion-cti/tree/main
-- **requirements**:
-> - Vysion python library
-> - Vysion API Key
-
------
-
 #### [whois](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/whois.py)
 
 Module to query a local instance of uwhois (https://github.com/rafiot/uwhoisd).

documentation/mkdocs/expansion.md

@@ -1941,27 +1941,6 @@ An expansion hover module to expand information about CVE id using Vulners API.
 
 -----
 
-#### [vysion](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/vysion.py)
-
-<img src=../logos/vysion.png height=60>
-
-Module to enrich the information by making use of the Vysion API.
-- **features**:
->This module gets correlated information from our dark web intelligence database. With this you will get several objects containing information related to, for example, an organization victim of a ransomware attack.
-- **input**:
->MISP Attribute which include: company(target-org), country, info.
-- **output**:
->MISP objects containing title, link to our webapp and TOR, i2p or clearnet URLs.
-- **references**:
-> - https://vysion.ai/
-> - https://developers.vysion.ai/
-> - https://github.com/ByronLabs/vysion-cti/tree/main
-- **requirements**:
-> - Vysion python library
-> - Vysion API Key
-
------
-
 #### [whois](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/whois.py)
 
 Module to query a local instance of uwhois (https://github.com/rafiot/uwhoisd).

documentation/website/expansion/vysion.json

@@ -1,16 +0,0 @@
-{
-    "description": "Module to enrich the information by making use of the Vysion API.",
-    "logo": "vysion.png",
-    "requirements": [
-        "Vysion python library",
-        "Vysion API Key"
-    ],
-    "input": "MISP Attribute which include: company(target-org), country, info.",
-    "output": "MISP objects containing title, link to our webapp and TOR, i2p or clearnet URLs.",
-    "references": [
-        "https://vysion.ai/",
-        "https://developers.vysion.ai/",
-        "https://github.com/ByronLabs/vysion-cti/tree/main"        
-    ],
-    "features": "This module gets correlated information from our dark web intelligence database. With this you will get several objects containing information related to, for example, an organization victim of a ransomware attack."
-}

misp_modules/modules/expansion/__init__.py

@@ -20,7 +20,7 @@ __all__ = ['cuckoo_submit', 'vmray_submit', 'bgpranking', 'circl_passivedns', 'c
            'trustar_enrich', 'recordedfuture', 'html_to_markdown', 'socialscan', 'passive-ssh',
            'qintel_qsentry', 'mwdb', 'hashlookup', 'mmdb_lookup', 'ipqs_fraud_and_risk_scoring',
            'clamav', 'jinja_template_rendering','hyasinsight', 'variotdbs', 'crowdsec',
-           'extract_url_components', 'ipinfo', 'whoisfreaks', 'ip2locationio', 'vysion', 'stairwell',
+           'extract_url_components', 'ipinfo', 'whoisfreaks', 'ip2locationio', 'stairwell',
            'google_threat_intelligence', 'vulnerability_lookup']
 
 

misp_modules/modules/expansion/vysion.py

@@ -1,212 +0,0 @@
-import json
-from pymisp import MISPAttribute, MISPEvent
-from urllib.parse import urlparse
-
-import logging
-
-import vysion.client as vysion
-
-import vysion.dto as dto
-from vysion.dto.util import MISPProcessor
-
-misperrors = {"error": "Error"}
-mispattributes = {
-    "input": [
-        "email",
-        "domain",
-        "hostname",
-        "url",
-        "text",
-        "btc",
-        "phone-number",
-        "target-org",
-    ],
-    "format": "misp_standard",
-}
-
-# possible module-types: 'expansion', 'hover' or both
-moduleinfo = {
-    "version": "1",
-    "author": "Byron Labs",
-    "description": "Enrich observables with the Vysion API",
-    "module-type": ["expansion"],
-}
-
-# config fields that your code expects from the site admin
-moduleconfig = [
-    "apikey",
-    "event_limit",
-    "proxy_host",
-    "proxy_port",
-    "proxy_username",
-    "proxy_password",
-]
-
-LOGGER = logging.getLogger("vysion")
-LOGGER.setLevel(logging.INFO)
-LOGGER.info("Starting Vysion")
-
-DEFAULT_RESULTS_LIMIT = 10
-
-
-def get_proxy_settings(config: dict) -> dict:
-    """Returns proxy settings in the requests format.
-    If no proxy settings are set, return None."""
-    proxies = None
-    host = config.get("proxy_host")
-    port = config.get("proxy_port")
-    username = config.get("proxy_username")
-    password = config.get("proxy_password")
-
-    if host:
-        if not port:
-            misperrors["error"] = (
-                "The vysion_proxy_host config is set, "
-                "please also set the vysion_proxy_port."
-            )
-            raise KeyError
-        parsed = urlparse(host)
-        if "http" in parsed.scheme:
-            scheme = "http"
-        else:
-            scheme = parsed.scheme
-        netloc = parsed.netloc
-        host = f"{netloc}:{port}"
-
-        if username:
-            if not password:
-                misperrors["error"] = (
-                    "The vysion_proxy_username config is set, "
-                    "please also set the vysion_proxy_password."
-                )
-                raise KeyError
-            auth = f"{username}:{password}"
-            host = auth + "@" + host
-
-        proxies = {"http": f"{scheme}://{host}", "https": f"{scheme}://{host}"}
-    return proxies
-
-
-def parse_error(status_code: int) -> str:
-
-    status_mapping = {
-        500: "Vysion is blind.",
-        400: "Incorrect request, please check the arguments.",
-        403: "You don't have enough privileges to make the request.",
-    }
-
-    if status_code in status_mapping:
-        return status_mapping[status_code]
-
-    return "Vysion may not be accessible."
-
-
-def handler(q=False):
-
-    if q is False:
-        return False
-
-    request = json.loads(q)
-
-    if not request.get("config") or not request["config"].get("apikey"):
-        misperrors["error"] = "A Vysion api key is required for this module."
-        return misperrors
-
-    if not request.get("attribute"):
-        return {
-            "error": "The request is missing required attribute information, which should contain at least a type, a value, and a UUID."
-        }
-
-    if request["attribute"]["type"] not in mispattributes["input"]:
-        return {"error": "Unsupported attribute type."}
-
-    # event_limit = request["config"].get("event_limit")
-    attribute = request["attribute"]
-    proxy_settings = get_proxy_settings(request.get("config"))
-
-    try:
-
-        client = vysion.Client(
-            api_key=request["config"]["apikey"],
-            headers={
-                "x-tool": "MISPModuleVysionExpansion",
-            },
-            proxy=proxy_settings["http"] if proxy_settings else None,
-        )
-
-        LOGGER.debug(attribute)
-
-        misp_attribute = MISPAttribute()
-        misp_attribute.from_dict(**attribute)
-
-        attribute_type = misp_attribute.type
-        attribute_value = misp_attribute.value
-
-        # https://www.misp-project.org/datamodels/#types
-
-        LOGGER.debug(attribute_type)
-
-        result = None
-
-        if attribute_type == "email":
-            result = client.find_email(attribute_value)
-        elif attribute_type == "domain":
-            result = client.search(attribute_value) 
-        elif attribute_type == "url":
-            result = client.search(
-                attribute_value
-            )  # TODO result = client.find_url(attribute_value)
-        elif attribute_type == "text":
-            result = client.search(attribute_value)
-        elif attribute_type == "target-org":
-            result = client.search(attribute_value, exact=True)
-        elif attribute_type == "btc":
-            result = client.search(attribute_value)  # TODO
-        elif attribute_type == "phone-number":
-            result = client.search(attribute_value)  # TODO
-
-        if result is None:
-            return {"results": {}}
-        elif isinstance(result, dto.VysionError):
-            LOGGER.error(str(result))
-            return {"results": {}}
-
-        p = MISPProcessor()
-        misp_event: MISPEvent = p.process(result, ref_attribute=misp_attribute)
-
-        LOGGER.info("Vysion client initialized")
-
-        LOGGER.info("Vysion result obtained")
-
-        return {
-            "results": {
-                "Object": [
-                    json.loads(object.to_json()) for object in misp_event.objects
-                ],
-                "Attribute": [
-                    json.loads(attribute.to_json())
-                    for attribute in misp_event.attributes
-                ], 
-                "Tag": [
-                    json.loads(tag.to_json())
-                    for tag in misp_event.tags
-                ]
-            }
-        }
-
-    except vysion.APIError as ex:
-
-        LOGGER.error("Error in Vysion")
-        LOGGER.error(ex)
-
-        misperrors["error"] = ex.message
-        return misperrors
-
-
-def introspection():
-    return mispattributes
-
-
-def version():
-    moduleinfo["config"] = moduleconfig
-    return moduleinfo

misp_modules/modules/import_mod/stiximport.py

@@ -1,59 +0,0 @@
-import json
-import base64
-
-from pymisp.tools import stix
-
-misperrors = {'error': 'Error'}
-userConfig = {}
-inputSource = ['file']
-
-moduleinfo = {'version': '0.2', 'author': 'Hannah Ward',
-              'description': 'Import some stix stuff',
-              'module-type': ['import']}
-
-moduleconfig = []
-
-
-def handler(q=False):
-    # Just in case we have no data
-    if q is False:
-        return False
-
-    # The return value
-    r = {'results': []}
-
-    # Load up that JSON
-    q = json.loads(q)
-
-    # It's b64 encoded, so decode that stuff
-    package = base64.b64decode(q.get("data")).decode('utf-8')
-
-    # If something really weird happened
-    if not package:
-        return json.dumps({"success": 0})
-
-    pkg = stix.load_stix(package)
-    for attrib in pkg.attributes:
-        r["results"].append({"values": [attrib.value], "types": [attrib.type], "categories": [attrib.category]})
-
-    return r
-
-
-def introspection():
-    modulesetup = {}
-    try:
-        userConfig
-        modulesetup['userConfig'] = userConfig
-    except NameError:
-        pass
-    try:
-        inputSource
-        modulesetup['inputSource'] = inputSource
-    except NameError:
-        pass
-    return modulesetup
-
-
-def version():
-    moduleinfo['config'] = moduleconfig
-    return moduleinfo