Merge branch 'main' of github.com:MISP/misp-modules

Script used

sed -e "s/\/master\//\/main\//"

.github/workflows/python-package.yml

@@ -13,7 +13,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10"]
+        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
 
     steps:
     - name: Install packages

Pipfile

@@ -62,7 +62,8 @@ assemblyline_client = "*"
 vt-graph-api = "*"
 trustar = { git = "https://github.com/SteveClement/trustar-python.git" }
 markdownify = "==0.5.3"
-socialscan = "*"
+socialscan = "==1.4"
+pycountry = "==22.3.5"
 dnsdb2 = "*"
 clamd = "*"
 aiohttp = ">=3.7.4"

README.md

@@ -27,6 +27,7 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj
 * [CIRCL Passive DNS](misp_modules/modules/expansion/circl_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information.
 * [CIRCL Passive SSL](misp_modules/modules/expansion/circl_passivessl.py) - a hover and expansion module to expand IP addresses with the X.509 certificate(s) seen.
 * [countrycode](misp_modules/modules/expansion/countrycode.py) - a hover module to tell you what country a URL belongs to.
+* [CrowdSec](misp_modules/modules/expansion/crowdsec.py) - a hover module to expand using CrowdSec's CTI API.
 * [CrowdStrike Falcon](misp_modules/modules/expansion/crowdstrike_falcon.py) - an expansion module to expand using CrowdStrike Falcon Intel Indicator API.
 * [CPE](misp_modules/modules/expansion/cpe.py) - An expansion module to query the CVE Search API with a cpe code, to get its related vulnerabilities.
 * [CVE](misp_modules/modules/expansion/cve.py) - a hover module to give more information about a vulnerability (CVE).
@@ -43,13 +44,14 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj
 * [GeoIP](misp_modules/modules/expansion/geoip_country.py) - a hover and expansion module to get GeoIP information from geolite/maxmind.
 * [GeoIP_City](misp_modules/modules/expansion/geoip_city.py) - a hover and expansion module to get GeoIP City information from geolite/maxmind.
 * [GeoIP_ASN](misp_modules/modules/expansion/geoip_asn.py) - a hover and expansion module to get GeoIP ASN information from geolite/maxmind.
-* [Greynoise](misp_modules/modules/expansion/greynoise.py) - a hover to get information from greynoise.
+* [GreyNoise](misp_modules/modules/expansion/greynoise.py) - a hover and expansion module to get IP and CVE information from GreyNoise.
 * [hashdd](misp_modules/modules/expansion/hashdd.py) - a hover module to check file hashes against [hashdd.com](http://www.hashdd.com) including NSLR dataset.
 * [hibp](misp_modules/modules/expansion/hibp.py) - a hover module to lookup against Have I Been Pwned?
 * [html_to_markdown](misp_modules/modules/expansion/html_to_markdown.py) - Simple HTML to markdown converter
 * [HYAS Insight](misp_modules/modules/expansion/hyasinsight.py) - a hover and expansion module to get information from [HYAS Insight](https://www.hyas.com/hyas-insight).
 * [intel471](misp_modules/modules/expansion/intel471.py) - an expansion module to get info from [Intel471](https://intel471.com).
 * [IPASN](misp_modules/modules/expansion/ipasn.py) - a hover and expansion to get the BGP ASN of an IP address.
+* [ipinfo.io](misp_modules/modules/expansion/ipinfo.py) - an expansion module to get additional information on an IP address using the ipinfo.io API
 * [iprep](misp_modules/modules/expansion/iprep.py) - an expansion module to get IP reputation from packetmail.net.
 * [Joe Sandbox submit](misp_modules/modules/expansion/joesandbox_submit.py) - Submit files and URLs to Joe Sandbox.
 * [Joe Sandbox query](misp_modules/modules/expansion/joesandbox_query.py) - Query Joe Sandbox with the link of an analysis and get the parsed data.

REQUIREMENTS

@@ -1,82 +1,84 @@
 -i https://pypi.org/simple
-aiohttp==3.8.3
-aiosignal==1.2.0 ; python_version >= '3.6'
+aiohttp==3.8.4
+aiosignal==1.3.1 ; python_version >= '3.7'
 antlr4-python3-runtime==4.9.3
-anyio==3.6.1 ; python_full_version >= '3.6.2'
+anyio==3.6.2 ; python_full_version >= '3.6.2'
 apiosintds==1.8.3
 appdirs==1.4.4
+argcomplete==3.0.8 ; python_version >= '3.6'
 argparse==1.4.0
-assemblyline-client==4.5.0
+assemblyline-client==4.5.1
 async-timeout==4.0.2 ; python_version >= '3.6'
 asynctest==0.13.0 ; python_version < '3.8'
-attrs==22.1.0 ; python_version >= '3.5'
-backoff==2.1.2 ; python_version >= '3.7' and python_version < '4.0'
+attrs==23.1.0 ; python_version >= '3.7'
+backoff==2.2.1 ; python_version >= '3.7' and python_version < '4.0'
 backports.zoneinfo==0.2.1 ; python_version < '3.9'
 backscatter==0.2.4
-beautifulsoup4==4.11.1
-bidict==0.22.0 ; python_version >= '3.7'
+beautifulsoup4==4.11.2
+bidict==0.22.1 ; python_version >= '3.7'
 blockchain==1.4.4
-censys==2.1.8
-certifi==2022.9.24 ; python_version >= '3.6'
+censys==2.2.2
+certifi==2023.5.7 ; python_version >= '3.6'
 cffi==1.15.1
-chardet==5.0.0
-charset-normalizer==2.1.1 ; python_full_version >= '3.6.0'
+chardet==5.1.0
+charset-normalizer==3.1.0 ; python_full_version >= '3.7.0'
 clamd==1.0.2
 click==8.1.3 ; python_version >= '3.7'
 click-plugins==1.1.1
-colorama==0.4.5 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+colorama==0.4.6 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6'
 colorclass==2.2.2 ; python_version >= '2.6'
-commonmark==0.9.1
 compressed-rtf==1.0.6
 configparser==5.3.0 ; python_version >= '3.7'
-crowdstrike-falconpy==1.2.2
-cryptography==38.0.1 ; python_version >= '3.6'
-dateparser==1.1.1 ; python_version >= '3.5'
+crowdstrike-falconpy==1.2.15
+cryptography==40.0.2 ; python_version >= '3.6'
+dateparser==1.1.8 ; python_version >= '3.7'
 decorator==5.1.1 ; python_version >= '3.5'
 deprecated==1.2.13 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 dnsdb2==1.1.4
-dnspython==2.2.1
+dnspython==2.3.0
 domaintools-api==1.0.1
 easygui==0.98.3
 ebcdic==1.1.1
 enum-compat==0.0.3
 et-xmlfile==1.1.0 ; python_version >= '3.6'
-extract-msg==0.36.3
+extract-msg==0.38.4
 ezodf==0.3.2
-filelock==3.8.0 ; python_version >= '3.7'
-frozenlist==1.3.1 ; python_version >= '3.7'
-future==0.18.2 ; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
-geoip2==4.6.0
-h11==0.12.0 ; python_version >= '3.6'
-httpcore==0.15.0 ; python_version >= '3.7'
-httplib2==0.20.4 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-httpx==0.23.0 ; python_version >= '3.7'
+filelock==3.12.0 ; python_version >= '3.7'
+frozenlist==1.3.3 ; python_version >= '3.7'
+future==0.18.3 ; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
+geoip2==4.7.0
+h11==0.14.0 ; python_version >= '3.7'
+httpcore==0.17.1 ; python_version >= '3.7'
+httplib2==0.22.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+httpx==0.24.1 ; python_version >= '3.7'
 idna==3.4 ; python_version >= '3.5'
 imapclient==2.3.1
-importlib-metadata==4.12.0 ; python_version < '3.8'
-importlib-resources==5.9.0 ; python_version < '3.9'
+importlib-metadata==4.13.0 ; python_version < '3.8'
+importlib-resources==5.12.0 ; python_version < '3.9'
 isodate==0.6.1
 itsdangerous==2.1.2 ; python_version >= '3.7'
 jaraco.classes==3.2.3 ; python_version >= '3.7'
-jbxapi==3.18.0
+jbxapi==3.21.0
 jeepney==0.8.0 ; sys_platform == 'linux'
 jinja2==3.1.2
-json-log-formatter==0.5.1
-jsonschema==4.16.0 ; python_version >= '3.7'
-keyring==23.9.3 ; python_version >= '3.7'
+json-log-formatter==0.5.2 ; python_version >= '2.7'
+jsonschema==4.17.3 ; python_version >= '3.7'
+keyring==23.13.1 ; python_version >= '3.7'
 lark-parser==0.12.0
-lief==0.12.1
-lxml==4.9.1
+lief==0.12.3
+lxml==4.9.2
 maclookup==1.0.3
+markdown-it-py==2.2.0 ; python_version >= '3.7'
 markdownify==0.5.3
-markupsafe==2.1.1 ; python_version >= '3.7'
+markupsafe==2.1.2 ; python_version >= '3.7'
 mattermostdriver==7.3.2
-maxminddb==2.2.0 ; python_version >= '3.6'
+maxminddb==2.3.0 ; python_version >= '3.7'
+mdurl==0.1.2 ; python_version >= '3.7'
 .
-more-itertools==8.14.0 ; python_version >= '3.5'
-msoffcrypto-tool==5.0.0 ; python_version >= '3' and platform_python_implementation != 'PyPy' or (platform_system != 'Windows' and platform_system != 'Darwin')
-multidict==6.0.2 ; python_version >= '3.7'
-mwdblib==4.3.1
+more-itertools==9.1.0 ; python_version >= '3.7'
+msoffcrypto-tool==5.0.1 ; python_version >= '3' and platform_python_implementation != 'PyPy' or (platform_system != 'Windows' and platform_system != 'Darwin')
+multidict==6.0.4 ; python_version >= '3.7'
+mwdblib==4.4.0
 ndjson==0.3.1
 np==1.0.2
 numpy==1.21.6 ; python_version < '3.10' and platform_machine == 'aarch64'
@@ -84,98 +86,99 @@ oauth2==1.9.0.post1
 git+https://github.com/cartertemm/ODTReader.git/@49d6938693f6faa3ff09998f86dba551ae3a996b#egg=odtreader
 olefile==0.46 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 oletools==0.60.1
-opencv-python==4.6.0.66
-openpyxl==3.0.10
-packaging==21.3 ; python_version >= '3.6'
+opencv-python==4.7.0.72
+openpyxl==3.1.2
+packaging==23.1 ; python_version >= '3.7'
 pandas==1.3.5
 pandas-ods-reader==0.1.2
 passivetotal==2.5.9
 pcodedmp==1.2.6
 pdftotext==2.2.2
-pillow==9.2.0
+pillow==9.5.0
 pkgutil-resolve-name==1.3.10 ; python_version < '3.9'
-progressbar2==4.0.0 ; python_full_version >= '3.7.0'
-psutil==5.9.2 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-publicsuffixlist==0.8.0 ; python_version >= '2.6'
+progressbar2==4.2.0 ; python_full_version >= '3.7.0'
+psutil==5.9.5 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+publicsuffixlist==0.9.4 ; python_version >= '2.6'
 git+https://github.com/D4-project/BGP-Ranking.git/@68de39f6c5196f796055c1ac34504054d688aa59#egg=pybgpranking&subdirectory=client
+pycountry==22.3.5
 pycparser==2.21
-pycryptodome==3.15.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
-pycryptodomex==3.15.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+pycryptodome==3.18.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+pycryptodomex==3.17 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
 pydeep2==0.5.1
 git+https://github.com/sebdraven/pydnstrails@48c1f740025c51289f43a24863d1845ff12fd21a#egg=pydnstrails
 pyeupi==1.1
 pyfaup==1.2
 pygeoip==0.3.2
-pygments==2.13.0 ; python_version >= '3.6'
+pygments==2.15.1 ; python_version >= '3.7'
 git+https://github.com/MISP/PyIntel471.git@917272fafa8e12102329faca52173e90c5256968#egg=pyintel471
 git+https://github.com/D4-project/IPASN-History.git/@a2853c39265cecdd0c0d16850bd34621c0551b87#egg=pyipasnhistory&subdirectory=client
-pymisp[email,fileobjects,openioc,pdfexport,url]==2.4.162
+pymisp[email,fileobjects,openioc,pdfexport,url]==2.4.167
 git+https://github.com/sebdraven/pyonyphe@d1d6741f8ea4475f3bb77ff20c876f08839cabd1#egg=pyonyphe
 pyparsing==2.4.7 ; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
 pypdns==1.5.2
 pypssl==2.2
-pyrsistent==0.18.1 ; python_version >= '3.7'
+pyrsistent==0.19.3 ; python_version >= '3.7'
 pytesseract==0.3.10
 python-baseconv==1.2.2
 python-dateutil==2.8.2 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 python-docx==0.8.11
-python-engineio==4.3.4 ; python_version >= '3.6'
+python-engineio==4.4.1 ; python_version >= '3.6'
 python-magic==0.4.27
 python-pptx==0.6.21
-python-socketio[client]==5.7.1 ; python_version >= '3.6'
-python-utils==3.3.3 ; python_version >= '3.7'
+python-socketio[client]==5.8.0 ; python_version >= '3.6'
+python-utils==3.5.2 ; python_version >= '3.7'
 pytz==2019.3
 pytz-deprecation-shim==0.1.0.post0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
 pyyaml==6.0 ; python_version >= '3.6'
 pyzbar==0.1.9
 pyzipper==0.3.6 ; python_version >= '3.5'
-rdflib==6.2.0 ; python_version >= '3.7'
-redis==4.3.4 ; python_version >= '3.6'
-regex==2022.3.2 ; python_version >= '3.6'
-reportlab==3.6.11
-requests==2.28.1
+rdflib==6.3.2 ; python_version >= '3.7' and python_version < '4.0'
+red-black-tree-mod==1.20
+redis==4.5.5 ; python_version >= '3.7'
+regex==2023.5.5 ; python_version >= '3.6'
+reportlab==3.6.13
+requests[security]==2.30.0
 requests-cache==0.6.4 ; python_version >= '3.6'
 requests-file==1.5.1
-rfc3986[idna2008]==1.5.0
-rich==12.5.1 ; python_full_version >= '3.6.3' and python_full_version < '4.0.0'
+rich==13.3.5 ; python_full_version >= '3.7.0'
 rtfde==0.0.2
 secretstorage==3.3.3 ; sys_platform == 'linux'
-setuptools==65.4.0 ; python_version >= '3.7'
-shodan==1.28.0
+setuptools==67.7.2 ; python_version >= '3.7'
+shodan==1.29.1
 sigmatools==0.19.1
-simplejson==3.17.6 ; python_version >= '2.5' and python_version not in '3.0, 3.1, 3.2, 3.3'
+simplejson==3.19.1 ; python_version >= '2.5' and python_version not in '3.0, 3.1, 3.2, 3.3'
 six==1.16.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 sniffio==1.3.0 ; python_version >= '3.7'
-socialscan==1.4.2
+socialscan==1.4
 socketio-client==0.5.7.4
-soupsieve==2.3.2.post1 ; python_version >= '3.6'
+soupsieve==2.4.1 ; python_version >= '3.7'
 sparqlwrapper==2.0.0
 stix2==3.0.1
 stix2-patterns==2.0.0
-tabulate==0.8.10 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
-tau-clients==0.2.9
+tabulate==0.9.0 ; python_version >= '3.7'
+tau-clients==0.3.0
 taxii2-client==2.3.0
-tldextract==3.3.1 ; python_version >= '3.7'
+tldextract==3.4.3 ; python_version >= '3.7'
 tornado==6.2 ; python_version >= '3.7'
-tqdm==4.64.1 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+tqdm==4.65.0 ; python_version >= '3.7'
 git+https://github.com/SteveClement/trustar-python.git@6954eae38e0c77eaeef26084b6c5fd033925c1c7#egg=trustar
-typing-extensions==4.3.0 ; python_version < '3.8'
-tzdata==2022.4 ; python_version >= '3.6'
+typing-extensions==4.5.0 ; python_version < '3.8'
+tzdata==2023.3 ; python_version >= '3.6'
 tzlocal==4.2 ; python_version >= '3.6'
 unicodecsv==0.14.1
 url-normalize==1.4.3 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
 urlarchiver==0.2
-urllib3==1.26.12 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_version < '4'
+urllib3==1.26.15 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
 validators==0.14.0
 vt-graph-api==2.2.0
-vt-py==0.17.1
-vulners==2.0.4
-wand==0.6.10
-websocket-client==1.4.1 ; python_version >= '3.7'
-websockets==10.3 ; python_version >= '3.7'
-wrapt==1.14.1 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+vt-py==0.17.5
+vulners==2.0.10
+wand==0.6.11
+websocket-client==1.5.1 ; python_version >= '3.7'
+websockets==11.0.3 ; python_version >= '3.7'
+wrapt==1.15.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
 xlrd==2.0.1
-xlsxwriter==3.0.3 ; python_version >= '3.4'
+xlsxwriter==3.1.0 ; python_version >= '3.6'
 yara-python==3.8.1
-yarl==1.8.1 ; python_version >= '3.7'
-zipp==3.8.1 ; python_version >= '3.7'
+yarl==1.9.2 ; python_version >= '3.7'
+zipp==3.15.0 ; python_version >= '3.7'

docs/index.md

@@ -2,7 +2,7 @@
 
 [![Build Status](https://travis-ci.org/MISP/misp-modules.svg?branch=master)](https://travis-ci.org/MISP/misp-modules)
 [![Coverage Status](https://coveralls.io/repos/github/MISP/misp-modules/badge.svg?branch=master)](https://coveralls.io/github/MISP/misp-modules?branch=master)
-[![codecov](https://codecov.io/gh/MISP/misp-modules/branch/master/graph/badge.svg)](https://codecov.io/gh/MISP/misp-modules)
+[![codecov](https://codecov.io/gh/MISP/misp-modules/branch/main/graph/badge.svg)](https://codecov.io/gh/MISP/misp-modules)
 [![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgithub.com%MISP%2Fmisp-modules.svg?type=shield)](https://app.fossa.io/projects/git%2Bgithub.com%2FMISP%2Fmisp-modules?ref=badge_shield)
 
 MISP modules are autonomous modules that can be used for expansion and other services in [MISP](https://github.com/MISP/MISP).
@@ -19,93 +19,93 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/
 
 ### Expansion modules
 
-* [Backscatter.io](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/backscatter_io.py) - a hover and expansion module to expand an IP address with mass-scanning observations.
-* [BGP Ranking](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/bgpranking.py) - a hover and expansion module to expand an AS number with the ASN description, its history, and position in BGP Ranking.
-* [BTC scam check](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/btc_scam_check.py) - An expansion hover module to instantly check if a BTC address has been abused.
-* [BTC transactions](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/btc_steroids.py) - An expansion hover module to get a blockchain balance and the transactions from a BTC address in MISP.
-* [CIRCL Passive DNS](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/circl_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information.
-* [CIRCL Passive SSL](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/circl_passivessl.py) - a hover and expansion module to expand IP addresses with the X.509 certificate seen.
-* [countrycode](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/countrycode.py) - a hover module to tell you what country a URL belongs to.
-* [CrowdStrike Falcon](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/crowdstrike_falcon.py) - an expansion module to expand using CrowdStrike Falcon Intel Indicator API.
-* [CVE](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/cve.py) - a hover module to give more information about a vulnerability (CVE).
-* [CVE advanced](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/cve_advanced.py) - An expansion module to query the CIRCL CVE search API for more information about a vulnerability (CVE).
-* [Cuckoo submit](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/cuckoo_submit.py) - A hover module to submit malware sample, url, attachment, domain to Cuckoo Sandbox.
-* [DBL Spamhaus](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/dbl_spamhaus.py) - a hover module to check Spamhaus DBL for a domain name.
-* [DNS](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/dns.py) - a simple module to resolve MISP attributes like hostname and domain to expand IP addresses attributes.
-* [docx-enrich](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/docx-enrich.py) - an enrichment module to get text out of Word document into MISP (using free-text parser).
-* [DomainTools](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/domaintools.py) - a hover and expansion module to get information from [DomainTools](http://www.domaintools.com/) whois.
-* [EUPI](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/eupi.py) - a hover and expansion module to get information about an URL from the [Phishing Initiative project](https://phishing-initiative.eu/?lang=en).
+* [Backscatter.io](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/backscatter_io.py) - a hover and expansion module to expand an IP address with mass-scanning observations.
+* [BGP Ranking](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/bgpranking.py) - a hover and expansion module to expand an AS number with the ASN description, its history, and position in BGP Ranking.
+* [BTC scam check](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/btc_scam_check.py) - An expansion hover module to instantly check if a BTC address has been abused.
+* [BTC transactions](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/btc_steroids.py) - An expansion hover module to get a blockchain balance and the transactions from a BTC address in MISP.
+* [CIRCL Passive DNS](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/circl_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information.
+* [CIRCL Passive SSL](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/circl_passivessl.py) - a hover and expansion module to expand IP addresses with the X.509 certificate seen.
+* [countrycode](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/countrycode.py) - a hover module to tell you what country a URL belongs to.
+* [CrowdStrike Falcon](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/crowdstrike_falcon.py) - an expansion module to expand using CrowdStrike Falcon Intel Indicator API.
+* [CVE](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/cve.py) - a hover module to give more information about a vulnerability (CVE).
+* [CVE advanced](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/cve_advanced.py) - An expansion module to query the CIRCL CVE search API for more information about a vulnerability (CVE).
+* [Cuckoo submit](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/cuckoo_submit.py) - A hover module to submit malware sample, url, attachment, domain to Cuckoo Sandbox.
+* [DBL Spamhaus](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/dbl_spamhaus.py) - a hover module to check Spamhaus DBL for a domain name.
+* [DNS](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/dns.py) - a simple module to resolve MISP attributes like hostname and domain to expand IP addresses attributes.
+* [docx-enrich](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/docx-enrich.py) - an enrichment module to get text out of Word document into MISP (using free-text parser).
+* [DomainTools](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/domaintools.py) - a hover and expansion module to get information from [DomainTools](http://www.domaintools.com/) whois.
+* [EUPI](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/eupi.py) - a hover and expansion module to get information about an URL from the [Phishing Initiative project](https://phishing-initiative.eu/?lang=en).
 * [EQL](misp_modules/modules/expansion/eql.py) - an expansion module to generate event query language (EQL) from an attribute. [Event Query Language](https://eql.readthedocs.io/en/latest/)
-* [Farsight DNSDB Passive DNS](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/farsight_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information.
-* [GeoIP](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/geoip_country.py) - a hover and expansion module to get GeoIP information from geolite/maxmind.
-* [Greynoise](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/greynoise.py) - a hover to get information from greynoise.
-* [hashdd](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/hashdd.py) - a hover module to check file hashes against [hashdd.com](http://www.hashdd.com) including NSLR dataset.
-* [hibp](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/hibp.py) - a hover module to lookup against Have I Been Pwned?
-* [intel471](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/intel471.py) - an expansion module to get info from [Intel471](https://intel471.com).
-* [IPASN](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/ipasn.py) - a hover and expansion to get the BGP ASN of an IP address.
-* [iprep](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/iprep.py) - an expansion module to get IP reputation from packetmail.net.
-* [Joe Sandbox submit](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/joesandbox_submit.py) - Submit files and URLs to Joe Sandbox.
-* [Joe Sandbox query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/joesandbox_query.py) - Query Joe Sandbox with the link of an analysis and get the parsed data.
-* [macaddress.io](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/macaddress_io.py) - a hover module to retrieve vendor details and other information regarding a given MAC address or an OUI from [MAC address Vendor Lookup](https://macaddress.io). See [integration tutorial here](https://macaddress.io/integrations/MISP-module).
-* [macvendors](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/macvendors.py) - a hover module to retrieve mac vendor information.
-* [ocr-enrich](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/ocr-enrich.py) - an enrichment module to get OCRized data from images into MISP.
-* [ods-enrich](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/ods-enrich.py) - an enrichment module to get text out of OpenOffice spreadsheet document into MISP (using free-text parser).
-* [odt-enrich](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/odt-enrich.py) - an enrichment module to get text out of OpenOffice document into MISP (using free-text parser).
-* [onyphe](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/onyphe.py) - a modules to process queries on Onyphe.
-* [onyphe_full](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/onyphe_full.py) - a modules to process full queries on Onyphe.
-* [OTX](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/otx.py) - an expansion module for [OTX](https://otx.alienvault.com/).
-* [passivetotal](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/passivetotal.py) - a [passivetotal](https://www.passivetotal.org/) module that queries a number of different PassiveTotal datasets.
-* [pdf-enrich](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/pdf-enrich.py) - an enrichment module to extract text from PDF into MISP (using free-text parser).
-* [pptx-enrich](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/pptx-enrich.py) - an enrichment module to get text out of PowerPoint document into MISP (using free-text parser).
-* [qrcode](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/qrcode.py) - a module decode QR code, barcode and similar codes from an image and enrich with the decoded values.
-* [rbl](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/rbl.py) - a module to get RBL (Real-Time Blackhost List) values from an attribute.
-* [reversedns](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/reversedns.py) - Simple Reverse DNS expansion service to resolve reverse DNS from MISP attributes.
-* [securitytrails](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/securitytrails.py) - an expansion module for [securitytrails](https://securitytrails.com/).
-* [shodan](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/shodan.py) - a minimal [shodan](https://www.shodan.io/) expansion module.
-* [Sigma queries](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/sigma_queries.py) - Experimental expansion module querying a sigma rule to convert it into all the available SIEM signatures.
-* [Sigma syntax validator](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/sigma_syntax_validator.py) - Sigma syntax validator.
-* [sourcecache](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/sourcecache.py) - a module to cache a specific link from a MISP instance.
-* [STIX2 pattern syntax validator](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/stix2_pattern_syntax_validator.py) - a module to check a STIX2 pattern syntax.
-* [ThreatCrowd](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/threatcrowd.py) - an expansion module for [ThreatCrowd](https://www.threatcrowd.org/).
-* [threatminer](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/threatminer.py) - an expansion module to expand from [ThreatMiner](https://www.threatminer.org/).
-* [urlhaus](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/urlhaus.py) - Query urlhaus to get additional data about a domain, hash, hostname, ip or url.
-* [urlscan](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/urlscan.py) - an expansion module to query [urlscan.io](https://urlscan.io).
-* [virustotal](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/virustotal.py) - an expansion module to query the [VirusTotal](https://www.virustotal.com/gui/home) API with a high request rate limit required. (More details about the API: [here](https://developers.virustotal.com/reference))
-* [virustotal_public](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/virustotal_public.py) - an expansion module to query the [VirusTotal](https://www.virustotal.com/gui/home) API with a public key and a low request rate limit. (More details about the API: [here](https://developers.virustotal.com/reference))
-* [VMray](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/vmray_submit.py) - a module to submit a sample to VMray.
-* [VulnDB](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/vulndb.py) - a module to query [VulnDB](https://www.riskbasedsecurity.com/).
-* [Vulners](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/vulners.py) - an expansion module to expand information about CVEs using Vulners API.
-* [whois](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/whois.py) - a module to query a local instance of [uwhois](https://github.com/rafiot/uwhoisd).
-* [wikidata](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/wiki.py) - a [wikidata](https://www.wikidata.org) expansion module.
-* [xforce](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/xforceexchange.py) - an IBM X-Force Exchange expansion module.
-* [xlsx-enrich](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/xlsx-enrich.py) - an enrichment module to get text out of an Excel document into MISP (using free-text parser).
-* [YARA query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/yara_query.py) - a module to create YARA rules from single hash attributes.
-* [YARA syntax validator](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/yara_syntax_validator.py) - YARA syntax validator.
+* [Farsight DNSDB Passive DNS](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/farsight_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information.
+* [GeoIP](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/geoip_country.py) - a hover and expansion module to get GeoIP information from geolite/maxmind.
+* [Greynoise](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/greynoise.py) - a hover to get information from greynoise.
+* [hashdd](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/hashdd.py) - a hover module to check file hashes against [hashdd.com](http://www.hashdd.com) including NSLR dataset.
+* [hibp](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/hibp.py) - a hover module to lookup against Have I Been Pwned?
+* [intel471](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/intel471.py) - an expansion module to get info from [Intel471](https://intel471.com).
+* [IPASN](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/ipasn.py) - a hover and expansion to get the BGP ASN of an IP address.
+* [iprep](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/iprep.py) - an expansion module to get IP reputation from packetmail.net.
+* [Joe Sandbox submit](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/joesandbox_submit.py) - Submit files and URLs to Joe Sandbox.
+* [Joe Sandbox query](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/joesandbox_query.py) - Query Joe Sandbox with the link of an analysis and get the parsed data.
+* [macaddress.io](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/macaddress_io.py) - a hover module to retrieve vendor details and other information regarding a given MAC address or an OUI from [MAC address Vendor Lookup](https://macaddress.io). See [integration tutorial here](https://macaddress.io/integrations/MISP-module).
+* [macvendors](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/macvendors.py) - a hover module to retrieve mac vendor information.
+* [ocr-enrich](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/ocr-enrich.py) - an enrichment module to get OCRized data from images into MISP.
+* [ods-enrich](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/ods-enrich.py) - an enrichment module to get text out of OpenOffice spreadsheet document into MISP (using free-text parser).
+* [odt-enrich](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/odt-enrich.py) - an enrichment module to get text out of OpenOffice document into MISP (using free-text parser).
+* [onyphe](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/onyphe.py) - a modules to process queries on Onyphe.
+* [onyphe_full](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/onyphe_full.py) - a modules to process full queries on Onyphe.
+* [OTX](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/otx.py) - an expansion module for [OTX](https://otx.alienvault.com/).
+* [passivetotal](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/passivetotal.py) - a [passivetotal](https://www.passivetotal.org/) module that queries a number of different PassiveTotal datasets.
+* [pdf-enrich](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/pdf-enrich.py) - an enrichment module to extract text from PDF into MISP (using free-text parser).
+* [pptx-enrich](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/pptx-enrich.py) - an enrichment module to get text out of PowerPoint document into MISP (using free-text parser).
+* [qrcode](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/qrcode.py) - a module decode QR code, barcode and similar codes from an image and enrich with the decoded values.
+* [rbl](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/rbl.py) - a module to get RBL (Real-Time Blackhost List) values from an attribute.
+* [reversedns](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/reversedns.py) - Simple Reverse DNS expansion service to resolve reverse DNS from MISP attributes.
+* [securitytrails](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/securitytrails.py) - an expansion module for [securitytrails](https://securitytrails.com/).
+* [shodan](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/shodan.py) - a minimal [shodan](https://www.shodan.io/) expansion module.
+* [Sigma queries](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/sigma_queries.py) - Experimental expansion module querying a sigma rule to convert it into all the available SIEM signatures.
+* [Sigma syntax validator](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/sigma_syntax_validator.py) - Sigma syntax validator.
+* [sourcecache](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/sourcecache.py) - a module to cache a specific link from a MISP instance.
+* [STIX2 pattern syntax validator](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/stix2_pattern_syntax_validator.py) - a module to check a STIX2 pattern syntax.
+* [ThreatCrowd](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/threatcrowd.py) - an expansion module for [ThreatCrowd](https://www.threatcrowd.org/).
+* [threatminer](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/threatminer.py) - an expansion module to expand from [ThreatMiner](https://www.threatminer.org/).
+* [urlhaus](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/urlhaus.py) - Query urlhaus to get additional data about a domain, hash, hostname, ip or url.
+* [urlscan](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/urlscan.py) - an expansion module to query [urlscan.io](https://urlscan.io).
+* [virustotal](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/virustotal.py) - an expansion module to query the [VirusTotal](https://www.virustotal.com/gui/home) API with a high request rate limit required. (More details about the API: [here](https://developers.virustotal.com/reference))
+* [virustotal_public](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/virustotal_public.py) - an expansion module to query the [VirusTotal](https://www.virustotal.com/gui/home) API with a public key and a low request rate limit. (More details about the API: [here](https://developers.virustotal.com/reference))
+* [VMray](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/vmray_submit.py) - a module to submit a sample to VMray.
+* [VulnDB](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/vulndb.py) - a module to query [VulnDB](https://www.riskbasedsecurity.com/).
+* [Vulners](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/vulners.py) - an expansion module to expand information about CVEs using Vulners API.
+* [whois](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/whois.py) - a module to query a local instance of [uwhois](https://github.com/rafiot/uwhoisd).
+* [wikidata](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/wiki.py) - a [wikidata](https://www.wikidata.org) expansion module.
+* [xforce](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/xforceexchange.py) - an IBM X-Force Exchange expansion module.
+* [xlsx-enrich](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/xlsx-enrich.py) - an enrichment module to get text out of an Excel document into MISP (using free-text parser).
+* [YARA query](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/yara_query.py) - a module to create YARA rules from single hash attributes.
+* [YARA syntax validator](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/yara_syntax_validator.py) - YARA syntax validator.
 
 ### Export modules
 
-* [CEF](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/cef_export.py) module to export Common Event Format (CEF).
-* [Cisco FireSight Manager ACL rule](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/cisco_firesight_manager_ACL_rule_export.py) module to export as rule for the Cisco FireSight manager ACL.
-* [GoAML export](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/goamlexport.py) module to export in [GoAML format](http://goaml.unodc.org/goaml/en/index.html).
-* [Lite Export](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/liteexport.py) module to export a lite event.
+* [CEF](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/export_mod/cef_export.py) module to export Common Event Format (CEF).
+* [Cisco FireSight Manager ACL rule](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/export_mod/cisco_firesight_manager_ACL_rule_export.py) module to export as rule for the Cisco FireSight manager ACL.
+* [GoAML export](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/export_mod/goamlexport.py) module to export in [GoAML format](http://goaml.unodc.org/goaml/en/index.html).
+* [Lite Export](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/export_mod/liteexport.py) module to export a lite event.
 * [Mass EQL Export](misp_modules/modules/export_mod/mass_eql_export.py) module to export applicable attributes from an event to a mass EQL query.
-* [PDF export](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/pdfexport.py) module to export an event in PDF.
-* [Nexthink query format](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/nexthinkexport.py) module to export in Nexthink query format.
-* [osquery](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/osqueryexport.py) module to export in [osquery](https://osquery.io/) query format.
-* [ThreatConnect](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/threat_connect_export.py) module to export in ThreatConnect CSV format.
-* [ThreatStream](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/threatStream_misp_export.py) module to export in ThreatStream format.
+* [PDF export](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/export_mod/pdfexport.py) module to export an event in PDF.
+* [Nexthink query format](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/export_mod/nexthinkexport.py) module to export in Nexthink query format.
+* [osquery](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/export_mod/osqueryexport.py) module to export in [osquery](https://osquery.io/) query format.
+* [ThreatConnect](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/export_mod/threat_connect_export.py) module to export in ThreatConnect CSV format.
+* [ThreatStream](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/export_mod/threatStream_misp_export.py) module to export in ThreatStream format.
 
 ### Import modules
 
-* [CSV import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/csvimport.py) Customizable CSV import module.
-* [Cuckoo JSON](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/cuckooimport.py) Cuckoo JSON import.
-* [Email Import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/email_import.py) Email import module for MISP to import basic metadata.
-* [GoAML import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/goamlimport.py) Module to import [GoAML](http://goaml.unodc.org/goaml/en/index.html) XML format.
-* [Joe Sandbox import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/joe_import.py) Parse data from a Joe Sandbox json report.
-* [OCR](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/ocr.py) Optical Character Recognition (OCR) module for MISP to import attributes from images, scan or faxes.
-* [OpenIOC](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/openiocimport.py) OpenIOC import based on PyMISP library.
-* [ThreatAnalyzer](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/threatanalyzer_import.py) - An import module to process ThreatAnalyzer archive.zip/analysis.json sandbox exports.
-* [VMRay](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/vmray_import.py) - An import module to process VMRay export.
+* [CSV import](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/import_mod/csvimport.py) Customizable CSV import module.
+* [Cuckoo JSON](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/import_mod/cuckooimport.py) Cuckoo JSON import.
+* [Email Import](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/import_mod/email_import.py) Email import module for MISP to import basic metadata.
+* [GoAML import](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/import_mod/goamlimport.py) Module to import [GoAML](http://goaml.unodc.org/goaml/en/index.html) XML format.
+* [Joe Sandbox import](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/import_mod/joe_import.py) Parse data from a Joe Sandbox json report.
+* [OCR](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/import_mod/ocr.py) Optical Character Recognition (OCR) module for MISP to import attributes from images, scan or faxes.
+* [OpenIOC](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/import_mod/openiocimport.py) OpenIOC import based on PyMISP library.
+* [ThreatAnalyzer](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/import_mod/threatanalyzer_import.py) - An import module to process ThreatAnalyzer archive.zip/analysis.json sandbox exports.
+* [VMRay](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/import_mod/vmray_import.py) - An import module to process VMRay export.
 
 
 ## How to contribute your own module?
@@ -117,4 +117,4 @@ For further information please see [Contribute](contribute/).
 ## Licenses
 [![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgithub.com%MISP%2Fmisp-modules.svg?type=large)](https://app.fossa.io/projects/git%2Bgithub.com%2FMISP%2Fmisp-modules?ref=badge_large)
 
-For further Information see also the [license file](license/).
+For further Information see also the [license file](license/).

documentation/README.md

@@ -242,6 +242,26 @@ An expansion module to query the CVE search API with a cpe code to get its relat
 
 -----
 
+#### [crowdsec](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/crowdsec.py)
+
+<img src=logos/crowdsec.png height=60>
+
+Hover module to lookup an IP in CrowdSec's CTI
+- **features**:
+>This module enables IP lookup from CrowdSec CTI API. It provides information about the IP, such as what kind of attacks it has been participant of as seen by CrowdSec's network. It also includes enrichment by CrowdSec like background noise score, aggressivity over time etc.
+- **input**:
+>An IP address.
+- **output**:
+>IP Lookup information from CrowdSec CTI API
+- **references**:
+> - https://www.crowdsec.net/
+> - https://docs.crowdsec.net/docs/cti_api/getting_started
+> - https://app.crowdsec.net/
+- **requirements**:
+>A CrowdSec CTI API key. Get yours by following https://docs.crowdsec.net/docs/cti_api/getting_started/#getting-an-api-key
+
+-----
+
 #### [crowdstrike_falcon](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/crowdstrike_falcon.py)
 
 <img src=logos/crowdstrike.png height=60>
@@ -776,6 +796,31 @@ Module to query an IP ASN history service (https://github.com/D4-project/IPASN-H
 
 -----
 
+#### [ipinfo](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/ipinfo.py)
+
+<img src=logos/ipinfo.png height=60>
+
+An expansion module to query ipinfo.io to gather more information on a given IP address.
+- **features**:
+>The module takes an IP address attribute as input and queries the ipinfo.io API.  
+>The geolocation information on the IP address is always returned.
+>
+>Depending on the subscription plan, the API returns different pieces of information then:
+>- With a basic plan (free) you get the AS number and the AS organisation name concatenated in the `org` field.
+>- With a paid subscription, the AS information is returned in the `asn` field with additional AS information, and depending on which plan the user has, you can also get information on the privacy method used to protect the IP address, the related domains, or the point of contact related to the IP address in case of an abuse.
+>
+>More information on the responses content is available in the [documentation](https://ipinfo.io/developers).
+- **input**:
+>IP address attribute.
+- **output**:
+>Additional information on the IP address, like its geolocation, the autonomous system it is included in, and the related domain(s).
+- **references**:
+>https://ipinfo.io/developers
+- **requirements**:
+>An ipinfo.io token
+
+-----
+
 #### [ipqs_fraud_and_risk_scoring](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/ipqs_fraud_and_risk_scoring.py)
 
 <img src=logos/ipqualityscore.png height=60>
@@ -818,11 +863,11 @@ Module to query IPRep data for IP addresses.
 
 Query Joe Sandbox API with a submission url to get the json report and extract its data that is parsed and converted into MISP attributes and objects.
 
-This url can by the way come from the result of the [joesandbox_submit expansion module](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/joesandbox_submit.py).
+This url can by the way come from the result of the [joesandbox_submit expansion module](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/joesandbox_submit.py).
 - **features**:
 >Module using the new format of modules able to return attributes and objects.
 >
->The module returns the same results as the import module [joe_import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/joe_import.py) taking directly the json report as input.
+>The module returns the same results as the import module [joe_import](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/import_mod/joe_import.py) taking directly the json report as input.
 >
 >Even if the introspection will allow all kinds of links to call this module, obviously only the ones presenting a sample or url submission in the Joe Sandbox API will return results.
 >
@@ -847,7 +892,7 @@ A module to submit files or URLs to Joe Sandbox for an advanced analysis, and re
 - **features**:
 >The module requires a Joe Sandbox API key to submit files or URL, and returns the link of the submitted analysis.
 >
->It is then possible, when the analysis is completed, to query the Joe Sandbox API to get the data related to the analysis, using the [joesandbox_query module](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/joesandbox_query.py) directly on this submission link.
+>It is then possible, when the analysis is completed, to query the Joe Sandbox API to get the data related to the analysis, using the [joesandbox_query module](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/joesandbox_query.py) directly on this submission link.
 - **input**:
 >Sample, url (or domain) to submit to Joe Sandbox for an advanced analysis.
 - **output**:
@@ -867,11 +912,11 @@ A module to submit files or URLs to Joe Sandbox for an advanced analysis, and re
 Deprecation notice: this module will be deprecated by December 2021, please use vmware_nsx module.
 
 Query Lastline with an analysis link and parse the report into MISP attributes and objects.
-The analysis link can also be retrieved from the output of the [lastline_submit](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_submit.py) expansion module.
+The analysis link can also be retrieved from the output of the [lastline_submit](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/lastline_submit.py) expansion module.
 - **features**:
 >The module requires a Lastline Portal `username` and `password`.
 >The module uses the new format and it is able to return MISP attributes and objects.
->The module returns the same results as the [lastline_import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/lastline_import.py) import module.
+>The module returns the same results as the [lastline_import](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/import_mod/lastline_import.py) import module.
 - **input**:
 >Link to a Lastline analysis.
 - **output**:
@@ -890,7 +935,7 @@ Deprecation notice: this module will be deprecated by December 2021, please use
 Module to submit a file or URL to Lastline.
 - **features**:
 >The module requires a Lastline Analysis `api_token` and `key`.
->When the analysis is completed, it is possible to import the generated report by feeding the analysis link to the [lastline_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_query.py) module.
+>When the analysis is completed, it is possible to import the generated report by feeding the analysis link to the [lastline_query](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/lastline_query.py) module.
 - **input**:
 >File or URL to submit to Lastline.
 - **output**:
@@ -1660,7 +1705,7 @@ Module to get advanced information from virustotal.
 >
 >A module to take a MISP attribute as input and query the VirusTotal API to get additional data about it.
 >
->Compared to the [standard VirusTotal expansion module](https://github.com/MISP/misp-modules/blob/master/misp_modules/modules/expansion/virustotal_public.py), this module is made for advanced parsing of VirusTotal report, with a recursive analysis of the elements found after the first request.
+>Compared to the [standard VirusTotal expansion module](https://github.com/MISP/misp-modules/blob/main/misp_modules/modules/expansion/virustotal_public.py), this module is made for advanced parsing of VirusTotal report, with a recursive analysis of the elements found after the first request.
 >
 >Thus, it requires a higher request rate limit to avoid the API to return a 204 error (Request rate limit exceeded), and the data parsed from the different requests are returned as MISP attributes and objects, with the corresponding relations between each one of them.
 - **input**:
@@ -1685,7 +1730,7 @@ Module to get information from VirusTotal.
 >
 >A module to take a MISP attribute as input and query the VirusTotal API to get additional data about it.
 >
->Compared to the [more advanced VirusTotal expansion module](https://github.com/MISP/misp-modules/blob/master/misp_modules/modules/expansion/virustotal.py), this module is made for VirusTotal users who have a low request rate limit.
+>Compared to the [more advanced VirusTotal expansion module](https://github.com/MISP/misp-modules/blob/main/misp_modules/modules/expansion/virustotal.py), this module is made for VirusTotal users who have a low request rate limit.
 >
 >Thus, it only queries the API once and returns the results that is parsed into MISP attributes and objects.
 - **input**:
@@ -2262,7 +2307,7 @@ A module to import data from a Joe Sandbox analysis json report.
 - **features**:
 >Module using the new format of modules able to return attributes and objects.
 >
->The module returns the same results as the expansion module [joesandbox_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/joesandbox_query.py) using the submission link of the analysis to get the json report.
+>The module returns the same results as the expansion module [joesandbox_query](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/joesandbox_query.py) using the submission link of the analysis to get the json report.
 - **input**:
 >Json report of a Joe Sandbox analysis.
 - **output**:
@@ -2283,7 +2328,7 @@ Module to import and parse reports from Lastline analysis links.
 - **features**:
 >The module requires a Lastline Portal `username` and `password`.
 >The module uses the new format and it is able to return MISP attributes and objects.
->The module returns the same results as the [lastline_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_query.py) expansion module.
+>The module returns the same results as the [lastline_query](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/lastline_query.py) expansion module.
 - **input**:
 >Link to a Lastline analysis.
 - **output**:

documentation/generate_documentation.py

@@ -2,6 +2,7 @@
 import os
 import json
 import sys
+from pathlib import Path
 
 module_types = ['expansion', 'export_mod', 'import_mod']
 titles = ['Expansion Modules', 'Export Modules', 'Import Modules']
@@ -53,7 +54,7 @@ def write_doc(root_path):
     for _path, title in zip(module_types, titles):
         markdown.append(f'\n## {title}\n')
         markdown.extend(generate_doc(_path, root_path))
-    with open('README.md', 'w') as w:
+    with open(root_path / 'README.md', 'w') as w:
         w.write(''.join(markdown))
 
 
@@ -65,6 +66,6 @@ def write_docs_for_mkdocs(root_path):
 
 
 if __name__ == '__main__':
-    root_path = os.path.dirname(os.path.realpath(__file__))
+    root_path = Path(__file__).resolve().parent
     write_doc(root_path)
     write_docs_for_mkdocs(root_path)

documentation/mkdocs/expansion.md

@@ -239,6 +239,26 @@ An expansion module to query the CVE search API with a cpe code to get its relat
 
 -----
 
+#### [crowdsec](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/crowdsec.py)
+
+<img src=../logos/crowdsec.png height=60>
+
+Hover module to lookup an IP in CrowdSec's CTI
+- **features**:
+>This module enables IP lookup from CrowdSec CTI API. It provides information about the IP, such as what kind of attacks it has been participant of as seen by CrowdSec's network. It also includes enrichment by CrowdSec like background noise score, aggressivity over time etc.
+- **input**:
+>An IP address.
+- **output**:
+>IP Lookup information from CrowdSec CTI API
+- **references**:
+> - https://www.crowdsec.net/
+> - https://docs.crowdsec.net/docs/cti_api/getting_started
+> - https://app.crowdsec.net/
+- **requirements**:
+>A CrowdSec CTI API key. Get yours by following https://docs.crowdsec.net/docs/cti_api/getting_started/#getting-an-api-key
+
+-----
+
 #### [crowdstrike_falcon](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/crowdstrike_falcon.py)
 
 <img src=../logos/crowdstrike.png height=60>
@@ -773,6 +793,31 @@ Module to query an IP ASN history service (https://github.com/D4-project/IPASN-H
 
 -----
 
+#### [ipinfo](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/ipinfo.py)
+
+<img src=../logos/ipinfo.png height=60>
+
+An expansion module to query ipinfo.io to gather more information on a given IP address.
+- **features**:
+>The module takes an IP address attribute as input and queries the ipinfo.io API.  
+>The geolocation information on the IP address is always returned.
+>
+>Depending on the subscription plan, the API returns different pieces of information then:
+>- With a basic plan (free) you get the AS number and the AS organisation name concatenated in the `org` field.
+>- With a paid subscription, the AS information is returned in the `asn` field with additional AS information, and depending on which plan the user has, you can also get information on the privacy method used to protect the IP address, the related domains, or the point of contact related to the IP address in case of an abuse.
+>
+>More information on the responses content is available in the [documentation](https://ipinfo.io/developers).
+- **input**:
+>IP address attribute.
+- **output**:
+>Additional information on the IP address, like its geolocation, the autonomous system it is included in, and the related domain(s).
+- **references**:
+>https://ipinfo.io/developers
+- **requirements**:
+>An ipinfo.io token
+
+-----
+
 #### [ipqs_fraud_and_risk_scoring](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/ipqs_fraud_and_risk_scoring.py)
 
 <img src=../logos/ipqualityscore.png height=60>
@@ -815,11 +860,11 @@ Module to query IPRep data for IP addresses.
 
 Query Joe Sandbox API with a submission url to get the json report and extract its data that is parsed and converted into MISP attributes and objects.
 
-This url can by the way come from the result of the [joesandbox_submit expansion module](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/joesandbox_submit.py).
+This url can by the way come from the result of the [joesandbox_submit expansion module](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/joesandbox_submit.py).
 - **features**:
 >Module using the new format of modules able to return attributes and objects.
 >
->The module returns the same results as the import module [joe_import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/joe_import.py) taking directly the json report as input.
+>The module returns the same results as the import module [joe_import](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/import_mod/joe_import.py) taking directly the json report as input.
 >
 >Even if the introspection will allow all kinds of links to call this module, obviously only the ones presenting a sample or url submission in the Joe Sandbox API will return results.
 >
@@ -844,7 +889,7 @@ A module to submit files or URLs to Joe Sandbox for an advanced analysis, and re
 - **features**:
 >The module requires a Joe Sandbox API key to submit files or URL, and returns the link of the submitted analysis.
 >
->It is then possible, when the analysis is completed, to query the Joe Sandbox API to get the data related to the analysis, using the [joesandbox_query module](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/joesandbox_query.py) directly on this submission link.
+>It is then possible, when the analysis is completed, to query the Joe Sandbox API to get the data related to the analysis, using the [joesandbox_query module](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/joesandbox_query.py) directly on this submission link.
 - **input**:
 >Sample, url (or domain) to submit to Joe Sandbox for an advanced analysis.
 - **output**:
@@ -864,11 +909,11 @@ A module to submit files or URLs to Joe Sandbox for an advanced analysis, and re
 Deprecation notice: this module will be deprecated by December 2021, please use vmware_nsx module.
 
 Query Lastline with an analysis link and parse the report into MISP attributes and objects.
-The analysis link can also be retrieved from the output of the [lastline_submit](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_submit.py) expansion module.
+The analysis link can also be retrieved from the output of the [lastline_submit](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/lastline_submit.py) expansion module.
 - **features**:
 >The module requires a Lastline Portal `username` and `password`.
 >The module uses the new format and it is able to return MISP attributes and objects.
->The module returns the same results as the [lastline_import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/lastline_import.py) import module.
+>The module returns the same results as the [lastline_import](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/import_mod/lastline_import.py) import module.
 - **input**:
 >Link to a Lastline analysis.
 - **output**:
@@ -887,7 +932,7 @@ Deprecation notice: this module will be deprecated by December 2021, please use
 Module to submit a file or URL to Lastline.
 - **features**:
 >The module requires a Lastline Analysis `api_token` and `key`.
->When the analysis is completed, it is possible to import the generated report by feeding the analysis link to the [lastline_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_query.py) module.
+>When the analysis is completed, it is possible to import the generated report by feeding the analysis link to the [lastline_query](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/lastline_query.py) module.
 - **input**:
 >File or URL to submit to Lastline.
 - **output**:
@@ -1657,7 +1702,7 @@ Module to get advanced information from virustotal.
 >
 >A module to take a MISP attribute as input and query the VirusTotal API to get additional data about it.
 >
->Compared to the [standard VirusTotal expansion module](https://github.com/MISP/misp-modules/blob/master/misp_modules/modules/expansion/virustotal_public.py), this module is made for advanced parsing of VirusTotal report, with a recursive analysis of the elements found after the first request.
+>Compared to the [standard VirusTotal expansion module](https://github.com/MISP/misp-modules/blob/main/misp_modules/modules/expansion/virustotal_public.py), this module is made for advanced parsing of VirusTotal report, with a recursive analysis of the elements found after the first request.
 >
 >Thus, it requires a higher request rate limit to avoid the API to return a 204 error (Request rate limit exceeded), and the data parsed from the different requests are returned as MISP attributes and objects, with the corresponding relations between each one of them.
 - **input**:
@@ -1682,7 +1727,7 @@ Module to get information from VirusTotal.
 >
 >A module to take a MISP attribute as input and query the VirusTotal API to get additional data about it.
 >
->Compared to the [more advanced VirusTotal expansion module](https://github.com/MISP/misp-modules/blob/master/misp_modules/modules/expansion/virustotal.py), this module is made for VirusTotal users who have a low request rate limit.
+>Compared to the [more advanced VirusTotal expansion module](https://github.com/MISP/misp-modules/blob/main/misp_modules/modules/expansion/virustotal.py), this module is made for VirusTotal users who have a low request rate limit.
 >
 >Thus, it only queries the API once and returns the results that is parsed into MISP attributes and objects.
 - **input**:

documentation/mkdocs/import_mod.md

@@ -92,7 +92,7 @@ A module to import data from a Joe Sandbox analysis json report.
 - **features**:
 >Module using the new format of modules able to return attributes and objects.
 >
->The module returns the same results as the expansion module [joesandbox_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/joesandbox_query.py) using the submission link of the analysis to get the json report.
+>The module returns the same results as the expansion module [joesandbox_query](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/joesandbox_query.py) using the submission link of the analysis to get the json report.
 - **input**:
 >Json report of a Joe Sandbox analysis.
 - **output**:
@@ -113,7 +113,7 @@ Module to import and parse reports from Lastline analysis links.
 - **features**:
 >The module requires a Lastline Portal `username` and `password`.
 >The module uses the new format and it is able to return MISP attributes and objects.
->The module returns the same results as the [lastline_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_query.py) expansion module.
+>The module returns the same results as the [lastline_query](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/lastline_query.py) expansion module.
 - **input**:
 >Link to a Lastline analysis.
 - **output**:

documentation/mkdocs/index.md

@@ -2,7 +2,7 @@
 
 [![Build Status](https://travis-ci.org/MISP/misp-modules.svg?branch=master)](https://travis-ci.org/MISP/misp-modules)
 [![Coverage Status](https://coveralls.io/repos/github/MISP/misp-modules/badge.svg?branch=master)](https://coveralls.io/github/MISP/misp-modules?branch=master)
-[![codecov](https://codecov.io/gh/MISP/misp-modules/branch/master/graph/badge.svg)](https://codecov.io/gh/MISP/misp-modules)
+[![codecov](https://codecov.io/gh/MISP/misp-modules/branch/main/graph/badge.svg)](https://codecov.io/gh/MISP/misp-modules)
 [![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgithub.com%MISP%2Fmisp-modules.svg?type=shield)](https://app.fossa.io/projects/git%2Bgithub.com%2FMISP%2Fmisp-modules?ref=badge_shield)
 
 MISP modules are autonomous modules that can be used for expansion and other services in [MISP](https://github.com/MISP/MISP).
@@ -19,93 +19,93 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/
 
 ### Expansion modules
 
-* [Backscatter.io](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/backscatter_io.py) - a hover and expansion module to expand an IP address with mass-scanning observations.
-* [BGP Ranking](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/bgpranking.py) - a hover and expansion module to expand an AS number with the ASN description, its history, and position in BGP Ranking.
-* [BTC scam check](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/btc_scam_check.py) - An expansion hover module to instantly check if a BTC address has been abused.
-* [BTC transactions](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/btc_steroids.py) - An expansion hover module to get a blockchain balance and the transactions from a BTC address in MISP.
-* [CIRCL Passive DNS](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/circl_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information.
-* [CIRCL Passive SSL](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/circl_passivessl.py) - a hover and expansion module to expand IP addresses with the X.509 certificate seen.
-* [countrycode](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/countrycode.py) - a hover module to tell you what country a URL belongs to.
-* [CrowdStrike Falcon](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/crowdstrike_falcon.py) - an expansion module to expand using CrowdStrike Falcon Intel Indicator API.
-* [CVE](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/cve.py) - a hover module to give more information about a vulnerability (CVE).
-* [CVE advanced](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/cve_advanced.py) - An expansion module to query the CIRCL CVE search API for more information about a vulnerability (CVE).
-* [Cuckoo submit](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/cuckoo_submit.py) - A hover module to submit malware sample, url, attachment, domain to Cuckoo Sandbox.
-* [DBL Spamhaus](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/dbl_spamhaus.py) - a hover module to check Spamhaus DBL for a domain name.
-* [DNS](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/dns.py) - a simple module to resolve MISP attributes like hostname and domain to expand IP addresses attributes.
-* [docx-enrich](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/docx-enrich.py) - an enrichment module to get text out of Word document into MISP (using free-text parser).
-* [DomainTools](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/domaintools.py) - a hover and expansion module to get information from [DomainTools](http://www.domaintools.com/) whois.
-* [EUPI](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/eupi.py) - a hover and expansion module to get information about an URL from the [Phishing Initiative project](https://phishing-initiative.eu/?lang=en).
+* [Backscatter.io](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/backscatter_io.py) - a hover and expansion module to expand an IP address with mass-scanning observations.
+* [BGP Ranking](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/bgpranking.py) - a hover and expansion module to expand an AS number with the ASN description, its history, and position in BGP Ranking.
+* [BTC scam check](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/btc_scam_check.py) - An expansion hover module to instantly check if a BTC address has been abused.
+* [BTC transactions](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/btc_steroids.py) - An expansion hover module to get a blockchain balance and the transactions from a BTC address in MISP.
+* [CIRCL Passive DNS](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/circl_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information.
+* [CIRCL Passive SSL](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/circl_passivessl.py) - a hover and expansion module to expand IP addresses with the X.509 certificate seen.
+* [countrycode](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/countrycode.py) - a hover module to tell you what country a URL belongs to.
+* [CrowdStrike Falcon](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/crowdstrike_falcon.py) - an expansion module to expand using CrowdStrike Falcon Intel Indicator API.
+* [CVE](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/cve.py) - a hover module to give more information about a vulnerability (CVE).
+* [CVE advanced](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/cve_advanced.py) - An expansion module to query the CIRCL CVE search API for more information about a vulnerability (CVE).
+* [Cuckoo submit](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/cuckoo_submit.py) - A hover module to submit malware sample, url, attachment, domain to Cuckoo Sandbox.
+* [DBL Spamhaus](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/dbl_spamhaus.py) - a hover module to check Spamhaus DBL for a domain name.
+* [DNS](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/dns.py) - a simple module to resolve MISP attributes like hostname and domain to expand IP addresses attributes.
+* [docx-enrich](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/docx-enrich.py) - an enrichment module to get text out of Word document into MISP (using free-text parser).
+* [DomainTools](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/domaintools.py) - a hover and expansion module to get information from [DomainTools](http://www.domaintools.com/) whois.
+* [EUPI](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/eupi.py) - a hover and expansion module to get information about an URL from the [Phishing Initiative project](https://phishing-initiative.eu/?lang=en).
 * [EQL](misp_modules/modules/expansion/eql.py) - an expansion module to generate event query language (EQL) from an attribute. [Event Query Language](https://eql.readthedocs.io/en/latest/)
-* [Farsight DNSDB Passive DNS](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/farsight_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information.
-* [GeoIP](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/geoip_country.py) - a hover and expansion module to get GeoIP information from geolite/maxmind.
-* [Greynoise](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/greynoise.py) - a hover to get information from greynoise.
-* [hashdd](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/hashdd.py) - a hover module to check file hashes against [hashdd.com](http://www.hashdd.com) including NSLR dataset.
-* [hibp](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/hibp.py) - a hover module to lookup against Have I Been Pwned?
-* [intel471](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/intel471.py) - an expansion module to get info from [Intel471](https://intel471.com).
-* [IPASN](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/ipasn.py) - a hover and expansion to get the BGP ASN of an IP address.
-* [iprep](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/iprep.py) - an expansion module to get IP reputation from packetmail.net.
-* [Joe Sandbox submit](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/joesandbox_submit.py) - Submit files and URLs to Joe Sandbox.
-* [Joe Sandbox query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/joesandbox_query.py) - Query Joe Sandbox with the link of an analysis and get the parsed data.
-* [macaddress.io](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/macaddress_io.py) - a hover module to retrieve vendor details and other information regarding a given MAC address or an OUI from [MAC address Vendor Lookup](https://macaddress.io). See [integration tutorial here](https://macaddress.io/integrations/MISP-module).
-* [macvendors](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/macvendors.py) - a hover module to retrieve mac vendor information.
-* [ocr-enrich](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/ocr-enrich.py) - an enrichment module to get OCRized data from images into MISP.
-* [ods-enrich](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/ods-enrich.py) - an enrichment module to get text out of OpenOffice spreadsheet document into MISP (using free-text parser).
-* [odt-enrich](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/odt-enrich.py) - an enrichment module to get text out of OpenOffice document into MISP (using free-text parser).
-* [onyphe](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/onyphe.py) - a modules to process queries on Onyphe.
-* [onyphe_full](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/onyphe_full.py) - a modules to process full queries on Onyphe.
-* [OTX](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/otx.py) - an expansion module for [OTX](https://otx.alienvault.com/).
-* [passivetotal](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/passivetotal.py) - a [passivetotal](https://www.passivetotal.org/) module that queries a number of different PassiveTotal datasets.
-* [pdf-enrich](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/pdf-enrich.py) - an enrichment module to extract text from PDF into MISP (using free-text parser).
-* [pptx-enrich](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/pptx-enrich.py) - an enrichment module to get text out of PowerPoint document into MISP (using free-text parser).
-* [qrcode](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/qrcode.py) - a module decode QR code, barcode and similar codes from an image and enrich with the decoded values.
-* [rbl](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/rbl.py) - a module to get RBL (Real-Time Blackhost List) values from an attribute.
-* [reversedns](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/reversedns.py) - Simple Reverse DNS expansion service to resolve reverse DNS from MISP attributes.
-* [securitytrails](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/securitytrails.py) - an expansion module for [securitytrails](https://securitytrails.com/).
-* [shodan](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/shodan.py) - a minimal [shodan](https://www.shodan.io/) expansion module.
-* [Sigma queries](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/sigma_queries.py) - Experimental expansion module querying a sigma rule to convert it into all the available SIEM signatures.
-* [Sigma syntax validator](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/sigma_syntax_validator.py) - Sigma syntax validator.
-* [sourcecache](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/sourcecache.py) - a module to cache a specific link from a MISP instance.
-* [STIX2 pattern syntax validator](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/stix2_pattern_syntax_validator.py) - a module to check a STIX2 pattern syntax.
-* [ThreatCrowd](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/threatcrowd.py) - an expansion module for [ThreatCrowd](https://www.threatcrowd.org/).
-* [threatminer](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/threatminer.py) - an expansion module to expand from [ThreatMiner](https://www.threatminer.org/).
-* [urlhaus](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/urlhaus.py) - Query urlhaus to get additional data about a domain, hash, hostname, ip or url.
-* [urlscan](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/urlscan.py) - an expansion module to query [urlscan.io](https://urlscan.io).
-* [virustotal](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/virustotal.py) - an expansion module to query the [VirusTotal](https://www.virustotal.com/gui/home) API with a high request rate limit required. (More details about the API: [here](https://developers.virustotal.com/reference))
-* [virustotal_public](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/virustotal_public.py) - an expansion module to query the [VirusTotal](https://www.virustotal.com/gui/home) API with a public key and a low request rate limit. (More details about the API: [here](https://developers.virustotal.com/reference))
-* [VMray](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/vmray_submit.py) - a module to submit a sample to VMray.
-* [VulnDB](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/vulndb.py) - a module to query [VulnDB](https://www.riskbasedsecurity.com/).
-* [Vulners](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/vulners.py) - an expansion module to expand information about CVEs using Vulners API.
-* [whois](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/whois.py) - a module to query a local instance of [uwhois](https://github.com/rafiot/uwhoisd).
-* [wikidata](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/wiki.py) - a [wikidata](https://www.wikidata.org) expansion module.
-* [xforce](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/xforceexchange.py) - an IBM X-Force Exchange expansion module.
-* [xlsx-enrich](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/xlsx-enrich.py) - an enrichment module to get text out of an Excel document into MISP (using free-text parser).
-* [YARA query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/yara_query.py) - a module to create YARA rules from single hash attributes.
-* [YARA syntax validator](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/yara_syntax_validator.py) - YARA syntax validator.
+* [Farsight DNSDB Passive DNS](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/farsight_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information.
+* [GeoIP](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/geoip_country.py) - a hover and expansion module to get GeoIP information from geolite/maxmind.
+* [Greynoise](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/greynoise.py) - a hover to get information from greynoise.
+* [hashdd](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/hashdd.py) - a hover module to check file hashes against [hashdd.com](http://www.hashdd.com) including NSLR dataset.
+* [hibp](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/hibp.py) - a hover module to lookup against Have I Been Pwned?
+* [intel471](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/intel471.py) - an expansion module to get info from [Intel471](https://intel471.com).
+* [IPASN](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/ipasn.py) - a hover and expansion to get the BGP ASN of an IP address.
+* [iprep](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/iprep.py) - an expansion module to get IP reputation from packetmail.net.
+* [Joe Sandbox submit](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/joesandbox_submit.py) - Submit files and URLs to Joe Sandbox.
+* [Joe Sandbox query](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/joesandbox_query.py) - Query Joe Sandbox with the link of an analysis and get the parsed data.
+* [macaddress.io](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/macaddress_io.py) - a hover module to retrieve vendor details and other information regarding a given MAC address or an OUI from [MAC address Vendor Lookup](https://macaddress.io). See [integration tutorial here](https://macaddress.io/integrations/MISP-module).
+* [macvendors](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/macvendors.py) - a hover module to retrieve mac vendor information.
+* [ocr-enrich](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/ocr-enrich.py) - an enrichment module to get OCRized data from images into MISP.
+* [ods-enrich](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/ods-enrich.py) - an enrichment module to get text out of OpenOffice spreadsheet document into MISP (using free-text parser).
+* [odt-enrich](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/odt-enrich.py) - an enrichment module to get text out of OpenOffice document into MISP (using free-text parser).
+* [onyphe](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/onyphe.py) - a modules to process queries on Onyphe.
+* [onyphe_full](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/onyphe_full.py) - a modules to process full queries on Onyphe.
+* [OTX](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/otx.py) - an expansion module for [OTX](https://otx.alienvault.com/).
+* [passivetotal](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/passivetotal.py) - a [passivetotal](https://www.passivetotal.org/) module that queries a number of different PassiveTotal datasets.
+* [pdf-enrich](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/pdf-enrich.py) - an enrichment module to extract text from PDF into MISP (using free-text parser).
+* [pptx-enrich](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/pptx-enrich.py) - an enrichment module to get text out of PowerPoint document into MISP (using free-text parser).
+* [qrcode](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/qrcode.py) - a module decode QR code, barcode and similar codes from an image and enrich with the decoded values.
+* [rbl](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/rbl.py) - a module to get RBL (Real-Time Blackhost List) values from an attribute.
+* [reversedns](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/reversedns.py) - Simple Reverse DNS expansion service to resolve reverse DNS from MISP attributes.
+* [securitytrails](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/securitytrails.py) - an expansion module for [securitytrails](https://securitytrails.com/).
+* [shodan](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/shodan.py) - a minimal [shodan](https://www.shodan.io/) expansion module.
+* [Sigma queries](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/sigma_queries.py) - Experimental expansion module querying a sigma rule to convert it into all the available SIEM signatures.
+* [Sigma syntax validator](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/sigma_syntax_validator.py) - Sigma syntax validator.
+* [sourcecache](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/sourcecache.py) - a module to cache a specific link from a MISP instance.
+* [STIX2 pattern syntax validator](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/stix2_pattern_syntax_validator.py) - a module to check a STIX2 pattern syntax.
+* [ThreatCrowd](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/threatcrowd.py) - an expansion module for [ThreatCrowd](https://www.threatcrowd.org/).
+* [threatminer](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/threatminer.py) - an expansion module to expand from [ThreatMiner](https://www.threatminer.org/).
+* [urlhaus](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/urlhaus.py) - Query urlhaus to get additional data about a domain, hash, hostname, ip or url.
+* [urlscan](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/urlscan.py) - an expansion module to query [urlscan.io](https://urlscan.io).
+* [virustotal](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/virustotal.py) - an expansion module to query the [VirusTotal](https://www.virustotal.com/gui/home) API with a high request rate limit required. (More details about the API: [here](https://developers.virustotal.com/reference))
+* [virustotal_public](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/virustotal_public.py) - an expansion module to query the [VirusTotal](https://www.virustotal.com/gui/home) API with a public key and a low request rate limit. (More details about the API: [here](https://developers.virustotal.com/reference))
+* [VMray](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/vmray_submit.py) - a module to submit a sample to VMray.
+* [VulnDB](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/vulndb.py) - a module to query [VulnDB](https://www.riskbasedsecurity.com/).
+* [Vulners](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/vulners.py) - an expansion module to expand information about CVEs using Vulners API.
+* [whois](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/whois.py) - a module to query a local instance of [uwhois](https://github.com/rafiot/uwhoisd).
+* [wikidata](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/wiki.py) - a [wikidata](https://www.wikidata.org) expansion module.
+* [xforce](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/xforceexchange.py) - an IBM X-Force Exchange expansion module.
+* [xlsx-enrich](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/xlsx-enrich.py) - an enrichment module to get text out of an Excel document into MISP (using free-text parser).
+* [YARA query](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/yara_query.py) - a module to create YARA rules from single hash attributes.
+* [YARA syntax validator](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/yara_syntax_validator.py) - YARA syntax validator.
 
 ### Export modules
 
-* [CEF](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/cef_export.py) module to export Common Event Format (CEF).
-* [Cisco FireSight Manager ACL rule](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/cisco_firesight_manager_ACL_rule_export.py) module to export as rule for the Cisco FireSight manager ACL.
-* [GoAML export](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/goamlexport.py) module to export in [GoAML format](http://goaml.unodc.org/goaml/en/index.html).
-* [Lite Export](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/liteexport.py) module to export a lite event.
+* [CEF](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/export_mod/cef_export.py) module to export Common Event Format (CEF).
+* [Cisco FireSight Manager ACL rule](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/export_mod/cisco_firesight_manager_ACL_rule_export.py) module to export as rule for the Cisco FireSight manager ACL.
+* [GoAML export](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/export_mod/goamlexport.py) module to export in [GoAML format](http://goaml.unodc.org/goaml/en/index.html).
+* [Lite Export](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/export_mod/liteexport.py) module to export a lite event.
 * [Mass EQL Export](misp_modules/modules/export_mod/mass_eql_export.py) module to export applicable attributes from an event to a mass EQL query.
-* [PDF export](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/pdfexport.py) module to export an event in PDF.
-* [Nexthink query format](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/nexthinkexport.py) module to export in Nexthink query format.
-* [osquery](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/osqueryexport.py) module to export in [osquery](https://osquery.io/) query format.
-* [ThreatConnect](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/threat_connect_export.py) module to export in ThreatConnect CSV format.
-* [ThreatStream](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/threatStream_misp_export.py) module to export in ThreatStream format.
+* [PDF export](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/export_mod/pdfexport.py) module to export an event in PDF.
+* [Nexthink query format](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/export_mod/nexthinkexport.py) module to export in Nexthink query format.
+* [osquery](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/export_mod/osqueryexport.py) module to export in [osquery](https://osquery.io/) query format.
+* [ThreatConnect](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/export_mod/threat_connect_export.py) module to export in ThreatConnect CSV format.
+* [ThreatStream](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/export_mod/threatStream_misp_export.py) module to export in ThreatStream format.
 
 ### Import modules
 
-* [CSV import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/csvimport.py) Customizable CSV import module.
-* [Cuckoo JSON](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/cuckooimport.py) Cuckoo JSON import.
-* [Email Import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/email_import.py) Email import module for MISP to import basic metadata.
-* [GoAML import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/goamlimport.py) Module to import [GoAML](http://goaml.unodc.org/goaml/en/index.html) XML format.
-* [Joe Sandbox import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/joe_import.py) Parse data from a Joe Sandbox json report.
-* [OCR](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/ocr.py) Optical Character Recognition (OCR) module for MISP to import attributes from images, scan or faxes.
-* [OpenIOC](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/openiocimport.py) OpenIOC import based on PyMISP library.
-* [ThreatAnalyzer](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/threatanalyzer_import.py) - An import module to process ThreatAnalyzer archive.zip/analysis.json sandbox exports.
-* [VMRay](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/vmray_import.py) - An import module to process VMRay export.
+* [CSV import](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/import_mod/csvimport.py) Customizable CSV import module.
+* [Cuckoo JSON](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/import_mod/cuckooimport.py) Cuckoo JSON import.
+* [Email Import](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/import_mod/email_import.py) Email import module for MISP to import basic metadata.
+* [GoAML import](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/import_mod/goamlimport.py) Module to import [GoAML](http://goaml.unodc.org/goaml/en/index.html) XML format.
+* [Joe Sandbox import](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/import_mod/joe_import.py) Parse data from a Joe Sandbox json report.
+* [OCR](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/import_mod/ocr.py) Optical Character Recognition (OCR) module for MISP to import attributes from images, scan or faxes.
+* [OpenIOC](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/import_mod/openiocimport.py) OpenIOC import based on PyMISP library.
+* [ThreatAnalyzer](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/import_mod/threatanalyzer_import.py) - An import module to process ThreatAnalyzer archive.zip/analysis.json sandbox exports.
+* [VMRay](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/import_mod/vmray_import.py) - An import module to process VMRay export.
 
 
 ## How to contribute your own module?
@@ -117,4 +117,4 @@ For further information please see [Contribute](contribute/).
 ## Licenses
 [![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgithub.com%MISP%2Fmisp-modules.svg?type=large)](https://app.fossa.io/projects/git%2Bgithub.com%2FMISP%2Fmisp-modules?ref=badge_large)
 
-For further Information see also the [license file](license/).
+For further Information see also the [license file](license/).

documentation/website/expansion/crowdsec.json

@@ -0,0 +1,15 @@
+{
+    "description": "Hover module to lookup an IP in CrowdSec's CTI",
+    "logo": "crowdsec.png",
+    "requirements": [
+        "A CrowdSec CTI API key. Get yours by following https://docs.crowdsec.net/docs/cti_api/getting_started/#getting-an-api-key"
+    ],
+    "input": "An IP address.",
+    "output": "IP Lookup information from CrowdSec CTI API",
+    "references": [
+        "https://www.crowdsec.net/",
+        "https://docs.crowdsec.net/docs/cti_api/getting_started",
+        "https://app.crowdsec.net/"
+    ],
+    "features": "This module enables IP lookup from CrowdSec CTI API. It provides information about the IP, such as what kind of attacks it has been participant of as seen by CrowdSec's network. It also includes enrichment by CrowdSec like background noise score, aggressivity over time etc."
+}

documentation/website/expansion/ipinfo.json

@@ -0,0 +1,13 @@
+{
+    "description": "An expansion module to query ipinfo.io to gather more information on a given IP address.",
+    "logo": "ipinfo.png",
+    "requirements": [
+        "An ipinfo.io token"
+    ],
+    "input": "IP address attribute.",
+    "output": "Additional information on the IP address, like its geolocation, the autonomous system it is included in, and the related domain(s).",
+    "references": [
+        "https://ipinfo.io/developers"
+    ],
+    "features": "The module takes an IP address attribute as input and queries the ipinfo.io API.  \nThe geolocation information on the IP address is always returned.\n\nDepending on the subscription plan, the API returns different pieces of information then:\n- With a basic plan (free) you get the AS number and the AS organisation name concatenated in the `org` field.\n- With a paid subscription, the AS information is returned in the `asn` field with additional AS information, and depending on which plan the user has, you can also get information on the privacy method used to protect the IP address, the related domains, or the point of contact related to the IP address in case of an abuse.\n\nMore information on the responses content is available in the [documentation](https://ipinfo.io/developers)."
+}

documentation/website/expansion/joesandbox_query.json

@@ -1,5 +1,5 @@
 {
-    "description": "Query Joe Sandbox API with a submission url to get the json report and extract its data that is parsed and converted into MISP attributes and objects.\n\nThis url can by the way come from the result of the [joesandbox_submit expansion module](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/joesandbox_submit.py).",
+    "description": "Query Joe Sandbox API with a submission url to get the json report and extract its data that is parsed and converted into MISP attributes and objects.\n\nThis url can by the way come from the result of the [joesandbox_submit expansion module](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/joesandbox_submit.py).",
     "logo": "joesandbox.png",
     "requirements": [
         "jbxapi: Joe Sandbox API python3 library"
@@ -10,5 +10,5 @@
         "https://www.joesecurity.org",
         "https://www.joesandbox.com/"
     ],
-    "features": "Module using the new format of modules able to return attributes and objects.\n\nThe module returns the same results as the import module [joe_import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/joe_import.py) taking directly the json report as input.\n\nEven if the introspection will allow all kinds of links to call this module, obviously only the ones presenting a sample or url submission in the Joe Sandbox API will return results.\n\nTo make it work you will need to fill the 'apikey' configuration with your Joe Sandbox API key and provide a valid link as input."
+    "features": "Module using the new format of modules able to return attributes and objects.\n\nThe module returns the same results as the import module [joe_import](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/import_mod/joe_import.py) taking directly the json report as input.\n\nEven if the introspection will allow all kinds of links to call this module, obviously only the ones presenting a sample or url submission in the Joe Sandbox API will return results.\n\nTo make it work you will need to fill the 'apikey' configuration with your Joe Sandbox API key and provide a valid link as input."
 }

documentation/website/expansion/joesandbox_submit.json

@@ -10,5 +10,5 @@
         "https://www.joesecurity.org",
         "https://www.joesandbox.com/"
     ],
-    "features": "The module requires a Joe Sandbox API key to submit files or URL, and returns the link of the submitted analysis.\n\nIt is then possible, when the analysis is completed, to query the Joe Sandbox API to get the data related to the analysis, using the [joesandbox_query module](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/joesandbox_query.py) directly on this submission link."
+    "features": "The module requires a Joe Sandbox API key to submit files or URL, and returns the link of the submitted analysis.\n\nIt is then possible, when the analysis is completed, to query the Joe Sandbox API to get the data related to the analysis, using the [joesandbox_query module](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/joesandbox_query.py) directly on this submission link."
 }

documentation/website/expansion/lastline_query.json

@@ -1,5 +1,5 @@
 {
-    "description": "Deprecation notice: this module will be deprecated by December 2021, please use vmware_nsx module.\n\nQuery Lastline with an analysis link and parse the report into MISP attributes and objects.\nThe analysis link can also be retrieved from the output of the [lastline_submit](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_submit.py) expansion module.",
+    "description": "Deprecation notice: this module will be deprecated by December 2021, please use vmware_nsx module.\n\nQuery Lastline with an analysis link and parse the report into MISP attributes and objects.\nThe analysis link can also be retrieved from the output of the [lastline_submit](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/lastline_submit.py) expansion module.",
     "logo": "lastline.png",
     "requirements": [],
     "input": "Link to a Lastline analysis.",
@@ -7,5 +7,5 @@
     "references": [
         "https://www.lastline.com"
     ],
-    "features": "The module requires a Lastline Portal `username` and `password`.\nThe module uses the new format and it is able to return MISP attributes and objects.\nThe module returns the same results as the [lastline_import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/lastline_import.py) import module."
+    "features": "The module requires a Lastline Portal `username` and `password`.\nThe module uses the new format and it is able to return MISP attributes and objects.\nThe module returns the same results as the [lastline_import](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/import_mod/lastline_import.py) import module."
 }

documentation/website/expansion/lastline_submit.json

@@ -7,5 +7,5 @@
     "references": [
         "https://www.lastline.com"
     ],
-    "features": "The module requires a Lastline Analysis `api_token` and `key`.\nWhen the analysis is completed, it is possible to import the generated report by feeding the analysis link to the [lastline_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_query.py) module."
+    "features": "The module requires a Lastline Analysis `api_token` and `key`.\nWhen the analysis is completed, it is possible to import the generated report by feeding the analysis link to the [lastline_query](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/lastline_query.py) module."
 }

documentation/website/expansion/virustotal.json

@@ -10,5 +10,5 @@
         "https://www.virustotal.com/",
         "https://developers.virustotal.com/reference"
     ],
-    "features": "New format of modules able to return attributes and objects.\n\nA module to take a MISP attribute as input and query the VirusTotal API to get additional data about it.\n\nCompared to the [standard VirusTotal expansion module](https://github.com/MISP/misp-modules/blob/master/misp_modules/modules/expansion/virustotal_public.py), this module is made for advanced parsing of VirusTotal report, with a recursive analysis of the elements found after the first request.\n\nThus, it requires a higher request rate limit to avoid the API to return a 204 error (Request rate limit exceeded), and the data parsed from the different requests are returned as MISP attributes and objects, with the corresponding relations between each one of them."
+    "features": "New format of modules able to return attributes and objects.\n\nA module to take a MISP attribute as input and query the VirusTotal API to get additional data about it.\n\nCompared to the [standard VirusTotal expansion module](https://github.com/MISP/misp-modules/blob/main/misp_modules/modules/expansion/virustotal_public.py), this module is made for advanced parsing of VirusTotal report, with a recursive analysis of the elements found after the first request.\n\nThus, it requires a higher request rate limit to avoid the API to return a 204 error (Request rate limit exceeded), and the data parsed from the different requests are returned as MISP attributes and objects, with the corresponding relations between each one of them."
 }

documentation/website/expansion/virustotal_public.json

@@ -10,5 +10,5 @@
         "https://www.virustotal.com",
         "https://developers.virustotal.com/reference"
     ],
-    "features": "New format of modules able to return attributes and objects.\n\nA module to take a MISP attribute as input and query the VirusTotal API to get additional data about it.\n\nCompared to the [more advanced VirusTotal expansion module](https://github.com/MISP/misp-modules/blob/master/misp_modules/modules/expansion/virustotal.py), this module is made for VirusTotal users who have a low request rate limit.\n\nThus, it only queries the API once and returns the results that is parsed into MISP attributes and objects."
+    "features": "New format of modules able to return attributes and objects.\n\nA module to take a MISP attribute as input and query the VirusTotal API to get additional data about it.\n\nCompared to the [more advanced VirusTotal expansion module](https://github.com/MISP/misp-modules/blob/main/misp_modules/modules/expansion/virustotal.py), this module is made for VirusTotal users who have a low request rate limit.\n\nThus, it only queries the API once and returns the results that is parsed into MISP attributes and objects."
 }

documentation/website/import_mod/joe_import.json

@@ -8,5 +8,5 @@
         "https://www.joesecurity.org",
         "https://www.joesandbox.com/"
     ],
-    "features": "Module using the new format of modules able to return attributes and objects.\n\nThe module returns the same results as the expansion module [joesandbox_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/joesandbox_query.py) using the submission link of the analysis to get the json report."
+    "features": "Module using the new format of modules able to return attributes and objects.\n\nThe module returns the same results as the expansion module [joesandbox_query](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/joesandbox_query.py) using the submission link of the analysis to get the json report."
 }

documentation/website/import_mod/lastline_import.json

@@ -7,5 +7,5 @@
     "references": [
         "https://www.lastline.com"
     ],
-    "features": "The module requires a Lastline Portal `username` and `password`.\nThe module uses the new format and it is able to return MISP attributes and objects.\nThe module returns the same results as the [lastline_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_query.py) expansion module."
+    "features": "The module requires a Lastline Portal `username` and `password`.\nThe module uses the new format and it is able to return MISP attributes and objects.\nThe module returns the same results as the [lastline_query](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/lastline_query.py) expansion module."
 }

misp_modules/modules/expansion/__init__.py

@@ -19,7 +19,8 @@ __all__ = ['cuckoo_submit', 'vmray_submit', 'bgpranking', 'circl_passivedns', 'c
            'lastline_query', 'lastline_submit', 'sophoslabs_intelix', 'cytomic_orion', 'censys_enrich',
            'trustar_enrich', 'recordedfuture', 'html_to_markdown', 'socialscan', 'passive-ssh',
            'qintel_qsentry', 'mwdb', 'hashlookup', 'mmdb_lookup', 'ipqs_fraud_and_risk_scoring',
-           'clamav', 'jinja_template_rendering','hyasinsight', 'variotdbs']
+           'clamav', 'jinja_template_rendering','hyasinsight', 'variotdbs', 'crowdsec',
+           'extract_url_components', 'ipinfo']
 
 
 minimum_required_fields = ('type', 'uuid', 'value')

misp_modules/modules/expansion/crowdsec.py

@@ -0,0 +1,136 @@
+import json
+import pycountry
+import requests
+from . import check_input_attribute, standard_error_message
+from pymisp import MISPEvent, MISPObject
+
+mispattributes = {"input": ["ip-dst", "ip-src"], "format": "misp_standard"}
+moduleinfo = {
+    "version": "2.0",
+    "author": "Shivam Sandbhor <shivam@crowdsec.net>",
+    "description": "Module to access CrowdSec CTI API.",
+    "module-type": ["hover", "expansion"],
+}
+moduleconfig = ["api_key"]
+
+
+def handler(q=False):
+    if q is False:
+        return False
+
+    request = json.loads(q)
+    if not request.get("config"):
+        return {"error": "Missing CrowdSec Config"}
+
+    if not request["config"].get("api_key"):
+        return {"error": "Missing CrowdSec API key"}
+
+    if not request.get('attribute') or not check_input_attribute(request['attribute']):
+        return {'error': f'{standard_error_message}, which shoul contain at least a type, a value and an uuid.'}
+
+    if request['attribute'].get('type') not in mispattributes['input']:
+        return {'error': f"Wrong input type. Please choose on of the following: {', '.join(mispattributes['input'])}"}
+
+    return _handler_v2(request)
+
+
+def _handler_v2(request_data):
+    attribute = request_data['attribute']
+    ip = attribute['value']
+
+    crowdsec_cti = requests.get(
+        f"https://cti.api.crowdsec.net/v2/smoke/{ip}",
+        headers={
+            "x-api-key": request_data["config"]["api_key"],
+            "User-Agent": "crowdsec-misp/v1.0.0",
+        }
+    )
+    crowdsec_cti.raise_for_status()
+    crowdsec_cti = crowdsec_cti.json()
+
+    misp_event = MISPEvent()
+    misp_attribute = misp_event.add_attribute(**attribute)
+    crowdsec_context_object = MISPObject("crowdsec-ip-context")
+    crowdsec_context_object.from_dict(
+        first_seen=crowdsec_cti["history"]["first_seen"],
+        last_seen=crowdsec_cti["history"]["last_seen"]
+    )
+    ip_attribute = crowdsec_context_object.add_attribute("ip", crowdsec_cti["ip"])
+    crowdsec_context_object.add_attribute("ip-range", crowdsec_cti["ip_range"])
+    crowdsec_context_object.add_attribute("ip-range-score", crowdsec_cti["ip_range_score"])
+    crowdsec_context_object.add_attribute(
+        "country", get_country_name_from_alpha_2(crowdsec_cti["location"]["country"])
+    )
+    crowdsec_context_object.add_attribute("country-code", crowdsec_cti["location"]["country"])
+    if crowdsec_cti["location"].get("city"):
+        crowdsec_context_object.add_attribute(
+            "city", crowdsec_cti["location"]["city"]
+        )
+    crowdsec_context_object.add_attribute("latitude", crowdsec_cti["location"]["latitude"])
+    crowdsec_context_object.add_attribute("longitude", crowdsec_cti["location"]["longitude"])
+    crowdsec_context_object.add_attribute("as-name", crowdsec_cti["as_name"])
+    crowdsec_context_object.add_attribute("as-num", crowdsec_cti["as_num"])
+    if crowdsec_cti.get('reverse_dns') is not None:
+        crowdsec_context_object.add_attribute("reverse-dns", crowdsec_cti["reverse_dns"])
+    crowdsec_context_object.add_attribute('background-noise', crowdsec_cti['background_noise_score'])
+    for behavior in crowdsec_cti["behaviors"]:
+        crowdsec_context_object.add_attribute(
+            "behaviors", behavior["label"],
+            comment=behavior['description']
+        )
+        tag = f'crowdsec:behavior="{behavior["name"]}"'
+        ip_attribute.add_tag(tag)
+    for feature, values in crowdsec_cti['classifications'].items():
+        field = feature[:-1]
+        for value in values:
+            crowdsec_context_object.add_attribute(
+                feature, value['label'], comment=value['description']
+            )
+            tag = f'crowdsec:{field}="{value["name"]}"'
+            ip_attribute.add_tag(tag)
+    crowdsec_context_object.add_attribute(
+        "attack-details",
+        ", ".join(
+            f"{scenario['name']} - {scenario['label']} ({scenario['description']})"
+            for scenario in crowdsec_cti["attack_details"]
+        )
+    )
+    crowdsec_context_object.add_attribute(
+        "target-countries",
+        ", ".join(
+            map(
+                get_country_name_from_alpha_2,
+                crowdsec_cti["target_countries"].keys()
+            )
+        )
+    )
+    crowdsec_context_object.add_attribute("trust", crowdsec_cti["scores"]["overall"]["trust"])
+    scores = []
+    for time_period, indicators in crowdsec_cti["scores"].items():
+        tp = ' '.join(map(str.capitalize, time_period.split('_')))
+        indicator = (
+            f'{indicator_type.capitalize()}: {indicator_value}'
+            for indicator_type, indicator_value in indicators.items()
+        )
+        scores.append(f"{tp}: {' - '.join(indicator)}")
+    crowdsec_context_object.add_attribute('scores', ', '.join(scores))
+    crowdsec_context_object.add_reference(misp_attribute.uuid, 'related-to')
+    misp_event.add_object(crowdsec_context_object)
+
+    event = json.loads(misp_event.to_json())
+    results = {key: event[key] for key in ("Attribute", "Object") if (key in event and event[key])}
+    return {"results": results}
+
+
+def get_country_name_from_alpha_2(alpha_2):
+    country_info = pycountry.countries.get(alpha_2=alpha_2)
+    return country_info.name
+
+
+def introspection():
+    return mispattributes
+
+
+def version():
+    moduleinfo["config"] = moduleconfig
+    return moduleinfo

misp_modules/modules/expansion/dbl_spamhaus.py

@@ -2,7 +2,10 @@ import json
 import sys
 
 try:
+    original_path = sys.path
+    sys.path = original_path[1:]
     import dns.resolver
+    sys.path = original_path
     resolver = dns.resolver.Resolver()
     resolver.timeout = 0.2
     resolver.lifetime = 0.2

misp_modules/modules/expansion/extract_url_components.py

@@ -0,0 +1,70 @@
+import json
+from pymisp import MISPEvent, MISPObject
+from . import check_input_attribute, standard_error_message
+from pyfaup.faup import Faup
+
+misperrors = {'error': 'Error'}
+mispattributes = {'input': ['url'], 'format': 'misp_standard'}
+moduleinfo = {'version': '1', 'author': 'MISP Team',
+              'description': "Extract URL components",
+              'module-type': ['expansion', 'hover']}
+moduleconfig = []
+
+
+def createObjectFromURL(url):
+    f = Faup()
+    f.decode(url)
+    parsed = f.get()
+    obj = MISPObject('url')
+    obj.add_attribute('url', type='url', value=url)
+    if parsed['tld'] is not None:
+        obj.add_attribute('tld', type='text', value=parsed['tld'])
+    if parsed['subdomain'] is not None:
+        obj.add_attribute('subdomain', type='text', value=parsed['subdomain'])
+    obj.add_attribute('scheme', type='text', value=parsed['scheme'])
+    obj.add_attribute('resource_path', type='text', value=parsed['resource_path'])
+    obj.add_attribute('query_string', type='text', value=parsed['query_string'])
+    obj.add_attribute('port', type='port', value=parsed['port'])
+    obj.add_attribute('host', type='hostname', value=parsed['host'])
+    if parsed['fragment'] is not None:
+        obj.add_attribute('fragment', type='text', value=parsed['fragment'])
+    obj.add_attribute('domain_without_tld', type='text', value=parsed['domain_without_tld'])
+    obj.add_attribute('domain', type='domain', value=parsed['domain'])
+    return obj
+
+
+def createEvent(urlObject, attributeUUID, urlAttribute):
+    mispEvent = MISPEvent()
+    mispEvent.add_attribute(**urlAttribute)
+    urlObject.add_reference(attributeUUID, 'generated-from')
+    mispEvent.add_object(urlObject)
+    return mispEvent
+
+def handler(q=False):
+    if q is False:
+        return False
+    request = json.loads(q)
+    if not request.get('attribute') or not check_input_attribute(request['attribute']):
+        return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'}
+    attribute = request['attribute']
+    
+    if attribute['type'] not in mispattributes['input']:
+        return {'error': 'Bad attribute type'} 
+
+    url = attribute['value'] 
+    urlObject = createObjectFromURL(url)
+
+    event = createEvent(urlObject, attribute['uuid'], attribute)
+    event = json.loads(event.to_json())
+    
+    result = {'results': {'Object': event['Object']}}
+    return result
+
+
+def introspection():
+    return mispattributes
+
+
+def version():
+    moduleinfo['config'] = moduleconfig
+    return moduleinfo

misp_modules/modules/expansion/greynoise.py

@@ -1,254 +1,333 @@
+import ipaddress
 import json
+import logging
 
-import requests
-from pymisp import MISPEvent, MISPObject
+
+try:
+    from greynoise import GreyNoise
+except ImportError:
+    print("greynoise module not installed.")
+from pymisp import MISPAttribute, MISPEvent, MISPObject
+
+from . import check_input_attribute, standard_error_message
+
+logger = logging.getLogger("greynoise")
+logger.setLevel(logging.INFO)
 
 misperrors = {"error": "Error"}
-mispattributes = {"input": ["ip-dst", "ip-src", "vulnerability"], "output": ["text"]}
+mispattributes = {"input": ["ip-src", "ip-dst", "vulnerability"], "format": "misp_standard"}
 moduleinfo = {
-    "version": "1.1",
+    "version": "1.2",
     "author": "Brad Chiappetta <brad@greynoise.io>",
-    "description": "Module to access GreyNoise.io API.",
-    "module-type": ["hover"],
+    "description": "Used to query IP and CVE intel from GreyNoise",
+    "module-type": ["expansion", "hover"],
 }
 moduleconfig = ["api_key", "api_type"]
-codes_mapping = {
-    "0x00": "The IP has never been observed scanning the Internet",
-    "0x01": "The IP has been observed by the GreyNoise sensor network",
-    "0x02": "The IP has been observed scanning the GreyNoise sensor network, "
-    "but has not completed a full connection, meaning this can be spoofed",
-    "0x03": "The IP is adjacent to another host that has been directly observed by the GreyNoise sensor network",
-    "0x04": "Reserved",
-    "0x05": "This IP is commonly spoofed in Internet-scan activity",
-    "0x06": "This IP has been observed as noise, but this host belongs to a cloud provider where IPs can be "
-            "cycled frequently",
-    "0x07": "This IP is invalid",
-    "0x08": "This IP was classified as noise, but has not been observed engaging in Internet-wide scans or "
-            "attacks in over 90 days",
-    "0x09": "IP was found in RIOT",
-    "0x10": "IP has been observed by the GreyNoise sensor network and is in RIOT",
-}
-vulnerability_mapping = {
-    "id": ("vulnerability", "CVE #"),
-    "details": ("text", "Details"),
-    "count": ("text", "Total Scanner Count"),
-}
-enterprise_context_basic_mapping = {"ip": ("text", "IP Address"), "code_message": ("text", "Code Message")}
-enterprise_context_advanced_mapping = {
-    "noise": ("text", "Is Internet Background Noise"),
-    "link": ("link", "Visualizer Link"),
-    "classification": ("text", "Classification"),
-    "actor": ("text", "Actor"),
-    "tags": ("text", "Tags"),
-    "cve": ("text", "CVEs"),
-    "first_seen": ("text", "First Seen Scanning"),
-    "last_seen": ("text", "Last Seen Scanning"),
-    "vpn": ("text", "Known VPN Service"),
-    "vpn_service": ("text", "VPN Service Name"),
-    "bot": ("text", "Known BOT"),
-}
-enterprise_context_advanced_metadata_mapping = {
-    "asn": ("text", "ASN"),
-    "rdns": ("text", "rDNS"),
-    "category": ("text", "Category"),
-    "tor": ("text", "Known Tor Exit Node"),
-    "region": ("text", "Region"),
-    "city": ("text", "City"),
-    "country": ("text", "Country"),
-    "country_code": ("text", "Country Code"),
-    "organization": ("text", "Organization"),
-}
-enterprise_riot_mapping = {
-    "riot": ("text", "Is Common Business Service"),
-    "link": ("link", "Visualizer Link"),
-    "category": ("text", "RIOT Category"),
-    "name": ("text", "Provider Name"),
-    "trust_level": ("text", "RIOT Trust Level"),
-    "last_updated": ("text", "Last Updated"),
-}
-community_found_mapping = {
-    "ip": ("text", "IP Address"),
-    "noise": ("text", "Is Internet Background Noise"),
-    "riot": ("text", "Is Common Business Service"),
-    "classification": ("text", "Classification"),
-    "last_seen": ("text", "Last Seen"),
-    "name": ("text", "Name"),
-    "link": ("link", "Visualizer Link"),
-}
-community_not_found_mapping = {
-    "ip": ("text", "IP Address"),
-    "noise": ("text", "Is Internet Background Noise"),
-    "riot": ("text", "Is Common Business Service"),
-    "message": ("text", "Message"),
-}
-misp_event = MISPEvent()
 
 
-def handler(q=False):  # noqa: C901
-    if q is False:
-        return False
-    request = json.loads(q)
-    if not request.get("config") or not request["config"].get("api_key"):
-        return {"error": "Missing Greynoise API key."}
+class GreyNoiseParser:
+    def __init__(self, attribute):
+        self.misp_event = MISPEvent()
+        self.attribute = MISPAttribute()
+        self.attribute.from_dict(**attribute)
+        self.misp_event.add_attribute(**self.attribute)
+        self.ip_address_enrich_mapping = {
+            "noise": {"type": "boolean", "object_relation": "noise"},
+            "riot": {"type": "boolean", "object_relation": "riot"},
+            "classification": {"type": "text", "object_relation": "classification"},
+            "actor": {"type": "text", "object_relation": "actor"},
+            "trust_level": {"type": "text", "object_relation": "trust-level"},
+            "name": {"type": "text", "object_relation": "provider"},
+            "first_seen": {"type": "datetime", "object_relation": "first-seen"},
+            "last_seen": {"type": "datetime", "object_relation": "last-seen"},
+            "link": {"type": "url", "object_relation": "link"},
+            "last_updated": {"type": "datetime", "object_relation": "last-seen"},
+        }
+        self.ip_address_hover_mapping = {
+            "noise": {"type": "boolean", "object_relation": "noise"},
+            "riot": {"type": "boolean", "object_relation": "riot"},
+            "classification": {"type": "text", "object_relation": "classification"},
+            "actor": {"type": "text", "object_relation": "actor"},
+            "tags": {"type": "text", "object_relation": "tags"},
+            "cve": {"type": "text", "object_relation": "cve"},
+            "vpn": {"type": "text", "object_relation": "vpn"},
+            "vpn_service": {"type": "text", "object_relation": "vpn_service"},
+            "bot": {"type": "text", "object_relation": "bot"},
+            "first_seen": {"type": "datetime", "object_relation": "first-seen"},
+            "last_seen": {"type": "datetime", "object_relation": "last-seen"},
+            "spoofable": {"type": "datetime", "object_relation": "spoofable"},
+            "link": {"type": "url", "object_relation": "link"},
+            "category": {"type": "text", "object_relation": "category"},
+            "name": {"type": "text", "object_relation": "provider"},
+            "trust_level": {"type": "text", "object_relation": "trust-level"},
+            "last_updated": {"type": "datetime", "object_relation": "last_updated"},
+        }
+        self.ip_address_metadata_mapping = {
+            "tor": {"type": "text", "object_relation": "tor"},
+            "asn": {"type": "AS", "object_relation": "asn"},
+            "city": {"type": "text", "object_relation": "city"},
+            "country_code": {"type": "text", "object_relation": "country-code"},
+            "country": {"type": "text", "object_relation": "country"},
+            "organization": {"type": "text", "object_relation": "organization"},
+            "destination_country_codes": {"type": "text", "object_relation": "destination-country-codes"},
+            "destination_countries": {"type": "text", "object_relation": "destination-countries"},
+            "category": {"type": "text", "object_relation": "category"},
+            "rdns": {"type": "text", "object_relation": "rdns"},
+        }
+        self.vulnerability_mapping = {
+            "id": {"type": "text", "object_relation": "id"},
+            "details": {"type": "text", "object_relation": "details"},
+            "count": {"type": "text", "object_relation": "total-count"},
+            "benign": {"type": "text", "object_relation": "benign-count"},
+            "malicious": {"type": "text", "object_relation": "malicious-count"},
+            "unknown": {"type": "text", "object_relation": "unknown-count"},
+        }
 
-    headers = {
-        "Accept": "application/json",
-        "key": request["config"]["api_key"],
-        "User-Agent": "greynoise-misp-module-{}".format(moduleinfo["version"]),
-    }
-
-    if not (request.get("vulnerability") or request.get("ip-dst") or request.get("ip-src")):
-        misperrors["error"] = "Vulnerability id missing"
-        return misperrors
-
-    ip = ""
-    vulnerability = ""
-
-    if request.get("ip-dst"):
-        ip = request.get("ip-dst")
-    elif request.get("ip-src"):
-        ip = request.get("ip-src")
-    else:
-        vulnerability = request.get("vulnerability")
-
-    if ip:
-        if request["config"]["api_type"] and request["config"]["api_type"] == "enterprise":
-            greynoise_api_url = "https://api.greynoise.io/v2/noise/quick/"
-        else:
-            greynoise_api_url = "https://api.greynoise.io/v3/community/"
-
-        response = requests.get(f"{greynoise_api_url}{ip}", headers=headers)  # Real request for IP Query
-        if response.status_code == 200:
-            if request["config"]["api_type"] == "enterprise":
-                response = response.json()
-                enterprise_context_object = MISPObject("greynoise-ip-context")
-                for feature in ("ip", "code_message"):
-                    if feature == "code_message":
-                        value = codes_mapping[response.get("code")]
-                    else:
-                        value = response.get(feature)
-                    if value:
-                        attribute_type, relation = enterprise_context_basic_mapping[feature]
-                        enterprise_context_object.add_attribute(relation, **{"type": attribute_type, "value": value})
-                if response["noise"]:
-                    greynoise_api_url = "https://api.greynoise.io/v2/noise/context/"
-                    context_response = requests.get(f"{greynoise_api_url}{ip}", headers=headers)
-                    context_response = context_response.json()
-                    context_response["link"] = "https://www.greynoise.io/viz/ip/" + ip
-                    if "tags" in context_response:
-                        context_response["tags"] = ",".join(context_response["tags"])
-                    if "cve" in context_response:
-                        context_response["cve"] = ",".join(context_response["cve"])
-                    for feature in enterprise_context_advanced_mapping.keys():
-                        value = context_response.get(feature)
-                        if value:
-                            attribute_type, relation = enterprise_context_advanced_mapping[feature]
-                            enterprise_context_object.add_attribute(
-                                relation, **{"type": attribute_type, "value": value}
-                            )
-                    for feature in enterprise_context_advanced_metadata_mapping.keys():
-                        value = context_response["metadata"].get(feature)
-                        if value:
-                            attribute_type, relation = enterprise_context_advanced_metadata_mapping[feature]
-                            enterprise_context_object.add_attribute(
-                                relation, **{"type": attribute_type, "value": value}
-                            )
-
-                if response["riot"]:
-                    greynoise_api_url = "https://api.greynoise.io/v2/riot/"
-                    riot_response = requests.get(f"{greynoise_api_url}{ip}", headers=headers)
-                    riot_response = riot_response.json()
-                    riot_response["link"] = "https://www.greynoise.io/viz/riot/" + ip
-                    for feature in enterprise_riot_mapping.keys():
-                        value = riot_response.get(feature)
-                        if value:
-                            attribute_type, relation = enterprise_riot_mapping[feature]
-                            enterprise_context_object.add_attribute(
-                                relation, **{"type": attribute_type, "value": value}
-                            )
-                misp_event.add_object(enterprise_context_object)
-                event = json.loads(misp_event.to_json())
-                results = {key: event[key] for key in ("Attribute", "Object") if (key in event and event[key])}
-                return {"results": results}
+    def query_greynoise_ip_hover(self, api_key, api_type):
+        if api_type == "enterprise":
+            logger.info(f"Starting hover enrichment for: {self.attribute.value} via GreyNoise ENT API")
+            integration_name = "greynoise-misp-module-{}".format(moduleinfo["version"])
+            session = GreyNoise(api_key=api_key, integration_name=integration_name)
+            quick_response = session.quick(self.attribute.value)
+            if len(quick_response) != 1:
+                misperrors["error"] = "Quick IP lookup returned unexpected response"
+                return misperrors
             else:
-                response = response.json()
-                community_context_object = MISPObject("greynoise-community-ip-context")
-                for feature in community_found_mapping.keys():
-                    value = response.get(feature)
-                    if value:
-                        attribute_type, relation = community_found_mapping[feature]
-                        community_context_object.add_attribute(relation, **{"type": attribute_type, "value": value})
-                misp_event.add_object(community_context_object)
-                event = json.loads(misp_event.to_json())
-                results = {key: event[key] for key in ("Attribute", "Object") if (key in event and event[key])}
-                return {"results": results}
-        if response.status_code == 404 and request["config"]["api_type"] != "enterprise":
-            response = response.json()
-            community_context_object = MISPObject("greynoise-community-ip-context")
-            for feature in community_not_found_mapping.keys():
-                value = response.get(feature)
-                if value:
-                    attribute_type, relation = community_not_found_mapping[feature]
-                    community_context_object.add_attribute(relation, **{"type": attribute_type, "value": value})
-            misp_event.add_object(community_context_object)
-            event = json.loads(misp_event.to_json())
-            results = {key: event[key] for key in ("Attribute", "Object") if (key in event and event[key])}
-            return {"results": results}
+                quick_response = quick_response[0]
+            context_response = session.ip(self.attribute.value)
+            riot_response = session.riot(self.attribute.value)
 
-    if vulnerability:
-        if request["config"]["api_type"] and request["config"]["api_type"] == "enterprise":
-            greynoise_api_url = "https://api.greynoise.io/v2/experimental/gnql/stats"
-            querystring = {"query": f"last_seen:1w cve:{vulnerability}"}
+            if riot_response and "trust_level" in riot_response:
+                if riot_response["trust_level"] == "1":
+                    riot_response["trust_level"] = "1 - Reasonably Ignore"
+                if riot_response["trust_level"] == "2":
+                    riot_response["trust_level"] = "2 - Commonly Seen"
+
+            if context_response and riot_response:
+                response = context_response.copy()
+                response.update(riot_response)
+                response.update(quick_response)
+            elif context_response:
+                response = context_response.copy()
+                response.update(quick_response)
+            elif riot_response:
+                response = riot_response.copy()
+                response.update(quick_response)
+
+            response["link"] = "https://viz.greynoise.io/ip/" + self.attribute.value
+
+            ip_address_attributes = []
+            for feature, mapping in self.ip_address_hover_mapping.items():
+                logger.debug(f"Checking feature {feature}")
+                if response.get(feature):
+                    if feature in ["cve", "tags"]:
+                        response[feature] = ", ".join(response[feature])
+                    if feature == "vpn_service" and response[feature] == "N/A":
+                        continue
+                    if feature == "actor" and response[feature] == "unknown":
+                        continue
+                    attribute = {"value": response[feature]}
+                    logger.debug(f"Adding Feature: {feature}, Attribute: {attribute}")
+                    attribute.update(mapping)
+                    ip_address_attributes.append(attribute)
+            if "metadata" in context_response:
+                for feature, mapping in self.ip_address_metadata_mapping.items():
+                    logger.debug(f"Checking metadata feature {feature}")
+                    if response["metadata"].get(feature):
+                        if feature in ["destination_countries", "destination_country_codes"]:
+                            response["metadata"][feature] = ", ".join(response["metadata"][feature])
+                        attribute = {"value": response["metadata"][feature]}
+                        logger.debug(f"Adding Feature: {feature}, Attribute: {attribute}")
+                        attribute.update(mapping)
+                        ip_address_attributes.append(attribute)
+            if ip_address_attributes:
+                logger.debug("creating greynoise ip object")
+                gn_ip_object = MISPObject("greynoise-ip-details")
+                for attribute in ip_address_attributes:
+                    logger.debug(f"adding attribute {attribute}")
+                    gn_ip_object.add_attribute(**attribute)
+                logger.debug(f"attribute id: {self.attribute.uuid}")
+                gn_ip_object.add_reference(self.attribute.uuid, "describes")
+                self.misp_event.add_object(gn_ip_object)
+        else:
+            logger.info(f"Starting hover enrichment for: {self.attribute.value} via GreyNoise Community API")
+            integration_name = "greynoise-community-misp-module-{}".format(moduleinfo["version"])
+            session = GreyNoise(api_key=api_key, integration_name=integration_name, offering="community")
+            community_response = session.ip(self.attribute.value)
+
+            if "noise" in community_response and community_response["noise"]:
+                community_response["actor"] = community_response["name"]
+                community_response.pop("name")
+
+            ip_address_attributes = []
+            for feature, mapping in self.ip_address_hover_mapping.items():
+                if community_response.get(feature):
+                    if feature == "actor" and community_response[feature] == "unknown":
+                        continue
+                    attribute = {"value": community_response[feature]}
+                    attribute.update(mapping)
+                    ip_address_attributes.append(attribute)
+            if ip_address_attributes:
+                ip_address_object = MISPObject("greynoise-ip-details")
+                for attribute in ip_address_attributes:
+                    ip_address_object.add_attribute(**attribute)
+                ip_address_object.add_reference(self.attribute.uuid, "describes")
+                self.misp_event.add_object(ip_address_object)
+
+    def query_greynoise_ip_expansion(self, api_key, api_type):
+        if api_type == "enterprise":
+            logger.info(f"Starting expansion enrichment for: {self.attribute.value} via GreyNoise ENT API")
+            integration_name = "greynoise-misp-module-{}".format(moduleinfo["version"])
+            session = GreyNoise(api_key=api_key, integration_name=integration_name)
+            quick_response = session.quick(self.attribute.value)
+            if len(quick_response) != 1:
+                misperrors["error"] = "Quick IP lookup returned unexpected response"
+                return misperrors
+            else:
+                quick_response = quick_response[0]
+            context_response = session.ip(self.attribute.value)
+            riot_response = session.riot(self.attribute.value)
+
+            if riot_response and "trust_level" in riot_response:
+                if riot_response["trust_level"] == "1":
+                    riot_response["trust_level"] = "1 - Reasonably Ignore"
+                if riot_response["trust_level"] == "2":
+                    riot_response["trust_level"] = "2 - Commonly Seen"
+
+            if context_response and riot_response:
+                response = context_response.copy()
+                response.update(riot_response)
+                response.update(quick_response)
+            elif context_response:
+                response = context_response.copy()
+                response.update(quick_response)
+            elif riot_response:
+                response = riot_response.copy()
+                response.update(quick_response)
+
+            response["link"] = "https://viz.greynoise.io/ip/" + self.attribute.value
+
+            ip_address_attributes = []
+            for feature, mapping in self.ip_address_enrich_mapping.items():
+                logger.debug(f"Checking feature {feature}")
+                if response.get(feature):
+                    if feature == "actor" and response[feature] == "unknown":
+                        continue
+                    attribute = {"value": response[feature]}
+                    logger.debug(f"Adding Feature: {feature}, Attribute: {attribute}")
+                    attribute.update(mapping)
+                    ip_address_attributes.append(attribute)
+            if ip_address_attributes:
+                logger.debug("creating greynoise ip object")
+                gn_ip_object = MISPObject("greynoise-ip")
+                for attribute in ip_address_attributes:
+                    logger.debug(f"adding attribute {attribute}")
+                    gn_ip_object.add_attribute(**attribute)
+                logger.debug(f"attribute id: {self.attribute.uuid}")
+                gn_ip_object.add_reference(self.attribute.uuid, "describes")
+                self.misp_event.add_object(gn_ip_object)
+        else:
+            logger.info(f"Starting expansion enrichment for: {self.attribute.value} via GreyNoise Community API")
+            integration_name = "greynoise-community-misp-module-{}".format(moduleinfo["version"])
+            session = GreyNoise(api_key=api_key, integration_name=integration_name, offering="community")
+            community_response = session.ip(self.attribute.value)
+
+            if "noise" in community_response and community_response["noise"]:
+                community_response["actor"] = community_response["name"]
+                community_response.pop("name")
+
+            ip_address_attributes = []
+            for feature, mapping in self.ip_address_enrich_mapping.items():
+                if community_response.get(feature):
+                    if feature == "actor" and community_response[feature] == "unknown":
+                        continue
+                    attribute = {"value": community_response[feature]}
+                    attribute.update(mapping)
+                    ip_address_attributes.append(attribute)
+            if ip_address_attributes:
+                ip_address_object = MISPObject("greynoise-ip")
+                for attribute in ip_address_attributes:
+                    ip_address_object.add_attribute(**attribute)
+                ip_address_object.add_reference(self.attribute.uuid, "describes")
+                self.misp_event.add_object(ip_address_object)
+
+    def query_greynoise_vulnerability(self, api_key, api_type):
+        if api_type == "enterprise":
+            logger.info(f"Starting expansion enrichment for: {self.attribute.value} via GreyNoise ENT API")
+            integration_name = "greynoise-misp-module-{}".format(moduleinfo["version"])
+            session = GreyNoise(api_key=api_key, integration_name=integration_name)
+            querystring = f"last_seen:1w cve:{self.attribute.value}"
         else:
             misperrors["error"] = "Vulnerability Not Supported with Community API Key"
             return misperrors
 
-        response = requests.get(f"{greynoise_api_url}", headers=headers, params=querystring)  # Real request
+        response = session.stats(querystring)
 
-        if response.status_code == 200:
-            response = response.json()
-            vulnerability_object = MISPObject("greynoise-vuln-info")
+        if "stats" in response:
             response["details"] = (
                 "The IP count below reflects the number of IPs seen "
                 "by GreyNoise in the last 7 days scanning for this CVE."
             )
-            response["id"] = vulnerability
-            for feature in ("id", "details", "count"):
-                value = response.get(feature)
-                if value:
-                    attribute_type, relation = vulnerability_mapping[feature]
-                    vulnerability_object.add_attribute(relation, **{"type": attribute_type, "value": value})
+            response["id"] = self.attribute.value
             classifications = response["stats"].get("classifications")
             for item in classifications:
                 if item["classification"] == "benign":
                     value = item["count"]
-                    attribute_type, relation = ("text", "Benign Scanner Count")
-                    vulnerability_object.add_attribute(relation, **{"type": attribute_type, "value": value})
+                    response["benign"] = value
                 if item["classification"] == "unknown":
                     value = item["count"]
-                    attribute_type, relation = ("text", "Unknown Scanner Count")
-                    vulnerability_object.add_attribute(relation, **{"type": attribute_type, "value": value})
+                    response["unknown"] = value
                 if item["classification"] == "malicious":
                     value = item["count"]
-                    attribute_type, relation = ("text", "Malicious Scanner Count")
-                    vulnerability_object.add_attribute(relation, **{"type": attribute_type, "value": value})
-            misp_event.add_object(vulnerability_object)
-            event = json.loads(misp_event.to_json())
-            results = {key: event[key] for key in ("Attribute", "Object") if (key in event and event[key])}
-            return {"results": results}
+                    response["malicious"] = value
+            vulnerability_attributes = []
+            for feature, mapping in self.vulnerability_mapping.items():
+                if response.get(feature):
+                    attribute = {"value": response[feature]}
+                    attribute.update(mapping)
+                    vulnerability_attributes.append(attribute)
+            if vulnerability_attributes:
+                vulnerability_object = MISPObject("greynoise-vuln-info")
+                for attribute in vulnerability_attributes:
+                    vulnerability_object.add_attribute(**attribute)
+                vulnerability_object.add_reference(self.attribute.uuid, "describes")
+                self.misp_event.add_object(vulnerability_object)
 
-    # There is an error
-    errors = {
-        400: "Bad request.",
-        404: "IP not observed scanning the internet or contained in RIOT data set.",
-        401: "Unauthorized. Please check your API key.",
-        429: "Too many requests. You've hit the rate-limit.",
-    }
-    try:
-        misperrors["error"] = errors[response.status_code]
-    except KeyError:
-        misperrors["error"] = f"GreyNoise API not accessible (HTTP {response.status_code})"
-    return misperrors
+    def get_result(self):
+        event = json.loads(self.misp_event.to_json())
+        results = {key: event[key] for key in ("Attribute", "Object") if (key in event and event[key])}
+        return {"results": results}
+
+
+def handler(q=False):
+    if q is False:
+        return False
+    request = json.loads(q)
+    if not request.get("config", {}).get("api_key"):
+        return {"error": "GreyNoise API Key required, but missing"}
+    if not request.get("config", {}).get("api_type"):
+        return {"error": "GreyNoise API type of enterprise or community required, but missing"}
+    if not request.get("attribute") or not check_input_attribute(request["attribute"]):
+        return {"error": f"{standard_error_message}, which should contain at least a type, a value and an uuid."}
+    attribute = request["attribute"]
+    if attribute["type"] not in mispattributes["input"]:
+        return {"error": "Unsupported attribute type."}
+    greynoise_parser = GreyNoiseParser(attribute)
+
+    if attribute["type"] in ["ip-dst", "ip-src"]:
+        try:
+            ipaddress.IPv4Address(attribute["value"])
+            if "persistent" in request:
+                greynoise_parser.query_greynoise_ip_hover(request["config"]["api_key"], request["config"]["api_type"])
+            else:
+                greynoise_parser.query_greynoise_ip_expansion(request["config"]["api_key"], request["config"]["api_type"])
+        except ValueError:
+            return {"error": "Not a valid IPv4 address"}
+
+    if attribute["type"] == "vulnerability":
+        greynoise_parser.query_greynoise_vulnerability(request["config"]["api_key"], request["config"]["api_type"])
+
+    return greynoise_parser.get_result()
 
 
 def introspection():

misp_modules/modules/expansion/ipinfo.py

@@ -0,0 +1,105 @@
+import json
+import requests
+from . import check_input_attribute, standard_error_message
+from pymisp import MISPAttribute, MISPEvent, MISPObject
+
+mispattributes = {
+    'input': ['ip-src', 'ip-dst'],
+    'format': 'misp_standard'
+}
+moduleinfo = {
+    'version': 1,
+    'author': 'Christian Studer',
+    'description': 'An expansion module to query ipinfo.io for additional information on an IP address',
+    'module-type': ['expansion', 'hover']
+}
+moduleconfig = ['token']
+
+_GEOLOCATION_OBJECT_MAPPING = {
+    'city': 'city',
+    'postal': 'zipcode',
+    'region': 'region',
+    'country': 'countrycode'
+}
+
+
+def handler(q=False):
+    # Input checks
+    if q is False:
+        return False
+    request = json.loads(q)
+    if not request.get('attribute') or not check_input_attribute(request['attribute']):
+        return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'}
+    attribute = request['attribute']
+    if attribute.get('type') not in mispattributes['input']:
+        return {'error': 'Wrong input attribute type.'}
+    if not request.get('config'):
+        return {'error': 'Missing ipinfo config.'}
+    if not request['config'].get('token'):
+        return {'error': 'Missing ipinfo token.'}
+
+    # Query ipinfo.io
+    query = requests.get(
+        f"https://ipinfo.io/{attribute['value']}/json?token={request['config']['token']}"
+    )
+    if query.status_code != 200:
+        return {'error': f'Error while querying ipinfo.io - {query.status_code}: {query.reason}'}
+    ipinfo = query.json()
+
+    # Check if the IP address is not reserved for special use
+    if ipinfo.get('bogon', False):
+        return {'error': 'The IP address is reserved for special use'}
+
+    # Initiate the MISP data structures
+    misp_event = MISPEvent()
+    input_attribute = MISPAttribute()
+    input_attribute.from_dict(**attribute)
+    misp_event.add_attribute(**input_attribute)
+
+    # Parse the geolocation information related to the IP address
+    geolocation = MISPObject('geolocation')
+    for field, relation in _GEOLOCATION_OBJECT_MAPPING.items():
+        geolocation.add_attribute(relation, ipinfo[field])
+    for relation, value in zip(('latitude', 'longitude'), ipinfo['loc'].split(',')):
+        geolocation.add_attribute(relation, value)
+    geolocation.add_reference(input_attribute.uuid, 'locates')
+    misp_event.add_object(geolocation)
+
+    # Parse the domain information
+    domain_ip = misp_event.add_object(name='domain-ip')
+    for feature in ('hostname', 'ip'):
+        domain_ip.add_attribute(feature, ipinfo[feature])
+    domain_ip.add_reference(input_attribute.uuid, 'resolves')
+    if ipinfo.get('domain') is not None:
+        for domain in ipinfo['domain']['domains']:
+            domain_ip.add_attribute('domain', domain)
+
+    # Parse the AS information
+    asn = MISPObject('asn')
+    asn.add_reference(input_attribute.uuid, 'includes')
+    if ipinfo.get('asn') is not None:
+        asn_info = ipinfo['asn']
+        asn.add_attribute('asn', asn_info['asn'])
+        asn.add_attribute('description', asn_info['name'])
+        misp_event.add_object(asn)
+    elif ipinfo.get('org'):
+        as_value, *description = ipinfo['org'].split(' ')
+        asn.add_attribute('asn', as_value)
+        asn.add_attribute('description', ' '.join(description))
+        misp_event.add_object(asn)
+    
+
+    # Return the results in MISP format
+    event = json.loads(misp_event.to_json())
+    return {
+        'results': {key: event[key] for key in ('Attribute', 'Object')}
+    }
+
+
+def introspection():
+    return mispattributes
+
+
+def version():
+    moduleinfo['config'] = moduleconfig
+    return moduleinfo

misp_modules/modules/export_mod/defender_endpoint_export.py

@@ -8,7 +8,7 @@ import json
 
 misperrors = {"error": "Error"}
 
-types_to_use = ['sha1', 'md5', 'domain', 'ip', 'url']
+types_to_use = ['sha256', 'sha1', 'md5', 'domain', 'ip', 'url']
 
 userConfig = {
 
@@ -20,11 +20,17 @@ inputSource = ['event']
 outputFileExtension = 'kql'
 responseType = 'application/txt'
 
-moduleinfo = {'version': '1.0', 'author': 'Julien Bachmann, Hacknowledge',
+moduleinfo = {'version': '1.1', 'author': 'Julien Bachmann, Hacknowledge, Maik Wuerth',
               'description': 'Defender for Endpoint KQL hunting query export module',
               'module-type': ['export']}
 
 
+def handle_sha256(value, period):
+    query = f"""find in (DeviceAlertEvents, DeviceFileEvents, DeviceImageLoadEvents, DeviceProcessEvents)
+        where SHA256 == '{value}' or InitiatingProcessSHA1 == '{value}'"""
+    return query.replace('\n', ' ')
+
+
 def handle_sha1(value, period):
     query = f"""find in (DeviceAlertEvents, DeviceFileEvents, DeviceImageLoadEvents, DeviceProcessEvents)
         where SHA1 == '{value}' or InitiatingProcessSHA1 == '{value}'"""
@@ -56,6 +62,7 @@ def handle_url(value, period):
 
 
 handlers = {
+    'sha256': handle_sha256,
     'sha1': handle_sha1,
     'md5': handle_md5,
     'domain': handle_domain,
@@ -75,6 +82,10 @@ def handler(q=False):
         for attribute in event["Attribute"]:
             if attribute['type'] in types_to_use:
                 output = output + handlers[attribute['type']](attribute['value'], config['Period']) + '\n'
+        for obj in event["Object"]:
+            for attribute in obj["Attribute"]:
+                if attribute['type'] in types_to_use:
+                    output = output + handlers[attribute['type']](attribute['value'], config['Period']) + '\n'
     r = {"response": [], "data": str(base64.b64encode(bytes(output, 'utf-8')), 'utf-8')}
     return r
 

misp_modules/modules/import_mod/__init__.py

@@ -15,5 +15,6 @@ __all__ = [
     'csvimport',
     'cof2misp',
     'joe_import',
-    'taxii21'
+    'taxii21',
+    'url_import'
 ]

misp_modules/modules/import_mod/import_blueprint.py

@@ -0,0 +1,84 @@
+import json
+import base64
+from pymisp import MISPEvent, MISPObject, MISPAttribute
+
+misperrors = {'error': 'Error'}
+userConfig = {
+    'number1': {
+        'type': 'Integer',
+        'regex': '/^[0-4]$/i',
+        'errorMessage': 'Expected a number in range [0-4]',
+        'message': 'Column number used for value'
+    },
+    'some_string': {
+        'type': 'String',
+        'message': 'A text field'
+    },
+    'boolean_field': {
+        'type': 'Boolean',
+        'message': 'Boolean field test'
+    },
+    'comment': {
+        'type': 'Integer',
+        'message': 'Column number used for comment'
+    }
+}
+
+mispattributes = {
+    'inputSource': ['file', 'paste'],
+    'output': ['MISP Format'],
+    'format': 'misp_standard'
+}
+
+
+moduleinfo = {'version': '0.1', 'author': 'Sami Mokaddem',
+              'description': 'Generic blueprint to be copy-pasted to quickly boostrap creation of import module.',
+              'module-type': ['import']}
+
+moduleconfig = []
+
+
+
+def generateData(event, data, config):
+    # attr = MISPAttribute()
+    # attr.from_dict(**{
+    #     'type': 'ip-src',
+    #     'value': '8.8.8.8',
+    #     'distribution': 2
+    # })
+    # event.add_attribute(attr)
+    pass
+
+
+def handler(q=False):
+    if q is False:
+        return False
+    request = json.loads(q)
+    data = getUploadedData(request)
+    config = getPassedConfig(request)
+    event = MISPEvent()
+    generateData(event, data, config)
+    return {"results": json.loads(event.to_json())}
+
+
+def getUploadedData(request):
+    return base64.b64decode(request['data']).decode('utf8')
+
+
+def getPassedConfig(request):
+    return request['config']
+
+
+def introspection():
+    modulesetup = mispattributes
+    try:
+        userConfig
+        modulesetup['userConfig'] = userConfig
+    except NameError:
+        pass
+    return modulesetup
+
+
+def version():
+    moduleinfo['config'] = moduleconfig
+    return moduleinfo

misp_modules/modules/import_mod/url_import.py

@@ -0,0 +1,86 @@
+import json
+import base64
+from pymisp import MISPEvent, MISPObject, MISPAttribute
+from pyfaup.faup import Faup
+
+misperrors = {'error': 'Error'}
+userConfig = {
+    'include_scheme': {
+        'type': 'Boolean',
+        'message': 'Include scheme'
+    },
+}
+
+mispattributes = {
+    'inputSource': ['file', 'paste'],
+    'output': ['MISP Format'],
+    'format': 'misp_standard'
+}
+
+
+moduleinfo = {'version': '0.1', 'author': 'Sami Mokaddem',
+              'description': 'Generic blueprint to be copy-pasted to quickly boostrap creation of import module.',
+              'module-type': ['import']}
+
+moduleconfig = []
+
+fp = Faup()
+
+def generateData(event, data, config):
+    for url in data.splitlines():
+        fp.decode(url)
+        parsed = fp.get()
+        obj = MISPObject('url')
+        obj.add_attribute('url', type='url', value=url)
+        if parsed['tld'] is not None:
+            obj.add_attribute('tld', type='text', value=parsed['tld'])
+        if parsed['subdomain'] is not None:
+            obj.add_attribute('subdomain', type='text', value=parsed['subdomain'])
+        if config['include_scheme'] is True:
+            obj.add_attribute('scheme', type='text', value=parsed['scheme'])
+        obj.add_attribute('resource_path', type='text', value=parsed['resource_path'])
+        obj.add_attribute('query_string', type='text', value=parsed['query_string'])
+        obj.add_attribute('port', type='port', value=parsed['port'])
+        obj.add_attribute('host', type='hostname', value=parsed['host'])
+        if parsed['fragment'] is not None:
+            obj.add_attribute('fragment', type='text', value=parsed['fragment'])
+        obj.add_attribute('domain_without_tld', type='text', value=parsed['domain_without_tld'])
+        obj.add_attribute('domain', type='domain', value=parsed['domain'])
+        event.objects.append(obj)
+
+
+def handler(q=False):
+    if q is False:
+        return False
+    request = json.loads(q)
+    data = getUploadedData(request)
+    config = getPassedConfig(request)
+    event = MISPEvent()
+    generateData(event, data, config)
+    return {"results": json.loads(event.to_json())}
+
+
+def getUploadedData(request):
+    return base64.b64decode(request['data']).decode('utf8')
+
+
+def getPassedConfig(request):
+    for k, v in userConfig.items():
+        if v['type'] == 'Boolean':
+            request['config'][k] = True if request['config'][k] == '1' else False
+    return request['config']
+
+
+def introspection():
+    modulesetup = mispattributes
+    try:
+        userConfig
+        modulesetup['userConfig'] = userConfig
+    except NameError:
+        pass
+    return modulesetup
+
+
+def version():
+    moduleinfo['config'] = moduleconfig
+    return moduleinfo

mkdocs.yml

@@ -11,12 +11,12 @@ site_url: https://www.misp-project.org/
 # Repository
 repo_name: 'MISP/misp-modules'
 repo_url: https://github.com/MISP/misp-modules/
-edit_uri: ""
+edit_uri: edit/main/docs/
 
 use_directory_urls: true
 
 # Copyright
-copyright: "Copyright © 2019-2022 MISP Project"
+copyright: "Copyright © 2019-2023 MISP Project"
 
 # Options
 extra:

tests/test_expansions.py

@@ -195,7 +195,7 @@ class TestExpansions(unittest.TestCase):
         query = {"module": "dbl_spamhaus", "domain": "totalmateria.net"}
         response = self.misp_modules_post(query)
         try:
-            self.assertEqual(self.get_values(response), 'totalmateria.net - spam domain')
+            self.assertEqual(self.get_values(response), 'totalmateria.net - spam test domain')
         except Exception:
             try:
                 self.assertTrue(self.get_values(response).startswith('None of DNS query names exist:'))
@@ -263,7 +263,7 @@ class TestExpansions(unittest.TestCase):
             self.assertEqual(to_check, 'OK (Not Found)', response)
         else:
             self.assertEqual(self.get_errors(response), 'Have I Been Pwned authentication is incomplete (no API key)')
-            
+
     def test_hyasinsight(self):
         module_name = "hyasinsight"
         query = {"module": module_name,
@@ -297,7 +297,7 @@ class TestExpansions(unittest.TestCase):
                 )
         else:
             response = self.misp_modules_post(query)
-            self.assertEqual(self.get_errors(response), 'Missing Greynoise API key.')
+            self.assertEqual(self.get_errors(response), 'GreyNoise API Key required, but missing')
 
     @unittest.skip("Service doesn't work")
     def test_ipasn(self):
@@ -432,7 +432,7 @@ class TestExpansions(unittest.TestCase):
             encoded = b64encode(f.read()).decode()
         query = {"module": "pdf_enrich", "attachment": filename, "data": encoded}
         response = self.misp_modules_post(query)
-        self.assertEqual(self.get_values(response), 'Pdf test')
+        self.assertRegex(self.get_values(response), r'^Pdf test')
 
     def test_pptx(self):
         filename = 'test.pptx'