MISP
/
misp-modules
mirror of https://github.com/MISP/misp-modules
2
0
Fork 0
Code Issues Releases Wiki Activity

Migrate to poetry and optimize dependencies

pull/681/head
Stefano Ortolani 2024-08-12 10:43:25 +01:00
parent 3a13af724f
commit b89a33b74f
35 changed files with 6531 additions and 4969 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
.github/workflows/python-package.yml vendored
View File

@ -19,50 +19,55 @@ jobs:
- name: Install packages
run: |
sudo apt-get install libpoppler-cpp-dev libzbar0 tesseract-ocr
- name: Compile and install yara
run: |
sudo apt-get install automake libtool make gcc pkg-config
wget https://github.com/VirusTotal/yara/archive/refs/tags/v4.5.0.tar.gz
tar xzf *.tar.gz
wget https://github.com/VirusTotal/yara/archive/refs/tags/v4.5.0.tar.gz -O yara.tar.gz
tar xzf yara.tar.gz
pushd yara-*
./bootstrap.sh
./configure
make
sudo make install
popd
- uses: actions/checkout@v4
- name: Checkout repository
uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
cache: 'pip'
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install flake8 pytest
pip install pipenv
sed -i "s/python_version.*/python_version = \"${{ matrix.python-version }}\"/" Pipfile
pipenv lock
pipenv requirements > requirements.txt
# pyfaul must be installed manually (?)
pip install -r requirements.txt pyfaup
pip install .
python -m pip install --upgrade poetry
poetry install
# Tell poetry to not use a virtual environment
# poetry config virtualenvs.create false
- name: Lint with flake8
run: |
# stop the build if there are Python syntax errors or undefined names
flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
poetry run flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
poetry run flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
- name: Run server in background
run: |
misp-modules -l 127.0.0.1 -s 2>error.log &
sleep 3
poetry run misp-modules -l 127.0.0.1 -s 2>error.log &
sleep 10
- name: Check if server is running
run: |
curl -sS localhost:6666/healthcheck
- name: Test with pytest
run: |
pytest tests
poetry run pytest
- name: Show error log
if: always()
run: |

1
.gitignore vendored
View File

@ -24,3 +24,4 @@ venv*
.vscode*
*.sqlite
website/conf/config.cfg
wheels

56
.travis.yml
View File

@ -1,56 +0,0 @@
language: python
services:
- redis-server
cache: pip
python:
- "3.6"
- "3.6-dev"
- "3.7-dev"
- "3.8-dev"
install:
- sudo apt-get install libzbar0 libzbar-dev libpoppler-cpp-dev tesseract-ocr libfuzzy-dev libcaca-dev liblua5.3-dev
- pip install pipenv
- pip install -r REQUIREMENTS
# - pipenv install --dev
# install gtcaca
- git clone git://github.com/stricaud/gtcaca.git
- mkdir -p gtcaca/build
- pushd gtcaca/build
- cmake .. && make
- sudo make install
- popd
# install pyfaup
- git clone https://github.com/stricaud/faup.git
- pushd faup/build
- cmake .. && make
- sudo make install
- popd
- sudo ldconfig
- pushd faup/src/lib/bindings/python
- pip install .
- popd
script:
- pip install coverage
- coverage run -m --parallel-mode --source=misp_modules misp_modules.__init__ -l 127.0.0.1 &
- pid=$!
- sleep 5
- nosetests --with-coverage --cover-package=misp_modules
- kill -s KILL $pid
- pushd ~/
- coverage run -m --parallel-mode --source=misp_modules misp_modules.__init__ -s -l 127.0.0.1 &
- pid=$!
- popd
- sleep 5
- nosetests --with-coverage --cover-package=misp_modules
- kill -s KILL $pid
- pip install flake8
- flake8 --ignore=E501,W503,E226,E126 misp_modules
after_success:
- coverage combine .coverage*
- codecov

3
DOC-REQUIREMENTS
View File

@ -1,3 +0,0 @@
mkdocs
pymdown-extensions
mkdocs-material

81
Makefile
View File

@ -1,38 +1,61 @@
# https://www.mkdocs.org/user-guide/deploying-your-docs/
# See: https://www.mkdocs.org/user-guide/deploying-your-docs/
# Running 'make' uses poetry-installed mkdocs
# Running 'USE_DOCKER=true make' uses docker mkdocs
.PHONY: prepare_docs generate_docs deploy test_docs
MKDOCS_DOCKER_IMAGE := squidfunk/mkdocs-material
DOCS_DIST_DIR := ./docs
DOCS_SRC_DIR := ./documentation
USE_DOCKER ?=
.DEFAULT_GOAL := generate_docs
.PHONY: prepare_docs generate_docs ci_generate_docs test_docs
prepare_docs:
pip3 install .
cd documentation; python3 generate_documentation.py
mkdir -p docs/expansion/logos docs/export_mod/logos docs/import_mod/logos
mkdir -p docs/logos
cd documentation; cp -R ./logos/* ../docs/logos
cd documentation; cp -R ./logos/* ../docs/expansion/logos
cd documentation; cp -R ./logos/* ../docs/export_mod/logos
cd documentation; cp -R ./logos/* ../docs/import_mod/logos
cp ./documentation/mkdocs/*.md ./docs
cp LICENSE ./docs/license.md
@echo "Preparing documentation."
poetry install --only docs
poetry run python $(DOCS_SRC_DIR)/generate_documentation.py
mkdir -p $(DOCS_DIST_DIR)/logos
mkdir -p $(DOCS_DIST_DIR)/expansion/logos
mkdir -p $(DOCS_DIST_DIR)/export_mod/logos
mkdir -p $(DOCS_DIST_DIR)/import_mod/logos
cp -R $(DOCS_SRC_DIR)/logos/* $(DOCS_DIST_DIR)/logos
cp -R $(DOCS_SRC_DIR)/logos/* $(DOCS_DIST_DIR)/expansion/logos
cp -R $(DOCS_SRC_DIR)/logos/* $(DOCS_DIST_DIR)/export_mod/logos
cp -R $(DOCS_SRC_DIR)/logos/* $(DOCS_DIST_DIR)/import_mod/logos
cp $(DOCS_SRC_DIR)/mkdocs/*.md $(DOCS_DIST_DIR)
cp LICENSE $(DOCS_DIST_DIR)/license.md
install_requirements:
pip install -r docs/REQUIREMENTS.txt
generate_docs: prepare_docs
mkdocs build
ifeq ($(USE_DOCKER), true)
@echo "Generating documentation using '$(MKDOCS_DOCKER_IMAGE)'."
docker run --rm -it -v $(PWD):/docs $(MKDOCS_DOCKER_IMAGE) build
else
@echo "Generating docunentation."
poetry run mkdocs build
endif
deploy: generate_docs
ifeq ($(USE_DOCKER), true)
@echo "Deploying documentation using '$(MKDOCS_DOCKER_IMAGE)'."
docker run --rm -it -v $(PWD):/docs -v /home/$(whoami)/.docker:/root/.docker:ro $(MKDOCS_DOCKER_IMAGE) gh-deploy
else
@echo "Deploying docunentation."
poetry run mkdocs gh-deploy
endif
deploy:
mkdocs gh-deploy
test_docs: prepare_docs
mkdocs serve
# DOCKER make commands
generate_docs_docker: prepare_docs
docker run --rm -it -v $(PWD):/docs squidfunk/mkdocs-material build
deploy_docker:
docker run --rm -it -v $(PWD):/docs -v /home/$(whoami)/.docker:/root/.docker:ro squidfunk/mkdocs-material gh-deploy
test_docs_docker: prepare_docs
docker run --rm -it -p 8000:8000 -v $(PWD):/docs squidfunk/mkdocs-material
ifeq ($(USE_DOCKER), true)
@echo "Serving documentation using '$(MKDOCS_DOCKER_IMAGE)'."
docker run --rm -it -v $(PWD):/docs -p 8000:8000 $(MKDOCS_DOCKER_IMAGE)
else
@echo "Serving docunentation."
poetry run mkdocs serve
endif

87
Pipfile
View File

@ -1,87 +0,0 @@
[[source]]
name = "pypi"
url = "https://pypi.org/simple"
verify_ssl = true
[dev-packages]
nose = "*"
codecov = "*"
pytest = "*"
flake8 = "*"
[packages]
numpy = "<2.0.0"
matplotlib = "*"
sigmf = "*"
pysafebrowsing = "*"
dnspython = "*"
requests = { extras = ["security"], version = "*" }
urlarchiver = "*"
passivetotal = "*"
pypdns = "*"
pypssl = "*"
pyeupi = "*"
pymisp = { extras = ["fileobjects,openioc,pdfexport,email,url"], version = "*" }
pyonyphe = { git = "https://github.com/sebdraven/pyonyphe" }
pydnstrails = { git = "https://github.com/sebdraven/pydnstrails" }
pytesseract = "*"
pygeoip = "*"
beautifulsoup4 = "*"
oauth2 = "*"
yara-python = "==4.5.0"
sigmatools = "*"
stix2 = "*"
stix2-patterns = "*"
taxii2-client = "*"
maclookup = "*"
vulners = "*"
blockchain = "*"
reportlab = "*"
pyintel471 = { git = "https://github.com/MISP/PyIntel471.git" }
shodan = "*"
Pillow = ">=8.2.0"
Wand = "*"
SPARQLWrapper = "*"
domaintools_api = "*"
misp-modules = { path = "." }
pybgpranking = { git = "https://github.com/D4-project/BGP-Ranking.git/", subdirectory = "client", ref = "68de39f6c5196f796055c1ac34504054d688aa59" }
pyipasnhistory = { git = "https://github.com/D4-project/IPASN-History.git/", subdirectory = "client", ref = "a2853c39265cecdd0c0d16850bd34621c0551b87" }
backscatter = "*"
pyzbar = "*"
opencv-python = "*"
np = "*"
ODTReader = { git = "https://github.com/cartertemm/ODTReader.git/" }
python-pptx = "*"
python-docx = "*"
ezodf = "*"
pandas = "*"
pandas_ods_reader = "*"
pdftotext = "*"
lxml = "*"
xlrd = "*"
jbxapi = "*"
geoip2 = "*"
apiosintDS = "*"
assemblyline_client = "*"
vt-graph-api = "*"
trustar = { git = "https://github.com/SteveClement/trustar-python.git" }
markdownify = "==0.5.3"
socialscan = "==1.4"
pycountry = "==22.3.5"
dnsdb2 = "*"
clamd = "*"
aiohttp = ">=3.7.4"
tau-clients = "*"
vt-py = ">=0.7.1"
crowdstrike-falconpy = "0.9.0"
censys = "2.0.9"
mwdblib = "3.4.1"
ndjson = "0.3.1"
Jinja2 = ">=3.1.2"
mattermostdriver = "7.3.2"
openpyxl = "*"
slack-sdk = "3.27.1"
vysion = "*"
[requires]
python_version = "3.12"

3645
Pipfile.lock generated
View File

File diff suppressed because it is too large Load Diff

2
README.md
View File

@ -42,7 +42,6 @@ For further Information see the [license file](https://misp.github.io/misp-modul
* [AssemblyLine Query](https://misp.github.io/misp-modules/expansion/#assemblyline-query) - A module tu query the AssemblyLine API with a submission ID to get the submission report and parse it.
* [AssemblyLine Submit](https://misp.github.io/misp-modules/expansion/#assemblyline-submit) - A module to submit samples and URLs to AssemblyLine for advanced analysis, and return the link of the submission.
* [Backscatter.io](https://misp.github.io/misp-modules/expansion/#backscatter.io) - Backscatter.io module to bring mass-scanning observations into MISP.
* [BGP Ranking](https://misp.github.io/misp-modules/expansion/#bgp-ranking) - Query BGP Ranking to get the ranking of an Autonomous System number.
* [BTC Scam Check](https://misp.github.io/misp-modules/expansion/#btc-scam-check) - An expansion hover module to query a special dns blacklist to check if a bitcoin address has been abused.
* [BTC Steroids](https://misp.github.io/misp-modules/expansion/#btc-steroids) - An expansion hover module to get a blockchain balance from a BTC address in MISP.
* [Censys Enrich](https://misp.github.io/misp-modules/expansion/#censys-enrich) - An expansion module to enrich attributes in MISP by quering the censys.io API
@ -136,7 +135,6 @@ For further Information see the [license file](https://misp.github.io/misp-modul
* [Vulnerability Lookup](https://misp.github.io/misp-modules/expansion/#vulnerability-lookup) - An expansion module to query Vulnerability Lookup
* [Vulners Lookup](https://misp.github.io/misp-modules/expansion/#vulners-lookup) - An expansion hover module to expand information about CVE id using Vulners API.
* [Vysion Enrich](https://misp.github.io/misp-modules/expansion/#vysion-enrich) - Module to enrich the information by making use of the Vysion API.
* [Whois Lookup](https://misp.github.io/misp-modules/expansion/#whois-lookup) - Module to query a local instance of uwhois (https://github.com/rafiot/uwhoisd).
* [WhoisFreaks Lookup](https://misp.github.io/misp-modules/expansion/#whoisfreaks-lookup) - An expansion module for https://whoisfreaks.com/ that will provide an enriched analysis of the provided domain, including WHOIS and DNS information.
* [Wikidata Lookup](https://misp.github.io/misp-modules/expansion/#wikidata-lookup) - An expansion hover module to extract information from Wikidata to have additional information about particular term for analysis.
* [IBM X-Force Exchange Lookup](https://misp.github.io/misp-modules/expansion/#ibm-x-force-exchange-lookup) - An expansion module for IBM X-Force Exchange.

188
REQUIREMENTS
View File

@ -1,188 +0,0 @@
-i https://pypi.org/simple
aiohappyeyeballs==2.3.5; python_version >= '3.8'
aiohttp==3.10.2; python_version >= '3.8'
aiosignal==1.3.1; python_version >= '3.7'
annotated-types==0.7.0; python_version >= '3.8'
antlr4-python3-runtime==4.9.3
anyio==4.4.0; python_version >= '3.8'
apiosintds==2.0.3; python_version >= '3.6'
appdirs==1.4.4
assemblyline-client==4.9.3
attrs==24.2.0; python_version >= '3.7'
backoff==1.11.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
backscatter==0.2.4
beautifulsoup4==4.12.3; python_full_version >= '3.6.0'
bidict==0.23.1; python_version >= '3.8'
blockchain==1.4.4
cattrs==23.2.3; python_version >= '3.8'
censys==2.0.9; python_version < '4.0' and python_full_version >= '3.6.2'
certifi==2024.7.4; python_version >= '3.6'
cffi==1.17.0; platform_python_implementation != 'PyPy'
chardet==5.2.0; python_version >= '3.7'
charset-normalizer==3.3.2; python_full_version >= '3.7.0'
clamd==1.0.2
click==8.1.7; python_version >= '3.7'
click-plugins==1.1.1
colorama==0.4.6; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6'
colorclass==2.2.2; python_version >= '2.6'
commonmark==0.9.1
compressed-rtf==1.0.6
configparser==7.0.0; python_version >= '3.8'
contourpy==1.2.1; python_version >= '3.9'
crowdstrike-falconpy==0.9.0; python_version >= '3.6'
cryptography==43.0.0; python_version >= '3.7'
cycler==0.12.1; python_version >= '3.8'
deprecated==1.2.14; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
dnsdb2==1.1.4
dnspython==2.6.1; python_version >= '3.8'
domaintools-api==2.0.0; python_version >= '3.6'
easygui==0.98.3
ebcdic==1.1.1
enum-compat==0.0.3
et-xmlfile==1.1.0; python_version >= '3.6'
extract-msg==0.48.7
ezodf==0.3.2
filelock==3.15.4; python_version >= '3.8'
fonttools==4.53.1; python_version >= '3.8'
frozenlist==1.4.1; python_version >= '3.8'
future==1.0.0; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
geoip2==4.8.0; python_version >= '3.8'
h11==0.14.0; python_version >= '3.7'
httpcore==1.0.5; python_version >= '3.8'
httplib2==0.22.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
httpx==0.27.0; python_version >= '3.8'
idna==3.7; python_version >= '3.5'
isodate==0.6.1
jbxapi==3.23.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
jinja2==3.1.4; python_version >= '3.7'
json-log-formatter==1.0; python_version >= '3.6'
jsonschema==4.23.0; python_version >= '3.8'
jsonschema-specifications==2023.12.1; python_version >= '3.8'
kiwisolver==1.4.5; python_version >= '3.7'
lark==1.1.9; python_version >= '3.6'
lief==0.15.1
lxml==5.2.2; python_version >= '3.6'
maclookup==1.0.3
markdownify==0.5.3
markupsafe==2.1.5; python_version >= '3.7'
matplotlib==3.9.1.post1; python_version >= '3.9'
mattermostdriver==7.3.2; python_version >= '3.5'
maxminddb==2.6.2; python_version >= '3.8'
-e .
more-itertools==10.4.0; python_version >= '3.8'
msoffcrypto-tool==5.4.2; python_version >= '3' and platform_python_implementation != 'PyPy' or (platform_system != 'Windows' and platform_system != 'Darwin')
multidict==6.0.5; python_version >= '3.7'
mwdblib==3.4.1
ndjson==0.3.1
np==1.0.2
numpy==1.26.4; python_version >= '3.9'
oauth2==1.9.0.post1
odtreader@ git+https://github.com/cartertemm/ODTReader.git/@49d6938693f6faa3ff09998f86dba551ae3a996b
olefile==0.47; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
oletools==0.60.2
opencv-python==4.10.0.84; python_version >= '3.6'
openpyxl==3.1.5; python_version >= '3.8'
packaging==24.1; python_version >= '3.8'
pandas==2.2.2; python_version >= '3.9'
pandas-ods-reader==1.0.1; python_version < '3.13' and python_version >= '3.9'
passivetotal==2.5.9
pcodedmp==1.2.6
pdftotext==2.2.2
pillow==10.4.0; python_version >= '3.8'
platformdirs==4.2.2; python_version >= '3.8'
progressbar2==4.4.2; python_version >= '3.8'
psutil==6.0.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
publicsuffixlist==1.0.2.20240810; python_version >= '3.5'
pybgpranking@ git+https://github.com/D4-project/BGP-Ranking.git/#subdirectory=client
pycountry==22.3.5; python_version >= '3.6' and python_version < '4'
pycparser==2.22; python_version >= '3.8'
pycryptodome==3.20.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
pycryptodomex==3.20.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
pydantic==2.8.2; python_version >= '3.8'
pydantic-core==2.20.1; python_version >= '3.8'
pydeep2==0.5.1
pydnstrails@ git+https://github.com/sebdraven/pydnstrails@48c1f740025c51289f43a24863d1845ff12fd21a
pyeupi==1.3.0; python_version >= '3.8' and python_version < '4.0'
pyfaup==1.2
pygeoip==0.3.2
pygments==2.18.0; python_version >= '3.8'
pyintel471@ git+https://github.com/MISP/PyIntel471.git@917272fafa8e12102329faca52173e90c5256968
pyipasnhistory@ git+https://github.com/D4-project/IPASN-History.git/#subdirectory=client
pymisp[fileobjects,openioc,pdfexport,email,url]==2.4.195; python_version >= '3.8' and python_version < '4.0'
pyonyphe@ git+https://github.com/sebdraven/pyonyphe@d1d6741f8ea4475f3bb77ff20c876f08839cabd1
pyparsing==2.4.7; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
pypdns==2.2.3; python_version >= '3.8' and python_version < '4.0'
pypssl==2.2; python_version >= '3.6' and python_version < '4.0'
pysafebrowsing==0.1.3
pytesseract==0.3.10; python_version >= '3.7'
python-baseconv==1.2.2
python-dateutil==2.9.0.post0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
python-docx==1.1.2; python_version >= '3.7'
python-engineio==4.9.1; python_version >= '3.6'
python-magic==0.4.27
python-pptx==1.0.2; python_version >= '3.8'
python-socketio[client]==5.11.3; python_version >= '3.8'
python-utils==3.8.2; python_version >= '3.9'
pytz==2024.1
pyyaml==6.0.2; python_version >= '3.8'
pyzbar==0.1.9
pyzipper==0.3.6; python_version >= '3.5'
rdflib==7.0.0; python_full_version >= '3.8.1' and python_full_version < '4.0.0'
red-black-tree-mod==1.20
redis==5.0.8; python_version >= '3.7'
referencing==0.35.1; python_version >= '3.8'
reportlab==4.2.2; python_version >= '3.7' and python_version < '4'
requests[security]==2.32.3; python_version >= '3.8'
requests-cache==1.2.1; python_version >= '3.8'
requests-file==2.1.0
rich==10.16.2; python_full_version >= '3.6.2' and python_full_version < '4.0.0'
rpds-py==0.20.0; python_version >= '3.8'
rtfde==0.1.2
ruamel.yaml==0.18.6; python_version >= '3.7'
ruamel.yaml.clib==0.2.8; python_version < '3.13' and platform_python_implementation == 'CPython'
setuptools==72.1.0; python_version >= '3.8'
shellingham==1.5.4; python_version >= '3.7'
shodan==1.31.0
sigmatools==0.23.1; python_version ~= '3.8'
sigmf==1.2.2; python_version >= '3.7'
simple-websocket==1.0.0; python_version >= '3.6'
simplejson==3.19.2; python_version >= '2.5' and python_version not in '3.0, 3.1, 3.2, 3.3'
six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
slack-sdk==3.27.1; python_version >= '3.6'
sniffio==1.3.1; python_version >= '3.7'
socialscan==1.4.0; python_version >= '3.6'
socketio-client==0.5.7.4
softenum==1.0.1; python_full_version >= '3.7.0' and python_full_version < '4.0.0'
soupsieve==2.5; python_version >= '3.8'
sparqlwrapper==2.0.0; python_version >= '3.7'
stix2==3.0.1; python_version >= '3.6'
stix2-patterns==2.0.0; python_version >= '3.6'
tabulate==0.9.0; python_version >= '3.7'
tau-clients==0.3.0; python_version >= '3.6'
taxii2-client==2.3.0
termcolor==2.4.0; python_version >= '3.8'
tldextract==5.1.2; python_version >= '3.8'
tornado==6.4.1; python_version >= '3.8'
tqdm==4.66.5; python_version >= '3.7'
trustar@ git+https://github.com/SteveClement/trustar-python.git@6954eae38e0c77eaeef26084b6c5fd033925c1c7
typer==0.12.3; python_version >= '3.7'
typing-extensions==4.12.2; python_version >= '3.8'
tzdata==2024.1; python_version >= '2'
tzlocal==5.2; python_version >= '3.8'
unicodecsv==0.14.1
url-normalize==1.4.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
urlarchiver==0.2
urllib3==2.2.2; python_version >= '3.8'
vt-graph-api==2.2.0
vt-py==0.18.3; python_full_version >= '3.7.0'
vulners==2.2.0; python_version >= '3.8'
vysion==2.0.8; python_full_version >= '3.8.0' and python_full_version < '4.0.0'
wand==0.6.13
websocket-client==1.8.0; python_version >= '3.8'
websockets==12.0; python_version >= '3.8'
wrapt==1.16.0; python_version >= '3.6'
wsproto==1.2.0; python_full_version >= '3.7.0'
xlrd==2.0.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
xlsxwriter==3.2.0; python_version >= '3.6'
yara-python==4.5.0
yarl==1.9.4; python_version >= '3.7'

130
docker/Dockerfile
View File

@ -1,130 +0,0 @@
FROM python:3.7-buster AS build
ENV DEBIAN_FRONTEND noninteractive
ENV WORKDIR="/usr/local/src/misp_modules"
ENV VENV_DIR="/misp_modules"
# Install Packages for build
RUN set -eu \
;mkdir -p ${WORKDIR} ${VENV_DIR} \
;apt-get update \
;apt-get install -y \
git \
libpq5 \
libjpeg-dev \
tesseract-ocr \
libpoppler-cpp-dev \
imagemagick \
virtualenv \
libopencv-dev \
zbar-tools \
libzbar0 \
libzbar-dev \
libfuzzy-dev \
;apt-get -y autoremove \
;apt-get -y clean \
;rm -rf /var/lib/apt/lists/* \
;
# Create MISP Modules
RUN set -eu \
;git clone https://github.com/MISP/misp-modules.git ${WORKDIR} \
;virtualenv -p python3 ${VENV_DIR}/venv \
;cd ${WORKDIR} \
;${VENV_DIR}/venv/bin/pip3 install -I -r REQUIREMENTS --no-cache-dir \
;${VENV_DIR}/venv/bin/pip3 install . --no-cache-dir \
;
#########################################
# Start Final Docker Image
#
FROM python:3.7-slim-buster AS final
ENV DEBIAN_FRONTEND noninteractive
ENV VENV_DIR="/misp_modules"
# Copy all builded files from build stage
COPY --from=build ${VENV_DIR} ${VENV_DIR}
# Install Packages to run it
RUN set -eu \
;apt-get update \
;apt-get install -y \
curl \
libpq5 \
# libjpeg-dev \
tesseract-ocr \
libpoppler-cpp-dev \
imagemagick \
# virtualenv \
# libopencv-dev \
zbar-tools \
libzbar0 \
# libzbar-dev \
# libfuzzy-dev \
;apt-get -y autoremove \
;apt-get -y clean \
;rm -rf /var/lib/apt/lists/* \
;chown -R nobody ${VENV_DIR} \
;
# Entrypoint
COPY files/entrypoint.sh /entrypoint.sh
ENTRYPOINT [ "/entrypoint.sh" ]
# Add Healthcheck Config
COPY files/healthcheck.sh /healthcheck.sh
HEALTHCHECK --interval=1m --timeout=45s --retries=3 CMD ["/healthcheck.sh"]
# Change Workdir
WORKDIR ${VENV_DIR}
# Change from root to www-data
USER nobody
# Expose Port
EXPOSE 6666
# Shortterm ARG Variables:
ARG VENDOR="MISP"
ARG COMPONENT="misp-modules"
ARG BUILD_DATE
ARG GIT_REPO="https://github.com/MISP/misp-modules"
ARG VCS_REF
ARG RELEASE_DATE
ARG NAME="MISP-dockerized-misp-modules"
ARG DESCRIPTION="This docker container contains MISP modules in an Debian Container."
ARG DOCUMENTATION="https://misp.github.io/misp-modules/"
ARG AUTHOR="MISP"
ARG LICENSE="BSD-3-Clause"
# Longterm Environment Variables
ENV \
BUILD_DATE=${BUILD_DATE} \
NAME=${NAME} \
PATH=$PATH:${VENV_DIR}/venv/bin
# Labels
LABEL org.label-schema.build-date="${BUILD_DATE}" \
org.label-schema.name="${NAME}" \
org.label-schema.description="${DESCRIPTION}" \
org.label-schema.vcs-ref="${VCS_REF}" \
org.label-schema.vcs-url="${GIT_REPO}" \
org.label-schema.url="${GIT_REPO}" \
org.label-schema.vendor="${VENDOR}" \
org.label-schema.version="${VERSION}" \
org.label-schema.usage="${DOCUMENTATION}" \
org.label-schema.schema-version="1.0.0-rc1"
LABEL org.opencontainers.image.created="${BUILD_DATE}" \
org.opencontainers.image.url="${GIT_REPO}" \
org.opencontainers.image.source="${GIT_REPO}" \
org.opencontainers.image.version="${VERSION}" \
org.opencontainers.image.revision="${VCS_REF}" \
org.opencontainers.image.vendor="${VENDOR}" \
org.opencontainers.image.title="${NAME}" \
org.opencontainers.image.description="${DESCRIPTION}" \
org.opencontainers.image.documentation="${DOCUMENTATION}" \
org.opencontainers.image.authors="${AUTHOR}" \
org.opencontainers.image.licenses="${LICENSE}"

37
docker/files/entrypoint.sh
View File

@ -1,37 +0,0 @@
#!/bin/sh
set -eu
# Variables
NC='\033[0m' # No Color
Light_Green='\033[1;32m'
STARTMSG="${Light_Green}[ENTRYPOINT_MISP_MODULES]${NC}"
VENV_DIR=${VENV_DIR:-"/misp-modules"}
MISP_MODULES_BINARY="${VENV_DIR}/venv/bin/misp-modules"
DEBUG=""
# Functions
echo (){
command echo "$STARTMSG $*"
}
# Environment Variables
MISP_MODULES_DEBUG=${MISP_MODULES_DEBUG:-"false"}
#
# MAIN
#
# Check if debugging mode should be enabled
[ "$MISP_MODULES_DEBUG" = "true" ] && DEBUG="-d"
# check if a command parameter exists and start misp-modules
if [ $# = 0 ]
then
# If no cmd parameter is set
echo "Start MISP Modules" && $MISP_MODULES_BINARY $DEBUG -l 0.0.0.0 > /dev/stdout 2> /dev/stderr
else
# If cmd parameter is set
echo "Start MISP Modules" && $MISP_MODULES_BINARY $DEBUG -l 0.0.0.0 > /dev/stdout 2> /dev/stderr &
exec "$@"
fi

4
docker/files/healthcheck.sh
View File

@ -1,4 +0,0 @@
#!/bin/sh
# If no contain is there or curl get an error back: exit 1. Docker restart then the container.
curl -fk http://0.0.0.0:6666/modules || exit 1

2
docs/index.md
View File

@ -21,7 +21,6 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj
* [AssemblyLine Query](https://misp.github.io/misp-modules/expansion/#assemblyline-query) - A module tu query the AssemblyLine API with a submission ID to get the submission report and parse it.
* [AssemblyLine Submit](https://misp.github.io/misp-modules/expansion/#assemblyline-submit) - A module to submit samples and URLs to AssemblyLine for advanced analysis, and return the link of the submission.
* [Backscatter.io](https://misp.github.io/misp-modules/expansion/#backscatter.io) - Backscatter.io module to bring mass-scanning observations into MISP.
* [BGP Ranking](https://misp.github.io/misp-modules/expansion/#bgp-ranking) - Query BGP Ranking to get the ranking of an Autonomous System number.
* [BTC Scam Check](https://misp.github.io/misp-modules/expansion/#btc-scam-check) - An expansion hover module to query a special dns blacklist to check if a bitcoin address has been abused.
* [BTC Steroids](https://misp.github.io/misp-modules/expansion/#btc-steroids) - An expansion hover module to get a blockchain balance from a BTC address in MISP.
* [Censys Enrich](https://misp.github.io/misp-modules/expansion/#censys-enrich) - An expansion module to enrich attributes in MISP by quering the censys.io API
@ -115,7 +114,6 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj
* [Vulnerability Lookup](https://misp.github.io/misp-modules/expansion/#vulnerability-lookup) - An expansion module to query Vulnerability Lookup
* [Vulners Lookup](https://misp.github.io/misp-modules/expansion/#vulners-lookup) - An expansion hover module to expand information about CVE id using Vulners API.
* [Vysion Enrich](https://misp.github.io/misp-modules/expansion/#vysion-enrich) - Module to enrich the information by making use of the Vysion API.
* [Whois Lookup](https://misp.github.io/misp-modules/expansion/#whois-lookup) - Module to query a local instance of uwhois (https://github.com/rafiot/uwhoisd).
* [WhoisFreaks Lookup](https://misp.github.io/misp-modules/expansion/#whoisfreaks-lookup) - An expansion module for https://whoisfreaks.com/ that will provide an enriched analysis of the provided domain, including WHOIS and DNS information.
* [Wikidata Lookup](https://misp.github.io/misp-modules/expansion/#wikidata-lookup) - An expansion hover module to extract information from Wikidata to have additional information about particular term for analysis.
* [IBM X-Force Exchange Lookup](https://misp.github.io/misp-modules/expansion/#ibm-x-force-exchange-lookup) - An expansion module for IBM X-Force Exchange.

258
docs/install.md
View File

@ -1,199 +1,147 @@
## How to install and start MISP modules (in a Python virtualenv)? (recommended)
IMPORTANT NOTE: we will soon be publishing `misp-modules` on PyPI.
***Be sure to run the latest version of `pip`***. To install the latest version of pip, `pip install --upgrade pip` will do the job.
## Install from pip
It is strongly recommended to use a virtual environment (see here for instructions https://docs.python.org/3/tutorial/venv.html).
Once the virtual environment is loaded just use the command:
~~~~bash
SUDO_WWW="sudo -u www-data"
sudo apt-get install -y \
python3-dev \
python3-pip \
git \
libpq5 \
libjpeg-dev \
tesseract-ocr \
libpoppler-cpp-dev \
imagemagick virtualenv \
libopencv-dev \
zbar-tools \
libzbar0 \
libzbar-dev \
libfuzzy-dev \
libcaca-dev \
build-essential
# BEGIN with virtualenv:
$SUDO_WWW virtualenv -p python3 /var/www/MISP/venv
# END with virtualenv
cd /usr/local/src/
# Ideally you add your user to the staff group and make /usr/local/src group writeable, below follows an example with user misp
sudo adduser misp staff
sudo chmod 2775 /usr/local/src
sudo chown root:staff /usr/local/src
git clone https://github.com/MISP/misp-modules.git
git clone git://github.com/stricaud/faup.git faup
git clone git://github.com/stricaud/gtcaca.git gtcaca
# Install gtcaca/faup
cd gtcaca
mkdir -p build
cd build
cmake .. && make
sudo make install
cd ../../faup
mkdir -p build
cd build
cmake .. && make
sudo make install
sudo ldconfig
cd ../../misp-modules
# BEGIN with virtualenv:
$SUDO_WWW /var/www/MISP/venv/bin/pip install -I -r REQUIREMENTS
$SUDO_WWW /var/www/MISP/venv/bin/pip install .
# END with virtualenv
# BEGIN without virtualenv:
sudo pip install -I -r REQUIREMENTS
sudo pip install .
# END without virtualenv
# Start misp-modules as a service
sudo cp etc/systemd/system/misp-modules.service /etc/systemd/system/
sudo systemctl daemon-reload
sudo systemctl enable --now misp-modules
sudo service misp-modules start # or
/var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s & # to start the modules manually
pip install misp-modules
~~~~
## How to install and start MISP modules on RHEL-based distributions ?
As of this writing, the official RHEL repositories only contain Ruby 2.0.0 and Ruby 2.1 or higher is required. As such, this guide installs Ruby 2.2 from the [SCL](https://access.redhat.com/documentation/en-us/red_hat_software_collections/3/html/3.2_release_notes/chap-installation#sect-Installation-Subscribe) repository.
## Install from cloned repository
In this case the only requirement is to install `poetry`. Normally you just need to run `pip install poetry`, but see here for more alternatives https://python-poetry.org/docs/#installation.
Once `poetry` is installed, you can clone the repository and install `misp-modules` as follows:
~~~~bash
SUDO_WWW="sudo -u apache"
sudo yum install \
rh-python36 \
rh-ruby22 \
openjpeg-devel \
rubygem-rouge \
rubygem-asciidoctor \
zbar-devel \
opencv-devel \
gcc-c++ \
pkgconfig \
poppler-cpp-devel \
python-devel \
redhat-rpm-config
cd /var/www/MISP
$SUDO_WWW git clone https://github.com/MISP/misp-modules.git
cd misp-modules
$SUDO_WWW /usr/bin/scl enable rh-python36 "virtualenv -p python3 /var/www/MISP/venv"
$SUDO_WWW /var/www/MISP/venv/bin/pip install -U -I -r REQUIREMENTS
$SUDO_WWW /var/www/MISP/venv/bin/pip install -U .
git clone https://github.com/MISP/misp-modules.git && cd misp-modules
git submodule update --init
poetry install
~~~~
Create the service file /etc/systemd/system/misp-modules.service :
Note that the dependencies will require a number of system packages installed. On Ubuntu these packages are `libpoppler-cpp-dev`, `libzbar0`, and `tesseract-ocr`. For an updated list, check the github action used to test the build inside `.github/workflows`.
## Install the systemd unit
To run `misp-modules` as a service on a distribution based on systemd, you need to create the unit as follows and store it in a file `/etc/systemd/system/misp-modules.service`:
~~~~bash
echo "[Unit]
Description=MISP's modules
After=misp-workers.service
[Unit]
Description=MISP modules
[Service]
Type=simple
User=apache
Group=apache
ExecStart=/usr/bin/scl enable rh-python36 rh-ruby22 '/var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s'
ExecStart='/path/to/venv/bin/misp-modules -l 127.0.0.1 -s'
Restart=always
RestartSec=10
[Install]
WantedBy=multi-user.target" | sudo tee /etc/systemd/system/misp-modules.service
WantedBy=multi-user.target
~~~~
The `After=misp-workers.service` must be changed or removed if you have not created a misp-workers service.
Then, enable the misp-modules service and start it:
~~~~bash
systemctl daemon-reload
systemctl enable --now misp-modules
~~~~
## How to use an MISP modules Docker container
### Docker build
## Run the tests
To run tests you need to install misp-modules from the cloned repository, run the server, and then run the tests. You can do all these step with `poetry`.
~~~~bash
docker build -t misp-modules \
--build-arg BUILD_DATE=$(date -u +"%Y-%m-%d") \
docker/
poetry install
poetry run misp-modules
~~~~
### Docker run
And in another terminal:
~~~~bash
# Start Redis
docker run --rm -d --name=misp-redis redis:alpine
# Start MISP-modules
docker run \
--rm -d --name=misp-modules \
-e REDIS_BACKEND=misp-redis \
-e REDIS_PORT="6379" \
-e REDIS_PW="" \
-e REDIS_DATABASE="245" \
-e MISP_MODULES_DEBUG="false" \
dcso/misp-dockerized-misp-modules
poetry run pytest
~~~~
### Docker-compose
~~~~yml
services:
misp-modules:
# https://hub.docker.com/r/dcso/misp-dockerized-misp-modules
image: dcso/misp-dockerized-misp-modules:3
## Build the documentation
# Local image:
#image: misp-modules
#build:
# context: docker/
To build the documentation you can use the provided `Makefile`.
Inside you will find three targets:
environment:
# Redis
REDIS_BACKEND: misp-redis
REDIS_PORT: "6379"
REDIS_DATABASE: "245"
# System PROXY (OPTIONAL)
http_proxy:
https_proxy:
no_proxy: 0.0.0.0
# Timezone (OPTIONAL)
TZ: Europe/Berlin
# MISP-Modules (OPTIONAL)
MISP_MODULES_DEBUG: "false"
# Logging options (OPTIONAL)
LOG_SYSLOG_ENABLED: "no"
misp-redis:
# https://hub.docker.com/_/redis or alternative https://hub.docker.com/r/dcso/misp-dockerized-redis/
image: redis:alpine
- `generate_docs`: install the depdendency and generate the documentation.
- `generate_docs`: build the documentation using `mkdocs`.
- `deploy`: deploy the documentation using `mkdocs gh-deploy`.
- `test-docs`: run a local server exposing the newly built documentation.
Note that you can either run the targets using `poetry` (default), or using the `squidfunk/mkdocs-material` by setting the environment variable `USE_DOCKER=true`.
## Run MISP modules
If you installed it using pip, you just need to execute the command `misp-modules` (source the virtual environment a second time to update the search paths). If you installed it from the cloned repository, just use poetry, i.e., `poetry run misp-modules`.
## Run MISP modules in Docker
You can find an up-to-date container image and related documentation at the following repository:m https://github.com/MISP/misp-docker
## Install misp-module on an offline instance
### If `misp-modules` is available on PyPI
Once `misp-modules` is available on PyPI, you can just download all the necessary packages:
~~~~bash
mkdir wheels
pip wheel misp-modules --no-cache-dir -w ./wheels
~~~~
## Install misp-module on an offline instance.
First, you need to grab all necessary packages for example like this :
Move the `wheels` directory to the target system, and install them there:
Use pip wheel to create an archive
~~~
mkdir misp-modules-offline
pip3 wheel -r REQUIREMENTS shodan --wheel-dir=./misp-modules-offline
tar -cjvf misp-module-bundeled.tar.bz2 ./misp-modules-offline/*
~~~
On offline machine :
~~~
mkdir misp-modules-bundle
tar xvf misp-module-bundeled.tar.bz2 -C misp-modules-bundle
cd misp-modules-bundle
ls -1|while read line; do sudo pip3 install --force-reinstall --ignore-installed --upgrade --no-index --no-deps ${line};done
~~~
Next you can follow standard install procedure.
~~~~bash
pip install --no-cache-dir --use-deprecated=legacy-resolver /wheels/*.whl
~~~~
Once again, using a virtual environment is recommended.
### If `misp-modules` is not available on PyPI
You have two choices, the first approach uses `poetry export` to export the entire virtual environment so you can copy and run it on the target system; the second one uses `poetry bundle` to export a `requirements.txt` file.
#### Using `poetry bundle`
This is quite straightforward but it assumes your target system is relatively similar (same distribution, architecture, libaries).
~~~~bash
poetry install
poetry self add poetry-plugin-bundle
poetry bundle venv /destination/path/
~~~~
#### Using `poetry export`
This is a bit more convoluted and it is similar to how you would install `misp-modules` on an offline instance.
Just follow those instructions but replace the package `misp-modules` with `-r requirements.txt`.
Before doing so you need to generate the `requirements.txt` file. Due to the fact we are still supporting Python 3.8 and that Poetry still has some limitations (soon to be resolved) you need to need to replace the line `python = ">=3.8.*,<3.13"` inside `pyproject.toml` with your exact version (just run `python --version`).
Once you have done that, run the following commands to generate your very own `requirements.txt`.
~~~~bash
poetry lock
poetry install
poetry self add poetry-plugin-export
poetry export --without-hashes -f requirements.txt -o requirements.txt
~~~~

48
documentation/README.md
View File

@ -175,28 +175,6 @@ Backscatter.io module to bring mass-scanning observations into MISP.
-----
#### [BGP Ranking](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/bgpranking.py)
Query BGP Ranking to get the ranking of an Autonomous System number.
[[source code](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/bgpranking.py)]
- **features**:
>The module takes an AS number attribute as input and displays its description as well as its ranking position in BGP Ranking for a given day.
- **input**:
>Autonomous system number.
- **output**:
>An asn object with its related bgp-ranking object.
- **references**:
>https://github.com/D4-project/BGP-Ranking/
- **requirements**:
>pybgpranking python library
-----
#### [BTC Scam Check](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/btc_scam_check.py)
<img src=logos/bitcoin.png height=60>
@ -2843,32 +2821,6 @@ Module to enrich the information by making use of the Vysion API.
-----
#### [Whois Lookup](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/whois.py)
Module to query a local instance of uwhois (https://github.com/rafiot/uwhoisd).
[[source code](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/whois.py)]
- **features**:
>This module takes a domain or IP address attribute as input and queries a 'Univseral Whois proxy server' to get the correct details of the Whois query on the input value (check the references for more details about this whois server).
- **config**:
> - server
> - port
- **input**:
>A domain or IP address attribute.
- **output**:
>Text describing the result of a whois request for the input value.
- **references**:
>https://github.com/rafiot/uwhoisd
- **requirements**:
>uwhois: A whois python library
-----
#### [WhoisFreaks Lookup](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/whoisfreaks.py)
<img src=logos/whoisfreaks.png height=60>

0
documentation/generate_documentation.py Normal file → Executable file
View File

96
documentation/migrate_web_json_to_module.py
View File

@ -1,96 +0,0 @@
import json
import re
from pathlib import Path
import os
module_types = ['expansion', 'export_mod', 'import_mod']
moduleinfo_template = {
'version': '1.0',
'author': '',
'module-type': [],
'description': '',
'logo': '',
'requirements': [],
'features': '',
'references': [],
'input': '',
'output': ''
}
if __name__ == '__main__':
exit("This code was temporary and should not be run again. It was used to migrate the JSON documentation to the module files.")
root_path = Path(__file__).resolve().parent.parent
modules_path = root_path / 'misp_modules' / 'modules'
for module_type in module_types:
files = sorted(os.listdir(modules_path / module_type))
for python_filename in files:
if not python_filename.endswith('.py') or '__init__' in python_filename:
continue
modulename = python_filename.split('.py')[0]
json_filename = root_path / 'documentation' / 'website' / module_type / f'{modulename}.json'
print(f"Processing type {module_type}:{modulename} in {python_filename} and {json_filename}")
json_exists = json_filename.exists()
if json_exists:
print(" Found JSON file")
with open(json_filename, 'rt') as f:
json_content = json.loads(f.read())
else:
json_content = {}
# if json does not exist, then still edit the python file and add the stub structure
with open(modules_path / module_type / python_filename, 'r+t') as python_f:
# read from python file, find moduleinfo and load it as python variable
python_content = python_f.read()
re_pattern = r'moduleinfo\s=\s{[^}]*}'
m = re.search(re_pattern, python_content, re.MULTILINE | re.DOTALL)
if not m:
print(f" Moduleinfo not found in {python_filename}")
continue
s = m.group(0)
moduleinfo = {}
exec(s) # we now have a moduleinfo dict
print(f" Moduleinfo found in {python_filename}: {moduleinfo}")
# populate from template
for k, v in moduleinfo_template.items():
if k not in moduleinfo or moduleinfo.get(k) == '' or moduleinfo.get(k) == []:
# print(f" Adding {k} = {v} to {python_filename}")
moduleinfo[k] = v
# populate from json
for k, v in json_content.items():
if k not in moduleinfo or moduleinfo.get(k) == '' or moduleinfo.get(k) == []:
# print(f" Adding {k} = {v} to {python_filename}")
moduleinfo[k] = v
if json_content and json_content.get('description') != moduleinfo.get('description'):
print(" WARNING: Description in JSON and Python file do not match:")
print("")
print(f" JSON: {json_content.get('description')}")
print("")
print(f" Python: {moduleinfo.get('description')}")
print("")
user_input = input("Which version do you want to use? Enter '[j]son' for JSON version or '[p]ython' for Python version, or any other text for a new description: ")
if user_input in ['json', 'j', 'JSON']:
moduleinfo['description'] = json_content['description']
elif user_input in ['python', 'p', 'PYTHON']:
pass
else:
moduleinfo['description'] = user_input.strip()
# write back to python file
new_moduleinfo_text = ['moduleinfo = {']
for k, v in moduleinfo.items():
v_updated = repr(v).replace('\\', '\\\\')
new_moduleinfo_text.append(f" '{k}': {v_updated},")
new_moduleinfo_text.append('}')
python_content_new, cnt = re.subn(re_pattern, '\n'.join(new_moduleinfo_text), python_content, re.MULTILINE | re.DOTALL)
if cnt == 0:
print(f" WARNING: Moduleinfo not replaced in {python_filename}")
continue
python_f.seek(0)
python_f.write(python_content_new)
python_f.truncate() # remove the rest of the file
pass
pass

3
documentation/mkdocs/REQUIREMENTS.txt
View File

@ -1,3 +0,0 @@
mkdocs
mkdocs-material
markdown_include

48
documentation/mkdocs/expansion.md
View File

@ -172,28 +172,6 @@ Backscatter.io module to bring mass-scanning observations into MISP.
-----
#### [BGP Ranking](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/bgpranking.py)
Query BGP Ranking to get the ranking of an Autonomous System number.
[[source code](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/bgpranking.py)]
- **features**:
>The module takes an AS number attribute as input and displays its description as well as its ranking position in BGP Ranking for a given day.
- **input**:
>Autonomous system number.
- **output**:
>An asn object with its related bgp-ranking object.
- **references**:
>https://github.com/D4-project/BGP-Ranking/
- **requirements**:
>pybgpranking python library
-----
#### [BTC Scam Check](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/btc_scam_check.py)
<img src=../logos/bitcoin.png height=60>
@ -2840,32 +2818,6 @@ Module to enrich the information by making use of the Vysion API.
-----
#### [Whois Lookup](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/whois.py)
Module to query a local instance of uwhois (https://github.com/rafiot/uwhoisd).
[[source code](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/whois.py)]
- **features**:
>This module takes a domain or IP address attribute as input and queries a 'Univseral Whois proxy server' to get the correct details of the Whois query on the input value (check the references for more details about this whois server).
- **config**:
> - server
> - port
- **input**:
>A domain or IP address attribute.
- **output**:
>Text describing the result of a whois request for the input value.
- **references**:
>https://github.com/rafiot/uwhoisd
- **requirements**:
>uwhois: A whois python library
-----
#### [WhoisFreaks Lookup](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/whoisfreaks.py)
<img src=../logos/whoisfreaks.png height=60>

2
documentation/mkdocs/index.md
View File

@ -21,7 +21,6 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj
* [AssemblyLine Query](https://misp.github.io/misp-modules/expansion/#assemblyline-query) - A module tu query the AssemblyLine API with a submission ID to get the submission report and parse it.
* [AssemblyLine Submit](https://misp.github.io/misp-modules/expansion/#assemblyline-submit) - A module to submit samples and URLs to AssemblyLine for advanced analysis, and return the link of the submission.
* [Backscatter.io](https://misp.github.io/misp-modules/expansion/#backscatter.io) - Backscatter.io module to bring mass-scanning observations into MISP.
* [BGP Ranking](https://misp.github.io/misp-modules/expansion/#bgp-ranking) - Query BGP Ranking to get the ranking of an Autonomous System number.
* [BTC Scam Check](https://misp.github.io/misp-modules/expansion/#btc-scam-check) - An expansion hover module to query a special dns blacklist to check if a bitcoin address has been abused.
* [BTC Steroids](https://misp.github.io/misp-modules/expansion/#btc-steroids) - An expansion hover module to get a blockchain balance from a BTC address in MISP.
* [Censys Enrich](https://misp.github.io/misp-modules/expansion/#censys-enrich) - An expansion module to enrich attributes in MISP by quering the censys.io API
@ -115,7 +114,6 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj
* [Vulnerability Lookup](https://misp.github.io/misp-modules/expansion/#vulnerability-lookup) - An expansion module to query Vulnerability Lookup
* [Vulners Lookup](https://misp.github.io/misp-modules/expansion/#vulners-lookup) - An expansion hover module to expand information about CVE id using Vulners API.
* [Vysion Enrich](https://misp.github.io/misp-modules/expansion/#vysion-enrich) - Module to enrich the information by making use of the Vysion API.
* [Whois Lookup](https://misp.github.io/misp-modules/expansion/#whois-lookup) - Module to query a local instance of uwhois (https://github.com/rafiot/uwhoisd).
* [WhoisFreaks Lookup](https://misp.github.io/misp-modules/expansion/#whoisfreaks-lookup) - An expansion module for https://whoisfreaks.com/ that will provide an enriched analysis of the provided domain, including WHOIS and DNS information.
* [Wikidata Lookup](https://misp.github.io/misp-modules/expansion/#wikidata-lookup) - An expansion hover module to extract information from Wikidata to have additional information about particular term for analysis.
* [IBM X-Force Exchange Lookup](https://misp.github.io/misp-modules/expansion/#ibm-x-force-exchange-lookup) - An expansion module for IBM X-Force Exchange.

270
documentation/mkdocs/install.md
View File

@ -1,199 +1,159 @@
## How to install and start MISP modules (in a Python virtualenv)? (recommended)
## Install from pip
***Be sure to run the latest version of `pip`***. To install the latest version of pip, `pip install --upgrade pip` will do the job.
It is strongly recommended to use a virtual environment (see here for instructions https://docs.python.org/3/tutorial/venv.html).
Once the virtual environment is loaded just use the command:
~~~~bash
SUDO_WWW="sudo -u www-data"
sudo apt-get install -y \
python3-dev \
python3-pip \
git \
libpq5 \
libjpeg-dev \
tesseract-ocr \
libpoppler-cpp-dev \
imagemagick virtualenv \
libopencv-dev \
zbar-tools \
libzbar0 \
libzbar-dev \
libfuzzy-dev \
libcaca-dev \
build-essential
# BEGIN with virtualenv:
$SUDO_WWW virtualenv -p python3 /var/www/MISP/venv
# END with virtualenv
cd /usr/local/src/
# Ideally you add your user to the staff group and make /usr/local/src group writeable, below follows an example with user misp
sudo adduser misp staff
sudo chmod 2775 /usr/local/src
sudo chown root:staff /usr/local/src
git clone https://github.com/MISP/misp-modules.git
git clone git://github.com/stricaud/faup.git faup
git clone git://github.com/stricaud/gtcaca.git gtcaca
# Install gtcaca/faup
cd gtcaca
mkdir -p build
cd build
cmake .. && make
sudo make install
cd ../../faup
mkdir -p build
cd build
cmake .. && make
sudo make install
sudo ldconfig
cd ../../misp-modules
# BEGIN with virtualenv:
$SUDO_WWW /var/www/MISP/venv/bin/pip install -I -r REQUIREMENTS
$SUDO_WWW /var/www/MISP/venv/bin/pip install .
# END with virtualenv
# BEGIN without virtualenv:
sudo pip install -I -r REQUIREMENTS
sudo pip install .
# END without virtualenv
# Start misp-modules as a service
sudo cp etc/systemd/system/misp-modules.service /etc/systemd/system/
sudo systemctl daemon-reload
sudo systemctl enable --now misp-modules
sudo service misp-modules start # or
/var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s & # to start the modules manually
pip install misp-modules
~~~~
## How to install and start MISP modules on RHEL-based distributions ?
Note: this install method might not yet be available.
As of this writing, the official RHEL repositories only contain Ruby 2.0.0 and Ruby 2.1 or higher is required. As such, this guide installs Ruby 2.2 from the [SCL](https://access.redhat.com/documentation/en-us/red_hat_software_collections/3/html/3.2_release_notes/chap-installation#sect-Installation-Subscribe) repository.
## Install from cloned repository
In this case the only requirement is to install `poetry`. Normally you just need to run `pip install poetry`, but see here for more alternatives https://python-poetry.org/docs/#installation.
Once `poetry` is installed, you can clone the repository and install `misp-modules` as follows:
~~~~bash
SUDO_WWW="sudo -u apache"
sudo yum install \
rh-python36 \
rh-ruby22 \
openjpeg-devel \
rubygem-rouge \
rubygem-asciidoctor \
zbar-devel \
opencv-devel \
gcc-c++ \
pkgconfig \
poppler-cpp-devel \
python-devel \
redhat-rpm-config
cd /var/www/MISP
$SUDO_WWW git clone https://github.com/MISP/misp-modules.git
cd misp-modules
$SUDO_WWW /usr/bin/scl enable rh-python36 "virtualenv -p python3 /var/www/MISP/venv"
$SUDO_WWW /var/www/MISP/venv/bin/pip install -U -I -r REQUIREMENTS
$SUDO_WWW /var/www/MISP/venv/bin/pip install -U .
git clone https://github.com/MISP/misp-modules.git && cd misp-modules
git submodule update --init
poetry install
~~~~
Create the service file /etc/systemd/system/misp-modules.service :
Note that the dependencies will require a number of system packages installed. On Ubuntu these packages are `libpoppler-cpp-dev`, `libzbar0`, and `tesseract-ocr`. For an updated list, check the github action used to test the build inside `.github/workflows`.
## Install the systemd unit
To run `misp-modules` as a service on a distribution based on systemd, you need to create the unit as follows and store it in a file `/etc/systemd/system/misp-modules.service`:
~~~~bash
echo "[Unit]
Description=MISP's modules
After=misp-workers.service
[Unit]
Description=MISP modules
[Service]
Type=simple
User=apache
Group=apache
ExecStart=/usr/bin/scl enable rh-python36 rh-ruby22 '/var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s'
ExecStart='/path/to/venv/bin/misp-modules -l 127.0.0.1 -s'
Restart=always
RestartSec=10
[Install]
WantedBy=multi-user.target" | sudo tee /etc/systemd/system/misp-modules.service
WantedBy=multi-user.target
~~~~
The `After=misp-workers.service` must be changed or removed if you have not created a misp-workers service.
Then, enable the misp-modules service and start it:
~~~~bash
systemctl daemon-reload
systemctl enable --now misp-modules
~~~~
## How to use an MISP modules Docker container
### Docker build
## Run the tests
To run tests you need to install misp-modules from the cloned repository, run the server, and then run the tests. You can do all these step with `poetry`.
~~~~bash
docker build -t misp-modules \
--build-arg BUILD_DATE=$(date -u +"%Y-%m-%d") \
docker/
poetry install
poetry run misp-modules
~~~~
### Docker run
And in another terminal:
~~~~bash
# Start Redis
docker run --rm -d --name=misp-redis redis:alpine
# Start MISP-modules
docker run \
--rm -d --name=misp-modules \
-e REDIS_BACKEND=misp-redis \
-e REDIS_PORT="6379" \
-e REDIS_PW="" \
-e REDIS_DATABASE="245" \
-e MISP_MODULES_DEBUG="false" \
dcso/misp-dockerized-misp-modules
poetry run pytest
~~~~
### Docker-compose
~~~~yml
services:
misp-modules:
# https://hub.docker.com/r/dcso/misp-dockerized-misp-modules
image: dcso/misp-dockerized-misp-modules:3
## Build the documentation
# Local image:
#image: misp-modules
#build:
# context: docker/
To build the documentation you can use the provided `Makefile`.
Inside you will find three targets:
environment:
# Redis
REDIS_BACKEND: misp-redis
REDIS_PORT: "6379"
REDIS_DATABASE: "245"
# System PROXY (OPTIONAL)
http_proxy:
https_proxy:
no_proxy: 0.0.0.0
# Timezone (OPTIONAL)
TZ: Europe/Berlin
# MISP-Modules (OPTIONAL)
MISP_MODULES_DEBUG: "false"
# Logging options (OPTIONAL)
LOG_SYSLOG_ENABLED: "no"
misp-redis:
# https://hub.docker.com/_/redis or alternative https://hub.docker.com/r/dcso/misp-dockerized-redis/
image: redis:alpine
- `generate_docs`: install the depdendency and generate the documentation.
- `generate_docs`: build the documentation using `mkdocs`.
- `deploy`: deploy the documentation using `mkdocs gh-deploy`.
- `test-docs`: run a local server exposing the newly built documentation.
Note: you can either run the targets using `poetry` (default), or using the Docker image `squidfunk/mkdocs-material` by setting the environment variable `USE_DOCKER=true`.
## Run MISP modules
If you installed it using pip, you just need to execute the command `misp-modules` (source the virtual environment a second time to update the search paths). If you installed it from the cloned repository, just use poetry, i.e., `poetry run misp-modules`.
## Run MISP modules in Docker
You can find an up-to-date container image and related documentation at the following repository: https://github.com/MISP/misp-docker .
## Install misp-module on an offline instance
### If `misp-modules` is available on PyPI
Once `misp-modules` is available on PyPI, you can just download all the necessary packages:
~~~~bash
mkdir wheels
pip wheel misp-modules --no-cache-dir -w ./wheels
~~~~
## Install misp-module on an offline instance.
First, you need to grab all necessary packages for example like this :
Move the `wheels` directory to the target system, and install them there:
~~~~bash
pip install --no-cache-dir --use-deprecated=legacy-resolver /wheels/*.whl
~~~~
Once again, using a virtual environment is recommended.
### If `misp-modules` is not available on PyPI
You have two choices, the first approach uses `poetry export` to export the entire virtual environment so you can copy and run it on the target system; the second one uses `poetry bundle` to export a `requirements.txt` file.
#### Using `poetry bundle`
This is quite straightforward but it assumes your target system is relatively similar (same distribution, architecture, libaries).
~~~~bash
poetry install
poetry self add poetry-plugin-bundle
poetry bundle venv /destination/path/
~~~~
#### Using `poetry export`
This is a bit more convoluted and it is similar to how you would install `misp-modules` on an offline instance.
Just follow those instructions but replace the package `misp-modules` with `-r requirements.txt`.
Before doing so you need to generate the `requirements.txt` file. Due to the fact we are still supporting Python 3.8 and that Poetry still has some limitations (soon to be resolved) you need to need to replace the line `python = ">=3.8.*,<3.13"` inside `pyproject.toml` with your exact version (just run `python --version`).
The following `sed` command does everything for you.
~~~~bash
sed -i "s/^python = .*/python = \"$(python -c 'import platform; print(platform.python_version())')\"/" pyproject.toml
~~~~
Then, run the following commands to generate your very own `requirements.txt`.
~~~~bash
poetry lock
poetry install
poetry self add poetry-plugin-export
poetry export --without-hashes -f requirements.txt -o requirements.txt
~~~~
Note that `misp-modules` will not be part of the `requirements.txt` file and you will need to create the wheel yourself:
~~~~bash
poetry build --output ./wheels
~~~~
Use pip wheel to create an archive
~~~
mkdir misp-modules-offline
pip3 wheel -r REQUIREMENTS shodan --wheel-dir=./misp-modules-offline
tar -cjvf misp-module-bundeled.tar.bz2 ./misp-modules-offline/*
~~~
On offline machine :
~~~
mkdir misp-modules-bundle
tar xvf misp-module-bundeled.tar.bz2 -C misp-modules-bundle
cd misp-modules-bundle
ls -1|while read line; do sudo pip3 install --force-reinstall --ignore-installed --upgrade --no-index --no-deps ${line};done
~~~
Next you can follow standard install procedure.

14
etc/systemd/system/misp-modules.service
View File

@ -1,14 +0,0 @@
[Unit]
Description=System-wide instance of the MISP Modules
After=network.target
[Service]
User=www-data
Group=www-data
WorkingDirectory=/usr/local/src/misp-modules
Environment="PATH=/var/www/MISP/venv/bin"
ExecStart=/var/www/MISP/venv/bin/misp-modules -l 127.0.0.1
[Install]
WantedBy=multi-user.target

49
misp_modules/__init__.py
View File

@ -28,6 +28,9 @@ import argparse
import re
import datetime
import psutil
import pkgutil
import platform
import typing
try:
import orjson as json
@ -41,6 +44,52 @@ from tornado.concurrent import run_on_executor
from concurrent.futures import ThreadPoolExecutor
from pymisp import pymisp_json_default
import warnings
warnings.filterwarnings("ignore", category=SyntaxWarning)
LIBFAUP_PATHS = [
"/usr/local/lib/",
"/usr/lib/",
"/opt/local/lib/",
]
ARCH_TO_EXTENSION = {
"linux": "so",
"darwin": "dylib",
}
def _get_libfaup_path(lib_path: str) -> str:
extension = ARCH_TO_EXTENSION.get(platform.system().lower(), "lib")
return f"{lib_path.rstrip('/')}/libfaupl.{extension}"
def _replace_libfaup_path(module_path: str, libfaup_path: str) -> None:
with open(module_path, "r") as f:
file_data = f.read()
file_data = re.sub(r"cdll.LoadLibrary\(.*\)", f"cdll.LoadLibrary(\"{libfaup_path}\")", file_data)
with open(module_path, "w") as f:
f.write(file_data)
def _try_pyfaup_import(lib_path: typing.Optional[str]) -> None:
package = pkgutil.get_loader("pyfaup")
if not package:
return
if lib_path:
_replace_libfaup_path(package.path, _get_libfaup_path(lib_path))
importlib.import_module("pyfaup")
for lib_path in [None, *LIBFAUP_PATHS]:
try:
_try_pyfaup_import(lib_path)
break
except OSError:
continue
try:
from .modules import * # noqa
HAS_PACKAGE_MODULES = True

7
misp_modules/helpers/cache.py
View File

@ -27,12 +27,13 @@ import hashlib
port = int(os.getenv("REDIS_PORT")) if os.getenv("REDIS_PORT") else 6379
hostname = os.getenv("REDIS_BACKEND") or '127.0.0.1'
db = int(os.getenv("REDIS_DATABASE")) if os.getenv("REDIS_DATABASE") else 0
password = os.getenv("REDIS_PW") or None
def selftest(enable=True):
if not enable:
return False
r = redis.Redis(host=hostname, port=port, db=db)
r = redis.Redis(host=hostname, password=password, port=port, db=db)
try:
r.ping()
except Exception:
@ -42,7 +43,7 @@ def selftest(enable=True):
def get(modulename=None, query=None, value=None, debug=False):
if (modulename is None or query is None):
return False
r = redis.Redis(host=hostname, port=port, db=db, decode_responses=True)
r = redis.Redis(host=hostname, password=password, port=port, db=db, decode_responses=True)
h = hashlib.sha1()
h.update(query.encode('UTF-8'))
hv = h.hexdigest()
@ -60,7 +61,7 @@ def get(modulename=None, query=None, value=None, debug=False):
def flush():
r = redis.StrictRedis(host=hostname, port=port, db=db, decode_responses=True)
r = redis.StrictRedis(host=hostname, password=password, port=port, db=db, decode_responses=True)
returncode = r.flushdb()
return returncode

4
misp_modules/modules/expansion/__init__.py
View File

@ -3,10 +3,10 @@ import sys
sys.path.append('{}/lib'.format('/'.join((os.path.realpath(__file__)).split('/')[:-3])))
__all__ = ['cuckoo_submit', 'vmray_submit', 'bgpranking', 'circl_passivedns', 'circl_passivessl',
__all__ = ['cuckoo_submit', 'vmray_submit', 'circl_passivedns', 'circl_passivessl',
'cluster25_expand', 'countrycode', 'cve', 'cve_advanced', 'cpe', 'dns', 'btc_steroids', 'domaintools',
'eupi', 'eql', 'farsight_passivedns', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal',
'whois', 'shodan', 'reversedns', 'geoip_asn', 'geoip_city', 'geoip_country', 'wiki', 'iprep',
'shodan', 'reversedns', 'geoip_asn', 'geoip_city', 'geoip_country', 'wiki', 'iprep',
'threatminer', 'otx', 'threatcrowd', 'vulndb', 'crowdstrike_falcon',
'yara_syntax_validator', 'hashdd', 'onyphe', 'onyphe_full', 'rbl',
'xforceexchange', 'sigma_syntax_validator', 'stix2_pattern_syntax_validator',

96
misp_modules/modules/expansion/bgpranking.py
View File

@ -1,96 +0,0 @@
# -*- coding: utf-8 -*-
import json
from . import check_input_attribute, standard_error_message
from datetime import date, datetime, timedelta
from pybgpranking import BGPRanking
from pymisp import MISPAttribute, MISPEvent, MISPObject
misperrors = {'error': 'Error'}
mispattributes = {'input': ['AS'], 'format': 'misp_standard'}
moduleinfo = {
'version': '0.1',
'author': 'Raphaël Vinot',
'description': 'Query BGP Ranking to get the ranking of an Autonomous System number.',
'module-type': ['expansion', 'hover'],
'name': 'BGP Ranking',
'logo': '',
'requirements': ['pybgpranking python library'],
'features': 'The module takes an AS number attribute as input and displays its description as well as its ranking position in BGP Ranking for a given day.',
'references': ['https://github.com/D4-project/BGP-Ranking/'],
'input': 'Autonomous system number.',
'output': 'An asn object with its related bgp-ranking object.',
}
def handler(q=False):
if q is False:
return False
request = json.loads(q)
if not request.get('attribute') or not check_input_attribute(request['attribute']):
return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'}
toquery = request['attribute']
if toquery['type'] not in mispattributes['input']:
return {'error': 'Unsupported attribute type.'}
bgpranking = BGPRanking()
value_toquery = int(toquery['value'][2:]) if toquery['value'].startswith('AS') else int(toquery['value'])
values = bgpranking.query(value_toquery, date=(date.today() - timedelta(1)).isoformat())
if not values['response'] or not values['response']['asn_description']:
misperrors['error'] = 'There is no result about this ASN in BGP Ranking'
return misperrors
event = MISPEvent()
attribute = MISPAttribute()
attribute.from_dict(**toquery)
event.add_attribute(**attribute)
asn_object = MISPObject('asn')
asn_object.add_attribute(**{
'type': 'AS',
'object_relation': 'asn',
'value': values['meta']['asn']
})
description, country = values['response']['asn_description'].split(', ')
for relation, value in zip(('description', 'country'), (description, country)):
asn_object.add_attribute(**{
'type': 'text',
'object_relation': relation,
'value': value
})
mapping = {
'address_family': {'type': 'text', 'object_relation': 'address-family'},
'date': {'type': 'datetime', 'object_relation': 'date'},
'position': {'type': 'float', 'object_relation': 'position'},
'rank': {'type': 'float', 'object_relation': 'ranking'}
}
bgp_object = MISPObject('bgp-ranking')
for feature in ('rank', 'position'):
bgp_attribute = {'value': values['response']['ranking'][feature]}
bgp_attribute.update(mapping[feature])
bgp_object.add_attribute(**bgp_attribute)
date_attribute = {'value': datetime.strptime(values['meta']['date'], '%Y-%m-%d')}
date_attribute.update(mapping['date'])
bgp_object.add_attribute(**date_attribute)
address_attribute = {'value': values['meta']['address_family']}
address_attribute.update(mapping['address_family'])
bgp_object.add_attribute(**address_attribute)
asn_object.add_reference(attribute.uuid, 'describes')
asn_object.add_reference(bgp_object.uuid, 'ranked-with')
event.add_object(asn_object)
event.add_object(bgp_object)
event = json.loads(event.to_json())
results = {key: event[key] for key in ('Attribute', 'Object')}
return {'results': results}
def introspection():
return mispattributes
def version():
return moduleinfo

7
misp_modules/modules/expansion/ods_enrich.py
View File

@ -47,8 +47,11 @@ def handler(q=False):
try:
for i in range(0, num_sheets):
rows = pandas_ods_reader.parsers.ods.get_rows(doc, i)
ods = pandas_ods_reader.algo.parse_data(pandas_ods_reader.parsers.ods, rows, headers=False, columns=[], skiprows=0)
ods = pandas_ods_reader.utils.sanitize_df(ods)
try:
ods = pandas_ods_reader.algo.parse_data(pandas_ods_reader.parsers.ods, rows, headers=False, columns=[], skiprows=0)
ods = pandas_ods_reader.utils.sanitize_df(ods)
except TypeError:
ods = pandas_ods_reader.algo.read_data(pandas_ods_reader.parsers.ods, ods_file, i, headers=False)
ods_content = ods_content + "\n" + ods.to_string(max_rows=None)
return {'results': [{'types': ['freetext'], 'values': ods_content, 'comment': ".ods-to-text from file " + filename},
{'types': ['text'], 'values': ods_content, 'comment': ".ods-to-text from file " + filename}]}

2
misp_modules/modules/expansion/vysion.py
View File

@ -54,7 +54,7 @@ moduleconfig = [
LOGGER = logging.getLogger("vysion")
LOGGER.setLevel(logging.INFO)
LOGGER.info("Starting Vysion")
LOGGER.debug("Starting Vysion")
DEFAULT_RESULTS_LIMIT = 10

62
misp_modules/modules/expansion/whois.py
View File

@ -1,62 +0,0 @@
# -*- coding: utf-8 -*-
import json
try:
from uwhois import Uwhois
except ImportError:
print("uwhois module not installed.")
misperrors = {'error': 'Error'}
mispattributes = {'input': ['domain', 'ip-src', 'ip-dst'], 'output': ['freetext']}
moduleinfo = {
'version': '0.1',
'author': 'Raphaël Vinot',
'description': 'Module to query a local instance of uwhois (https://github.com/rafiot/uwhoisd).',
'module-type': ['expansion'],
'name': 'Whois Lookup',
'logo': '',
'requirements': ['uwhois: A whois python library'],
'features': "This module takes a domain or IP address attribute as input and queries a 'Univseral Whois proxy server' to get the correct details of the Whois query on the input value (check the references for more details about this whois server).",
'references': ['https://github.com/rafiot/uwhoisd'],
'input': 'A domain or IP address attribute.',
'output': 'Text describing the result of a whois request for the input value.',
}
moduleconfig = ['server', 'port']
def handler(q=False):
if q is False:
return False
request = json.loads(q)
if request.get('domain'):
toquery = request['domain']
elif request.get('ip-src'):
toquery = request['ip-src']
elif request.get('ip-dst'):
toquery = request['ip-dst']
else:
misperrors['error'] = "Unsupported attributes type"
return misperrors
if not request.get('config') or (not request['config'].get('server') and not request['config'].get('port')):
misperrors['error'] = 'Whois local instance address is missing'
return misperrors
uwhois = Uwhois(request['config']['server'], int(request['config']['port']))
if 'event_id' in request:
return handle_expansion(uwhois, toquery)
def handle_expansion(w, domain):
return {'results': [{'types': mispattributes['output'], 'values': w.query(domain)}]}
def introspection():
return mispattributes
def version():
moduleinfo['config'] = moduleconfig
return moduleinfo

6058
poetry.lock generated Normal file
View File

File diff suppressed because it is too large Load Diff

121
pyproject.toml
View File

@ -1,3 +1,120 @@
[build-system]
requires = ["setuptools", "wheel"]
build-backend = "setuptools.build_meta:__legacy__"
requires = ["poetry-core>=1.0.0"]
build-backend = "poetry.core.masonry.api"
[tool.poetry]
name = "misp-modules"
version = "2.4.195"
description = "MISP modules are autonomous modules that can be used for expansion and other services in MISP"
authors = ["Alexandre Dulaunoy <alexandre.dulaunoy@circl.lu>"]
license = "AGPL-3.0-only"
repository = "https://github.com/MISP/misp-modules"
documentation = "https://misp.github.io/misp-modules"
readme = "README.md"
classifiers=[
'License :: OSI Approved :: GNU Affero General Public License v3',
'Development Status :: 5 - Production/Stable',
'Environment :: Console',
'Intended Audience :: Science/Research',
'Programming Language :: Python :: 3',
'Topic :: Security',
]
packages = [{include = "misp_modules"}]
[tool.poetry.urls]
"Bug Tracker" = "https://github.com/MISP/misp-modules/issues"
"Source" = "https://github.com/MISP/misp-modules"
[tool.poetry.scripts]
misp-modules = "misp_modules:main"
[tool.poetry.dependencies]
## platform (pin this to your python version, for 'poetry export' to work)
python = ">=3.8.*,<3.13"
## core dependencies
psutil = "*"
pyparsing = "*"
redis = "*"
tornado = "*"
## module dependencies (if a dependency fails loading with '*', pin it here)
censys = "2.0.9"
socialscan = "<2.0.0"
yara-python = "4.5.0"
# required to support both python 3.8 and wheel builds on python 3.12
numpy = [{version = "1.24.4", python = "3.8.*"}, {version = ">=1.26.4,<2.0.0", python = ">=3.9"}]
pandas = [{version = "1.5.3", python = "3.8.*"}, {version = ">=2.0.0", python = ">=3.9"}]
pandas_ods_reader = [{version = "0.1.4", python = "3.8.*"}, {version = ">=1.0.0", python = ">=3.9"}]
## module dependencies
apiosintds = "*"
assemblyline_client = "*"
backscatter = "*"
blockchain = "*"
clamd = "*"
crowdstrike-falconpy = "*"
dnsdb2 = "*"
domaintools_api = "*"
geoip2 = "*"
google-search-api = { git = "https://github.com/abenassi/Google-Search-API" }
greynoise = "*"
jbxapi = "*"
maclookup = "*"
markdownify = "*"
matplotlib = "*"
mattermostdriver = "*"
mwdblib = "*"
ndjson = "*"
np = "*"
oauth2 = "*"
odtreader = { git = "https://github.com/cartertemm/ODTReader.git" }
opencv-python = "*"
openpyxl = "*"
passivetotal = "*"
pdftotext = "*"
pycountry = "*"
pydnstrails = { git = "https://github.com/sebdraven/pydnstrails.git" }
pyeti-python3 = "*"
pyeupi = "*"
pyfaup = "*"
pygeoip = "*"
pyintel471 = { git = "https://github.com/MISP/PyIntel471.git" }
pyipasnhistory = "*"
pymisp = { version = "*", extras = ["fileobjects", "openioc", "pdfexport", "email", "url"] }
pyonyphe = { git = "https://github.com/sebdraven/pyonyphe.git" }
pypdns = "*"
pypssl = "*"
pysafebrowsing = "*"
pytesseract = "*"
python-docx = "*"
python-pptx = "*"
pyzbar = "*"
requests = { version = "*", extras = ["security"] }
shodan = "*"
sigmatools = "*"
sigmf = "*"
slack-sdk = "*"
sparqlwrapper = "*"
stix2 = "*"
tau-clients = "*"
taxii2-client = "*"
trustar = { git = "https://github.com/SteveClement/trustar-python.git" }
urlarchiver = "*"
vt-graph-api = "*"
vt-py = "*"
vulners = "*"
vysion = "*"
wand = "*"
xlrd = "*"
[tool.poetry.group.dev.dependencies]
codecov = "*"
flake8 = "*"
nose = "*"
pytest = "*"
[tool.poetry.group.docs]
optional = true
[tool.poetry.group.docs.dependencies]
mkdocs = "*"
mkdocs-material = "*"
markdown_include = "*"

31
setup.py
View File

@ -1,31 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from setuptools import setup, find_packages
setup(
name='misp-modules',
version='1.0',
author='Alexandre Dulaunoy',
author_email='alexandre.dulaunoy@circl.lu',
maintainer='Alexandre Dulaunoy',
url='https://github.com/MISP/misp-modules',
description='MISP modules are autonomous modules that can be used for expansion and other services in MISP',
packages=find_packages(),
entry_points={'console_scripts': ['misp-modules = misp_modules:main']},
scripts=['tools/update_misp_modules.sh'],
test_suite="tests",
classifiers=[
'License :: OSI Approved :: GNU Affero General Public License v3',
'Development Status :: 5 - Production/Stable',
'Environment :: Console',
'Intended Audience :: Science/Research',
'Programming Language :: Python :: 3',
'Topic :: Security',
],
install_requires=[
'tornado',
'psutil',
'redis>=3',
'pyparsing==2.4.7'
],
)

12
tests/test_expansions.py
View File

@ -133,18 +133,6 @@ class TestExpansions(unittest.TestCase):
response = self.misp_modules_post(query)
self.assertEqual(self.get_errors(response), 'An API key for APIVoid is required.')
def test_bgpranking(self):
query = {
"module": "bgpranking",
"attribute": {
"type": "AS",
"value": "13335",
"uuid": "ea89a33b-4ab7-4515-9f02-922a0bee333d"
}
}
response = self.misp_modules_post(query)
self.assertEqual(self.get_first_object_type(response), 'asn')
def test_btc_steroids(self):
if LiveCI:
return True

37
tools/update_misp_modules.sh
View File

@ -1,37 +0,0 @@
#!/usr/bin/env bash
set -e
set -x
# Updates the MISP Modules while respecting the current permissions
# It aims to support the two following installation methods:
# * Everything is runinng on the same machine following the MISP installation guide.
# * The modules are installed using pipenv on a different machine from the one where MISP is running.
if [ -d "/var/www/MISP" ] && [ -d "/usr/local/src/misp-modules" ]
then
echo "MISP is installed on the same machine, following the recommanded install script. Using MISP virtualenv."
PATH_TO_MISP="/var/www/MISP"
PATH_TO_MISP_MODULES="/usr/local/src/misp-modules"
pushd ${PATH_TO_MISP_MODULES}
USER=`stat -c "%U" .`
sudo -H -u ${USER} git pull
sudo -H -u ${USER} ${PATH_TO_MISP}/venv/bin/pip install -U -r REQUIREMENTS
sudo -H -u ${USER} ${PATH_TO_MISP}/venv/bin/pip install -U -e .
service misp-modules restart
popd
else
if ! [ -x "$(command -v pipenv)" ]; then
echo 'Error: pipenv not available, unable to automatically update.' >&2
exit 1
fi
echo "Standalone mode, use pipenv from the current directory."
git pull
pipenv install
fi

0
var/.gitkeep
View File