mirror of https://github.com/MISP/misp-modules
Migrate to poetry and optimize dependencies
parent
3a13af724f
commit
b89a33b74f
|
@ -19,50 +19,55 @@ jobs:
|
|||
- name: Install packages
|
||||
run: |
|
||||
sudo apt-get install libpoppler-cpp-dev libzbar0 tesseract-ocr
|
||||
|
||||
- name: Compile and install yara
|
||||
run: |
|
||||
sudo apt-get install automake libtool make gcc pkg-config
|
||||
wget https://github.com/VirusTotal/yara/archive/refs/tags/v4.5.0.tar.gz
|
||||
tar xzf *.tar.gz
|
||||
wget https://github.com/VirusTotal/yara/archive/refs/tags/v4.5.0.tar.gz -O yara.tar.gz
|
||||
tar xzf yara.tar.gz
|
||||
pushd yara-*
|
||||
./bootstrap.sh
|
||||
./configure
|
||||
make
|
||||
sudo make install
|
||||
popd
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
cache: 'pip'
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install flake8 pytest
|
||||
pip install pipenv
|
||||
sed -i "s/python_version.*/python_version = \"${{ matrix.python-version }}\"/" Pipfile
|
||||
pipenv lock
|
||||
pipenv requirements > requirements.txt
|
||||
# pyfaul must be installed manually (?)
|
||||
pip install -r requirements.txt pyfaup
|
||||
pip install .
|
||||
python -m pip install --upgrade poetry
|
||||
poetry install
|
||||
# Tell poetry to not use a virtual environment
|
||||
# poetry config virtualenvs.create false
|
||||
|
||||
- name: Lint with flake8
|
||||
run: |
|
||||
# stop the build if there are Python syntax errors or undefined names
|
||||
flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
|
||||
poetry run flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
|
||||
# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
|
||||
flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
|
||||
poetry run flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
|
||||
|
||||
- name: Run server in background
|
||||
run: |
|
||||
misp-modules -l 127.0.0.1 -s 2>error.log &
|
||||
sleep 3
|
||||
poetry run misp-modules -l 127.0.0.1 -s 2>error.log &
|
||||
sleep 10
|
||||
|
||||
- name: Check if server is running
|
||||
run: |
|
||||
curl -sS localhost:6666/healthcheck
|
||||
|
||||
- name: Test with pytest
|
||||
run: |
|
||||
pytest tests
|
||||
poetry run pytest
|
||||
|
||||
- name: Show error log
|
||||
if: always()
|
||||
run: |
|
||||
|
|
|
@ -24,3 +24,4 @@ venv*
|
|||
.vscode*
|
||||
*.sqlite
|
||||
website/conf/config.cfg
|
||||
wheels
|
||||
|
|
56
.travis.yml
56
.travis.yml
|
@ -1,56 +0,0 @@
|
|||
language: python
|
||||
|
||||
services:
|
||||
- redis-server
|
||||
|
||||
cache: pip
|
||||
|
||||
python:
|
||||
- "3.6"
|
||||
- "3.6-dev"
|
||||
- "3.7-dev"
|
||||
- "3.8-dev"
|
||||
|
||||
install:
|
||||
- sudo apt-get install libzbar0 libzbar-dev libpoppler-cpp-dev tesseract-ocr libfuzzy-dev libcaca-dev liblua5.3-dev
|
||||
- pip install pipenv
|
||||
- pip install -r REQUIREMENTS
|
||||
# - pipenv install --dev
|
||||
# install gtcaca
|
||||
- git clone git://github.com/stricaud/gtcaca.git
|
||||
- mkdir -p gtcaca/build
|
||||
- pushd gtcaca/build
|
||||
- cmake .. && make
|
||||
- sudo make install
|
||||
- popd
|
||||
# install pyfaup
|
||||
- git clone https://github.com/stricaud/faup.git
|
||||
- pushd faup/build
|
||||
- cmake .. && make
|
||||
- sudo make install
|
||||
- popd
|
||||
- sudo ldconfig
|
||||
- pushd faup/src/lib/bindings/python
|
||||
- pip install .
|
||||
- popd
|
||||
|
||||
script:
|
||||
- pip install coverage
|
||||
- coverage run -m --parallel-mode --source=misp_modules misp_modules.__init__ -l 127.0.0.1 &
|
||||
- pid=$!
|
||||
- sleep 5
|
||||
- nosetests --with-coverage --cover-package=misp_modules
|
||||
- kill -s KILL $pid
|
||||
- pushd ~/
|
||||
- coverage run -m --parallel-mode --source=misp_modules misp_modules.__init__ -s -l 127.0.0.1 &
|
||||
- pid=$!
|
||||
- popd
|
||||
- sleep 5
|
||||
- nosetests --with-coverage --cover-package=misp_modules
|
||||
- kill -s KILL $pid
|
||||
- pip install flake8
|
||||
- flake8 --ignore=E501,W503,E226,E126 misp_modules
|
||||
|
||||
after_success:
|
||||
- coverage combine .coverage*
|
||||
- codecov
|
|
@ -1,3 +0,0 @@
|
|||
mkdocs
|
||||
pymdown-extensions
|
||||
mkdocs-material
|
81
Makefile
81
Makefile
|
@ -1,38 +1,61 @@
|
|||
# https://www.mkdocs.org/user-guide/deploying-your-docs/
|
||||
# See: https://www.mkdocs.org/user-guide/deploying-your-docs/
|
||||
# Running 'make' uses poetry-installed mkdocs
|
||||
# Running 'USE_DOCKER=true make' uses docker mkdocs
|
||||
|
||||
.PHONY: prepare_docs generate_docs deploy test_docs
|
||||
|
||||
MKDOCS_DOCKER_IMAGE := squidfunk/mkdocs-material
|
||||
|
||||
DOCS_DIST_DIR := ./docs
|
||||
|
||||
DOCS_SRC_DIR := ./documentation
|
||||
|
||||
USE_DOCKER ?=
|
||||
|
||||
.DEFAULT_GOAL := generate_docs
|
||||
|
||||
.PHONY: prepare_docs generate_docs ci_generate_docs test_docs
|
||||
|
||||
prepare_docs:
|
||||
pip3 install .
|
||||
cd documentation; python3 generate_documentation.py
|
||||
mkdir -p docs/expansion/logos docs/export_mod/logos docs/import_mod/logos
|
||||
mkdir -p docs/logos
|
||||
cd documentation; cp -R ./logos/* ../docs/logos
|
||||
cd documentation; cp -R ./logos/* ../docs/expansion/logos
|
||||
cd documentation; cp -R ./logos/* ../docs/export_mod/logos
|
||||
cd documentation; cp -R ./logos/* ../docs/import_mod/logos
|
||||
cp ./documentation/mkdocs/*.md ./docs
|
||||
cp LICENSE ./docs/license.md
|
||||
@echo "Preparing documentation."
|
||||
poetry install --only docs
|
||||
poetry run python $(DOCS_SRC_DIR)/generate_documentation.py
|
||||
mkdir -p $(DOCS_DIST_DIR)/logos
|
||||
mkdir -p $(DOCS_DIST_DIR)/expansion/logos
|
||||
mkdir -p $(DOCS_DIST_DIR)/export_mod/logos
|
||||
mkdir -p $(DOCS_DIST_DIR)/import_mod/logos
|
||||
cp -R $(DOCS_SRC_DIR)/logos/* $(DOCS_DIST_DIR)/logos
|
||||
cp -R $(DOCS_SRC_DIR)/logos/* $(DOCS_DIST_DIR)/expansion/logos
|
||||
cp -R $(DOCS_SRC_DIR)/logos/* $(DOCS_DIST_DIR)/export_mod/logos
|
||||
cp -R $(DOCS_SRC_DIR)/logos/* $(DOCS_DIST_DIR)/import_mod/logos
|
||||
cp $(DOCS_SRC_DIR)/mkdocs/*.md $(DOCS_DIST_DIR)
|
||||
cp LICENSE $(DOCS_DIST_DIR)/license.md
|
||||
|
||||
install_requirements:
|
||||
pip install -r docs/REQUIREMENTS.txt
|
||||
|
||||
generate_docs: prepare_docs
|
||||
mkdocs build
|
||||
ifeq ($(USE_DOCKER), true)
|
||||
@echo "Generating documentation using '$(MKDOCS_DOCKER_IMAGE)'."
|
||||
docker run --rm -it -v $(PWD):/docs $(MKDOCS_DOCKER_IMAGE) build
|
||||
else
|
||||
@echo "Generating docunentation."
|
||||
poetry run mkdocs build
|
||||
endif
|
||||
|
||||
|
||||
deploy: generate_docs
|
||||
ifeq ($(USE_DOCKER), true)
|
||||
@echo "Deploying documentation using '$(MKDOCS_DOCKER_IMAGE)'."
|
||||
docker run --rm -it -v $(PWD):/docs -v /home/$(whoami)/.docker:/root/.docker:ro $(MKDOCS_DOCKER_IMAGE) gh-deploy
|
||||
else
|
||||
@echo "Deploying docunentation."
|
||||
poetry run mkdocs gh-deploy
|
||||
endif
|
||||
|
||||
deploy:
|
||||
mkdocs gh-deploy
|
||||
|
||||
test_docs: prepare_docs
|
||||
mkdocs serve
|
||||
|
||||
|
||||
# DOCKER make commands
|
||||
generate_docs_docker: prepare_docs
|
||||
docker run --rm -it -v $(PWD):/docs squidfunk/mkdocs-material build
|
||||
|
||||
deploy_docker:
|
||||
docker run --rm -it -v $(PWD):/docs -v /home/$(whoami)/.docker:/root/.docker:ro squidfunk/mkdocs-material gh-deploy
|
||||
|
||||
test_docs_docker: prepare_docs
|
||||
docker run --rm -it -p 8000:8000 -v $(PWD):/docs squidfunk/mkdocs-material
|
||||
ifeq ($(USE_DOCKER), true)
|
||||
@echo "Serving documentation using '$(MKDOCS_DOCKER_IMAGE)'."
|
||||
docker run --rm -it -v $(PWD):/docs -p 8000:8000 $(MKDOCS_DOCKER_IMAGE)
|
||||
else
|
||||
@echo "Serving docunentation."
|
||||
poetry run mkdocs serve
|
||||
endif
|
||||
|
|
87
Pipfile
87
Pipfile
|
@ -1,87 +0,0 @@
|
|||
[[source]]
|
||||
name = "pypi"
|
||||
url = "https://pypi.org/simple"
|
||||
verify_ssl = true
|
||||
|
||||
[dev-packages]
|
||||
nose = "*"
|
||||
codecov = "*"
|
||||
pytest = "*"
|
||||
flake8 = "*"
|
||||
|
||||
[packages]
|
||||
numpy = "<2.0.0"
|
||||
matplotlib = "*"
|
||||
sigmf = "*"
|
||||
pysafebrowsing = "*"
|
||||
dnspython = "*"
|
||||
requests = { extras = ["security"], version = "*" }
|
||||
urlarchiver = "*"
|
||||
passivetotal = "*"
|
||||
pypdns = "*"
|
||||
pypssl = "*"
|
||||
pyeupi = "*"
|
||||
pymisp = { extras = ["fileobjects,openioc,pdfexport,email,url"], version = "*" }
|
||||
pyonyphe = { git = "https://github.com/sebdraven/pyonyphe" }
|
||||
pydnstrails = { git = "https://github.com/sebdraven/pydnstrails" }
|
||||
pytesseract = "*"
|
||||
pygeoip = "*"
|
||||
beautifulsoup4 = "*"
|
||||
oauth2 = "*"
|
||||
yara-python = "==4.5.0"
|
||||
sigmatools = "*"
|
||||
stix2 = "*"
|
||||
stix2-patterns = "*"
|
||||
taxii2-client = "*"
|
||||
maclookup = "*"
|
||||
vulners = "*"
|
||||
blockchain = "*"
|
||||
reportlab = "*"
|
||||
pyintel471 = { git = "https://github.com/MISP/PyIntel471.git" }
|
||||
shodan = "*"
|
||||
Pillow = ">=8.2.0"
|
||||
Wand = "*"
|
||||
SPARQLWrapper = "*"
|
||||
domaintools_api = "*"
|
||||
misp-modules = { path = "." }
|
||||
pybgpranking = { git = "https://github.com/D4-project/BGP-Ranking.git/", subdirectory = "client", ref = "68de39f6c5196f796055c1ac34504054d688aa59" }
|
||||
pyipasnhistory = { git = "https://github.com/D4-project/IPASN-History.git/", subdirectory = "client", ref = "a2853c39265cecdd0c0d16850bd34621c0551b87" }
|
||||
backscatter = "*"
|
||||
pyzbar = "*"
|
||||
opencv-python = "*"
|
||||
np = "*"
|
||||
ODTReader = { git = "https://github.com/cartertemm/ODTReader.git/" }
|
||||
python-pptx = "*"
|
||||
python-docx = "*"
|
||||
ezodf = "*"
|
||||
pandas = "*"
|
||||
pandas_ods_reader = "*"
|
||||
pdftotext = "*"
|
||||
lxml = "*"
|
||||
xlrd = "*"
|
||||
jbxapi = "*"
|
||||
geoip2 = "*"
|
||||
apiosintDS = "*"
|
||||
assemblyline_client = "*"
|
||||
vt-graph-api = "*"
|
||||
trustar = { git = "https://github.com/SteveClement/trustar-python.git" }
|
||||
markdownify = "==0.5.3"
|
||||
socialscan = "==1.4"
|
||||
pycountry = "==22.3.5"
|
||||
dnsdb2 = "*"
|
||||
clamd = "*"
|
||||
aiohttp = ">=3.7.4"
|
||||
tau-clients = "*"
|
||||
vt-py = ">=0.7.1"
|
||||
crowdstrike-falconpy = "0.9.0"
|
||||
censys = "2.0.9"
|
||||
mwdblib = "3.4.1"
|
||||
ndjson = "0.3.1"
|
||||
Jinja2 = ">=3.1.2"
|
||||
mattermostdriver = "7.3.2"
|
||||
openpyxl = "*"
|
||||
slack-sdk = "3.27.1"
|
||||
vysion = "*"
|
||||
|
||||
[requires]
|
||||
python_version = "3.12"
|
File diff suppressed because it is too large
Load Diff
|
@ -42,7 +42,6 @@ For further Information see the [license file](https://misp.github.io/misp-modul
|
|||
* [AssemblyLine Query](https://misp.github.io/misp-modules/expansion/#assemblyline-query) - A module tu query the AssemblyLine API with a submission ID to get the submission report and parse it.
|
||||
* [AssemblyLine Submit](https://misp.github.io/misp-modules/expansion/#assemblyline-submit) - A module to submit samples and URLs to AssemblyLine for advanced analysis, and return the link of the submission.
|
||||
* [Backscatter.io](https://misp.github.io/misp-modules/expansion/#backscatter.io) - Backscatter.io module to bring mass-scanning observations into MISP.
|
||||
* [BGP Ranking](https://misp.github.io/misp-modules/expansion/#bgp-ranking) - Query BGP Ranking to get the ranking of an Autonomous System number.
|
||||
* [BTC Scam Check](https://misp.github.io/misp-modules/expansion/#btc-scam-check) - An expansion hover module to query a special dns blacklist to check if a bitcoin address has been abused.
|
||||
* [BTC Steroids](https://misp.github.io/misp-modules/expansion/#btc-steroids) - An expansion hover module to get a blockchain balance from a BTC address in MISP.
|
||||
* [Censys Enrich](https://misp.github.io/misp-modules/expansion/#censys-enrich) - An expansion module to enrich attributes in MISP by quering the censys.io API
|
||||
|
@ -136,7 +135,6 @@ For further Information see the [license file](https://misp.github.io/misp-modul
|
|||
* [Vulnerability Lookup](https://misp.github.io/misp-modules/expansion/#vulnerability-lookup) - An expansion module to query Vulnerability Lookup
|
||||
* [Vulners Lookup](https://misp.github.io/misp-modules/expansion/#vulners-lookup) - An expansion hover module to expand information about CVE id using Vulners API.
|
||||
* [Vysion Enrich](https://misp.github.io/misp-modules/expansion/#vysion-enrich) - Module to enrich the information by making use of the Vysion API.
|
||||
* [Whois Lookup](https://misp.github.io/misp-modules/expansion/#whois-lookup) - Module to query a local instance of uwhois (https://github.com/rafiot/uwhoisd).
|
||||
* [WhoisFreaks Lookup](https://misp.github.io/misp-modules/expansion/#whoisfreaks-lookup) - An expansion module for https://whoisfreaks.com/ that will provide an enriched analysis of the provided domain, including WHOIS and DNS information.
|
||||
* [Wikidata Lookup](https://misp.github.io/misp-modules/expansion/#wikidata-lookup) - An expansion hover module to extract information from Wikidata to have additional information about particular term for analysis.
|
||||
* [IBM X-Force Exchange Lookup](https://misp.github.io/misp-modules/expansion/#ibm-x-force-exchange-lookup) - An expansion module for IBM X-Force Exchange.
|
||||
|
|
188
REQUIREMENTS
188
REQUIREMENTS
|
@ -1,188 +0,0 @@
|
|||
-i https://pypi.org/simple
|
||||
aiohappyeyeballs==2.3.5; python_version >= '3.8'
|
||||
aiohttp==3.10.2; python_version >= '3.8'
|
||||
aiosignal==1.3.1; python_version >= '3.7'
|
||||
annotated-types==0.7.0; python_version >= '3.8'
|
||||
antlr4-python3-runtime==4.9.3
|
||||
anyio==4.4.0; python_version >= '3.8'
|
||||
apiosintds==2.0.3; python_version >= '3.6'
|
||||
appdirs==1.4.4
|
||||
assemblyline-client==4.9.3
|
||||
attrs==24.2.0; python_version >= '3.7'
|
||||
backoff==1.11.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
|
||||
backscatter==0.2.4
|
||||
beautifulsoup4==4.12.3; python_full_version >= '3.6.0'
|
||||
bidict==0.23.1; python_version >= '3.8'
|
||||
blockchain==1.4.4
|
||||
cattrs==23.2.3; python_version >= '3.8'
|
||||
censys==2.0.9; python_version < '4.0' and python_full_version >= '3.6.2'
|
||||
certifi==2024.7.4; python_version >= '3.6'
|
||||
cffi==1.17.0; platform_python_implementation != 'PyPy'
|
||||
chardet==5.2.0; python_version >= '3.7'
|
||||
charset-normalizer==3.3.2; python_full_version >= '3.7.0'
|
||||
clamd==1.0.2
|
||||
click==8.1.7; python_version >= '3.7'
|
||||
click-plugins==1.1.1
|
||||
colorama==0.4.6; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6'
|
||||
colorclass==2.2.2; python_version >= '2.6'
|
||||
commonmark==0.9.1
|
||||
compressed-rtf==1.0.6
|
||||
configparser==7.0.0; python_version >= '3.8'
|
||||
contourpy==1.2.1; python_version >= '3.9'
|
||||
crowdstrike-falconpy==0.9.0; python_version >= '3.6'
|
||||
cryptography==43.0.0; python_version >= '3.7'
|
||||
cycler==0.12.1; python_version >= '3.8'
|
||||
deprecated==1.2.14; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||
dnsdb2==1.1.4
|
||||
dnspython==2.6.1; python_version >= '3.8'
|
||||
domaintools-api==2.0.0; python_version >= '3.6'
|
||||
easygui==0.98.3
|
||||
ebcdic==1.1.1
|
||||
enum-compat==0.0.3
|
||||
et-xmlfile==1.1.0; python_version >= '3.6'
|
||||
extract-msg==0.48.7
|
||||
ezodf==0.3.2
|
||||
filelock==3.15.4; python_version >= '3.8'
|
||||
fonttools==4.53.1; python_version >= '3.8'
|
||||
frozenlist==1.4.1; python_version >= '3.8'
|
||||
future==1.0.0; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||
geoip2==4.8.0; python_version >= '3.8'
|
||||
h11==0.14.0; python_version >= '3.7'
|
||||
httpcore==1.0.5; python_version >= '3.8'
|
||||
httplib2==0.22.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||
httpx==0.27.0; python_version >= '3.8'
|
||||
idna==3.7; python_version >= '3.5'
|
||||
isodate==0.6.1
|
||||
jbxapi==3.23.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
|
||||
jinja2==3.1.4; python_version >= '3.7'
|
||||
json-log-formatter==1.0; python_version >= '3.6'
|
||||
jsonschema==4.23.0; python_version >= '3.8'
|
||||
jsonschema-specifications==2023.12.1; python_version >= '3.8'
|
||||
kiwisolver==1.4.5; python_version >= '3.7'
|
||||
lark==1.1.9; python_version >= '3.6'
|
||||
lief==0.15.1
|
||||
lxml==5.2.2; python_version >= '3.6'
|
||||
maclookup==1.0.3
|
||||
markdownify==0.5.3
|
||||
markupsafe==2.1.5; python_version >= '3.7'
|
||||
matplotlib==3.9.1.post1; python_version >= '3.9'
|
||||
mattermostdriver==7.3.2; python_version >= '3.5'
|
||||
maxminddb==2.6.2; python_version >= '3.8'
|
||||
-e .
|
||||
more-itertools==10.4.0; python_version >= '3.8'
|
||||
msoffcrypto-tool==5.4.2; python_version >= '3' and platform_python_implementation != 'PyPy' or (platform_system != 'Windows' and platform_system != 'Darwin')
|
||||
multidict==6.0.5; python_version >= '3.7'
|
||||
mwdblib==3.4.1
|
||||
ndjson==0.3.1
|
||||
np==1.0.2
|
||||
numpy==1.26.4; python_version >= '3.9'
|
||||
oauth2==1.9.0.post1
|
||||
odtreader@ git+https://github.com/cartertemm/ODTReader.git/@49d6938693f6faa3ff09998f86dba551ae3a996b
|
||||
olefile==0.47; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
|
||||
oletools==0.60.2
|
||||
opencv-python==4.10.0.84; python_version >= '3.6'
|
||||
openpyxl==3.1.5; python_version >= '3.8'
|
||||
packaging==24.1; python_version >= '3.8'
|
||||
pandas==2.2.2; python_version >= '3.9'
|
||||
pandas-ods-reader==1.0.1; python_version < '3.13' and python_version >= '3.9'
|
||||
passivetotal==2.5.9
|
||||
pcodedmp==1.2.6
|
||||
pdftotext==2.2.2
|
||||
pillow==10.4.0; python_version >= '3.8'
|
||||
platformdirs==4.2.2; python_version >= '3.8'
|
||||
progressbar2==4.4.2; python_version >= '3.8'
|
||||
psutil==6.0.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
|
||||
publicsuffixlist==1.0.2.20240810; python_version >= '3.5'
|
||||
pybgpranking@ git+https://github.com/D4-project/BGP-Ranking.git/#subdirectory=client
|
||||
pycountry==22.3.5; python_version >= '3.6' and python_version < '4'
|
||||
pycparser==2.22; python_version >= '3.8'
|
||||
pycryptodome==3.20.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
|
||||
pycryptodomex==3.20.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
|
||||
pydantic==2.8.2; python_version >= '3.8'
|
||||
pydantic-core==2.20.1; python_version >= '3.8'
|
||||
pydeep2==0.5.1
|
||||
pydnstrails@ git+https://github.com/sebdraven/pydnstrails@48c1f740025c51289f43a24863d1845ff12fd21a
|
||||
pyeupi==1.3.0; python_version >= '3.8' and python_version < '4.0'
|
||||
pyfaup==1.2
|
||||
pygeoip==0.3.2
|
||||
pygments==2.18.0; python_version >= '3.8'
|
||||
pyintel471@ git+https://github.com/MISP/PyIntel471.git@917272fafa8e12102329faca52173e90c5256968
|
||||
pyipasnhistory@ git+https://github.com/D4-project/IPASN-History.git/#subdirectory=client
|
||||
pymisp[fileobjects,openioc,pdfexport,email,url]==2.4.195; python_version >= '3.8' and python_version < '4.0'
|
||||
pyonyphe@ git+https://github.com/sebdraven/pyonyphe@d1d6741f8ea4475f3bb77ff20c876f08839cabd1
|
||||
pyparsing==2.4.7; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||
pypdns==2.2.3; python_version >= '3.8' and python_version < '4.0'
|
||||
pypssl==2.2; python_version >= '3.6' and python_version < '4.0'
|
||||
pysafebrowsing==0.1.3
|
||||
pytesseract==0.3.10; python_version >= '3.7'
|
||||
python-baseconv==1.2.2
|
||||
python-dateutil==2.9.0.post0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||
python-docx==1.1.2; python_version >= '3.7'
|
||||
python-engineio==4.9.1; python_version >= '3.6'
|
||||
python-magic==0.4.27
|
||||
python-pptx==1.0.2; python_version >= '3.8'
|
||||
python-socketio[client]==5.11.3; python_version >= '3.8'
|
||||
python-utils==3.8.2; python_version >= '3.9'
|
||||
pytz==2024.1
|
||||
pyyaml==6.0.2; python_version >= '3.8'
|
||||
pyzbar==0.1.9
|
||||
pyzipper==0.3.6; python_version >= '3.5'
|
||||
rdflib==7.0.0; python_full_version >= '3.8.1' and python_full_version < '4.0.0'
|
||||
red-black-tree-mod==1.20
|
||||
redis==5.0.8; python_version >= '3.7'
|
||||
referencing==0.35.1; python_version >= '3.8'
|
||||
reportlab==4.2.2; python_version >= '3.7' and python_version < '4'
|
||||
requests[security]==2.32.3; python_version >= '3.8'
|
||||
requests-cache==1.2.1; python_version >= '3.8'
|
||||
requests-file==2.1.0
|
||||
rich==10.16.2; python_full_version >= '3.6.2' and python_full_version < '4.0.0'
|
||||
rpds-py==0.20.0; python_version >= '3.8'
|
||||
rtfde==0.1.2
|
||||
ruamel.yaml==0.18.6; python_version >= '3.7'
|
||||
ruamel.yaml.clib==0.2.8; python_version < '3.13' and platform_python_implementation == 'CPython'
|
||||
setuptools==72.1.0; python_version >= '3.8'
|
||||
shellingham==1.5.4; python_version >= '3.7'
|
||||
shodan==1.31.0
|
||||
sigmatools==0.23.1; python_version ~= '3.8'
|
||||
sigmf==1.2.2; python_version >= '3.7'
|
||||
simple-websocket==1.0.0; python_version >= '3.6'
|
||||
simplejson==3.19.2; python_version >= '2.5' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||
six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||
slack-sdk==3.27.1; python_version >= '3.6'
|
||||
sniffio==1.3.1; python_version >= '3.7'
|
||||
socialscan==1.4.0; python_version >= '3.6'
|
||||
socketio-client==0.5.7.4
|
||||
softenum==1.0.1; python_full_version >= '3.7.0' and python_full_version < '4.0.0'
|
||||
soupsieve==2.5; python_version >= '3.8'
|
||||
sparqlwrapper==2.0.0; python_version >= '3.7'
|
||||
stix2==3.0.1; python_version >= '3.6'
|
||||
stix2-patterns==2.0.0; python_version >= '3.6'
|
||||
tabulate==0.9.0; python_version >= '3.7'
|
||||
tau-clients==0.3.0; python_version >= '3.6'
|
||||
taxii2-client==2.3.0
|
||||
termcolor==2.4.0; python_version >= '3.8'
|
||||
tldextract==5.1.2; python_version >= '3.8'
|
||||
tornado==6.4.1; python_version >= '3.8'
|
||||
tqdm==4.66.5; python_version >= '3.7'
|
||||
trustar@ git+https://github.com/SteveClement/trustar-python.git@6954eae38e0c77eaeef26084b6c5fd033925c1c7
|
||||
typer==0.12.3; python_version >= '3.7'
|
||||
typing-extensions==4.12.2; python_version >= '3.8'
|
||||
tzdata==2024.1; python_version >= '2'
|
||||
tzlocal==5.2; python_version >= '3.8'
|
||||
unicodecsv==0.14.1
|
||||
url-normalize==1.4.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
|
||||
urlarchiver==0.2
|
||||
urllib3==2.2.2; python_version >= '3.8'
|
||||
vt-graph-api==2.2.0
|
||||
vt-py==0.18.3; python_full_version >= '3.7.0'
|
||||
vulners==2.2.0; python_version >= '3.8'
|
||||
vysion==2.0.8; python_full_version >= '3.8.0' and python_full_version < '4.0.0'
|
||||
wand==0.6.13
|
||||
websocket-client==1.8.0; python_version >= '3.8'
|
||||
websockets==12.0; python_version >= '3.8'
|
||||
wrapt==1.16.0; python_version >= '3.6'
|
||||
wsproto==1.2.0; python_full_version >= '3.7.0'
|
||||
xlrd==2.0.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
|
||||
xlsxwriter==3.2.0; python_version >= '3.6'
|
||||
yara-python==4.5.0
|
||||
yarl==1.9.4; python_version >= '3.7'
|
|
@ -1,130 +0,0 @@
|
|||
FROM python:3.7-buster AS build
|
||||
|
||||
ENV DEBIAN_FRONTEND noninteractive
|
||||
ENV WORKDIR="/usr/local/src/misp_modules"
|
||||
ENV VENV_DIR="/misp_modules"
|
||||
|
||||
# Install Packages for build
|
||||
RUN set -eu \
|
||||
;mkdir -p ${WORKDIR} ${VENV_DIR} \
|
||||
;apt-get update \
|
||||
;apt-get install -y \
|
||||
git \
|
||||
libpq5 \
|
||||
libjpeg-dev \
|
||||
tesseract-ocr \
|
||||
libpoppler-cpp-dev \
|
||||
imagemagick \
|
||||
virtualenv \
|
||||
libopencv-dev \
|
||||
zbar-tools \
|
||||
libzbar0 \
|
||||
libzbar-dev \
|
||||
libfuzzy-dev \
|
||||
;apt-get -y autoremove \
|
||||
;apt-get -y clean \
|
||||
;rm -rf /var/lib/apt/lists/* \
|
||||
;
|
||||
|
||||
# Create MISP Modules
|
||||
RUN set -eu \
|
||||
;git clone https://github.com/MISP/misp-modules.git ${WORKDIR} \
|
||||
;virtualenv -p python3 ${VENV_DIR}/venv \
|
||||
;cd ${WORKDIR} \
|
||||
;${VENV_DIR}/venv/bin/pip3 install -I -r REQUIREMENTS --no-cache-dir \
|
||||
;${VENV_DIR}/venv/bin/pip3 install . --no-cache-dir \
|
||||
;
|
||||
|
||||
#########################################
|
||||
# Start Final Docker Image
|
||||
#
|
||||
FROM python:3.7-slim-buster AS final
|
||||
|
||||
ENV DEBIAN_FRONTEND noninteractive
|
||||
ENV VENV_DIR="/misp_modules"
|
||||
|
||||
# Copy all builded files from build stage
|
||||
COPY --from=build ${VENV_DIR} ${VENV_DIR}
|
||||
|
||||
# Install Packages to run it
|
||||
RUN set -eu \
|
||||
;apt-get update \
|
||||
;apt-get install -y \
|
||||
curl \
|
||||
libpq5 \
|
||||
# libjpeg-dev \
|
||||
tesseract-ocr \
|
||||
libpoppler-cpp-dev \
|
||||
imagemagick \
|
||||
# virtualenv \
|
||||
# libopencv-dev \
|
||||
zbar-tools \
|
||||
libzbar0 \
|
||||
# libzbar-dev \
|
||||
# libfuzzy-dev \
|
||||
;apt-get -y autoremove \
|
||||
;apt-get -y clean \
|
||||
;rm -rf /var/lib/apt/lists/* \
|
||||
;chown -R nobody ${VENV_DIR} \
|
||||
;
|
||||
|
||||
# Entrypoint
|
||||
COPY files/entrypoint.sh /entrypoint.sh
|
||||
ENTRYPOINT [ "/entrypoint.sh" ]
|
||||
|
||||
# Add Healthcheck Config
|
||||
COPY files/healthcheck.sh /healthcheck.sh
|
||||
HEALTHCHECK --interval=1m --timeout=45s --retries=3 CMD ["/healthcheck.sh"]
|
||||
|
||||
# Change Workdir
|
||||
WORKDIR ${VENV_DIR}
|
||||
|
||||
# Change from root to www-data
|
||||
USER nobody
|
||||
|
||||
# Expose Port
|
||||
EXPOSE 6666
|
||||
|
||||
# Shortterm ARG Variables:
|
||||
ARG VENDOR="MISP"
|
||||
ARG COMPONENT="misp-modules"
|
||||
ARG BUILD_DATE
|
||||
ARG GIT_REPO="https://github.com/MISP/misp-modules"
|
||||
ARG VCS_REF
|
||||
ARG RELEASE_DATE
|
||||
ARG NAME="MISP-dockerized-misp-modules"
|
||||
ARG DESCRIPTION="This docker container contains MISP modules in an Debian Container."
|
||||
ARG DOCUMENTATION="https://misp.github.io/misp-modules/"
|
||||
ARG AUTHOR="MISP"
|
||||
ARG LICENSE="BSD-3-Clause"
|
||||
|
||||
# Longterm Environment Variables
|
||||
ENV \
|
||||
BUILD_DATE=${BUILD_DATE} \
|
||||
NAME=${NAME} \
|
||||
PATH=$PATH:${VENV_DIR}/venv/bin
|
||||
|
||||
# Labels
|
||||
LABEL org.label-schema.build-date="${BUILD_DATE}" \
|
||||
org.label-schema.name="${NAME}" \
|
||||
org.label-schema.description="${DESCRIPTION}" \
|
||||
org.label-schema.vcs-ref="${VCS_REF}" \
|
||||
org.label-schema.vcs-url="${GIT_REPO}" \
|
||||
org.label-schema.url="${GIT_REPO}" \
|
||||
org.label-schema.vendor="${VENDOR}" \
|
||||
org.label-schema.version="${VERSION}" \
|
||||
org.label-schema.usage="${DOCUMENTATION}" \
|
||||
org.label-schema.schema-version="1.0.0-rc1"
|
||||
|
||||
LABEL org.opencontainers.image.created="${BUILD_DATE}" \
|
||||
org.opencontainers.image.url="${GIT_REPO}" \
|
||||
org.opencontainers.image.source="${GIT_REPO}" \
|
||||
org.opencontainers.image.version="${VERSION}" \
|
||||
org.opencontainers.image.revision="${VCS_REF}" \
|
||||
org.opencontainers.image.vendor="${VENDOR}" \
|
||||
org.opencontainers.image.title="${NAME}" \
|
||||
org.opencontainers.image.description="${DESCRIPTION}" \
|
||||
org.opencontainers.image.documentation="${DOCUMENTATION}" \
|
||||
org.opencontainers.image.authors="${AUTHOR}" \
|
||||
org.opencontainers.image.licenses="${LICENSE}"
|
||||
|
|
@ -1,37 +0,0 @@
|
|||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
# Variables
|
||||
NC='\033[0m' # No Color
|
||||
Light_Green='\033[1;32m'
|
||||
STARTMSG="${Light_Green}[ENTRYPOINT_MISP_MODULES]${NC}"
|
||||
VENV_DIR=${VENV_DIR:-"/misp-modules"}
|
||||
MISP_MODULES_BINARY="${VENV_DIR}/venv/bin/misp-modules"
|
||||
DEBUG=""
|
||||
|
||||
# Functions
|
||||
echo (){
|
||||
command echo "$STARTMSG $*"
|
||||
}
|
||||
|
||||
# Environment Variables
|
||||
MISP_MODULES_DEBUG=${MISP_MODULES_DEBUG:-"false"}
|
||||
|
||||
#
|
||||
# MAIN
|
||||
#
|
||||
|
||||
|
||||
# Check if debugging mode should be enabled
|
||||
[ "$MISP_MODULES_DEBUG" = "true" ] && DEBUG="-d"
|
||||
|
||||
# check if a command parameter exists and start misp-modules
|
||||
if [ $# = 0 ]
|
||||
then
|
||||
# If no cmd parameter is set
|
||||
echo "Start MISP Modules" && $MISP_MODULES_BINARY $DEBUG -l 0.0.0.0 > /dev/stdout 2> /dev/stderr
|
||||
else
|
||||
# If cmd parameter is set
|
||||
echo "Start MISP Modules" && $MISP_MODULES_BINARY $DEBUG -l 0.0.0.0 > /dev/stdout 2> /dev/stderr &
|
||||
exec "$@"
|
||||
fi
|
|
@ -1,4 +0,0 @@
|
|||
#!/bin/sh
|
||||
|
||||
# If no contain is there or curl get an error back: exit 1. Docker restart then the container.
|
||||
curl -fk http://0.0.0.0:6666/modules || exit 1
|
|
@ -21,7 +21,6 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj
|
|||
* [AssemblyLine Query](https://misp.github.io/misp-modules/expansion/#assemblyline-query) - A module tu query the AssemblyLine API with a submission ID to get the submission report and parse it.
|
||||
* [AssemblyLine Submit](https://misp.github.io/misp-modules/expansion/#assemblyline-submit) - A module to submit samples and URLs to AssemblyLine for advanced analysis, and return the link of the submission.
|
||||
* [Backscatter.io](https://misp.github.io/misp-modules/expansion/#backscatter.io) - Backscatter.io module to bring mass-scanning observations into MISP.
|
||||
* [BGP Ranking](https://misp.github.io/misp-modules/expansion/#bgp-ranking) - Query BGP Ranking to get the ranking of an Autonomous System number.
|
||||
* [BTC Scam Check](https://misp.github.io/misp-modules/expansion/#btc-scam-check) - An expansion hover module to query a special dns blacklist to check if a bitcoin address has been abused.
|
||||
* [BTC Steroids](https://misp.github.io/misp-modules/expansion/#btc-steroids) - An expansion hover module to get a blockchain balance from a BTC address in MISP.
|
||||
* [Censys Enrich](https://misp.github.io/misp-modules/expansion/#censys-enrich) - An expansion module to enrich attributes in MISP by quering the censys.io API
|
||||
|
@ -115,7 +114,6 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj
|
|||
* [Vulnerability Lookup](https://misp.github.io/misp-modules/expansion/#vulnerability-lookup) - An expansion module to query Vulnerability Lookup
|
||||
* [Vulners Lookup](https://misp.github.io/misp-modules/expansion/#vulners-lookup) - An expansion hover module to expand information about CVE id using Vulners API.
|
||||
* [Vysion Enrich](https://misp.github.io/misp-modules/expansion/#vysion-enrich) - Module to enrich the information by making use of the Vysion API.
|
||||
* [Whois Lookup](https://misp.github.io/misp-modules/expansion/#whois-lookup) - Module to query a local instance of uwhois (https://github.com/rafiot/uwhoisd).
|
||||
* [WhoisFreaks Lookup](https://misp.github.io/misp-modules/expansion/#whoisfreaks-lookup) - An expansion module for https://whoisfreaks.com/ that will provide an enriched analysis of the provided domain, including WHOIS and DNS information.
|
||||
* [Wikidata Lookup](https://misp.github.io/misp-modules/expansion/#wikidata-lookup) - An expansion hover module to extract information from Wikidata to have additional information about particular term for analysis.
|
||||
* [IBM X-Force Exchange Lookup](https://misp.github.io/misp-modules/expansion/#ibm-x-force-exchange-lookup) - An expansion module for IBM X-Force Exchange.
|
||||
|
|
258
docs/install.md
258
docs/install.md
|
@ -1,199 +1,147 @@
|
|||
## How to install and start MISP modules (in a Python virtualenv)? (recommended)
|
||||
IMPORTANT NOTE: we will soon be publishing `misp-modules` on PyPI.
|
||||
|
||||
***Be sure to run the latest version of `pip`***. To install the latest version of pip, `pip install --upgrade pip` will do the job.
|
||||
|
||||
## Install from pip
|
||||
|
||||
It is strongly recommended to use a virtual environment (see here for instructions https://docs.python.org/3/tutorial/venv.html).
|
||||
|
||||
Once the virtual environment is loaded just use the command:
|
||||
|
||||
~~~~bash
|
||||
SUDO_WWW="sudo -u www-data"
|
||||
|
||||
sudo apt-get install -y \
|
||||
python3-dev \
|
||||
python3-pip \
|
||||
git \
|
||||
libpq5 \
|
||||
libjpeg-dev \
|
||||
tesseract-ocr \
|
||||
libpoppler-cpp-dev \
|
||||
imagemagick virtualenv \
|
||||
libopencv-dev \
|
||||
zbar-tools \
|
||||
libzbar0 \
|
||||
libzbar-dev \
|
||||
libfuzzy-dev \
|
||||
libcaca-dev \
|
||||
build-essential
|
||||
|
||||
# BEGIN with virtualenv:
|
||||
$SUDO_WWW virtualenv -p python3 /var/www/MISP/venv
|
||||
# END with virtualenv
|
||||
|
||||
cd /usr/local/src/
|
||||
# Ideally you add your user to the staff group and make /usr/local/src group writeable, below follows an example with user misp
|
||||
sudo adduser misp staff
|
||||
sudo chmod 2775 /usr/local/src
|
||||
sudo chown root:staff /usr/local/src
|
||||
git clone https://github.com/MISP/misp-modules.git
|
||||
git clone git://github.com/stricaud/faup.git faup
|
||||
git clone git://github.com/stricaud/gtcaca.git gtcaca
|
||||
|
||||
# Install gtcaca/faup
|
||||
cd gtcaca
|
||||
mkdir -p build
|
||||
cd build
|
||||
cmake .. && make
|
||||
sudo make install
|
||||
cd ../../faup
|
||||
mkdir -p build
|
||||
cd build
|
||||
cmake .. && make
|
||||
sudo make install
|
||||
sudo ldconfig
|
||||
|
||||
cd ../../misp-modules
|
||||
|
||||
# BEGIN with virtualenv:
|
||||
$SUDO_WWW /var/www/MISP/venv/bin/pip install -I -r REQUIREMENTS
|
||||
$SUDO_WWW /var/www/MISP/venv/bin/pip install .
|
||||
# END with virtualenv
|
||||
|
||||
# BEGIN without virtualenv:
|
||||
sudo pip install -I -r REQUIREMENTS
|
||||
sudo pip install .
|
||||
# END without virtualenv
|
||||
|
||||
# Start misp-modules as a service
|
||||
sudo cp etc/systemd/system/misp-modules.service /etc/systemd/system/
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl enable --now misp-modules
|
||||
sudo service misp-modules start # or
|
||||
/var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s & # to start the modules manually
|
||||
pip install misp-modules
|
||||
~~~~
|
||||
|
||||
## How to install and start MISP modules on RHEL-based distributions ?
|
||||
|
||||
As of this writing, the official RHEL repositories only contain Ruby 2.0.0 and Ruby 2.1 or higher is required. As such, this guide installs Ruby 2.2 from the [SCL](https://access.redhat.com/documentation/en-us/red_hat_software_collections/3/html/3.2_release_notes/chap-installation#sect-Installation-Subscribe) repository.
|
||||
## Install from cloned repository
|
||||
|
||||
In this case the only requirement is to install `poetry`. Normally you just need to run `pip install poetry`, but see here for more alternatives https://python-poetry.org/docs/#installation.
|
||||
|
||||
Once `poetry` is installed, you can clone the repository and install `misp-modules` as follows:
|
||||
|
||||
~~~~bash
|
||||
SUDO_WWW="sudo -u apache"
|
||||
sudo yum install \
|
||||
rh-python36 \
|
||||
rh-ruby22 \
|
||||
openjpeg-devel \
|
||||
rubygem-rouge \
|
||||
rubygem-asciidoctor \
|
||||
zbar-devel \
|
||||
opencv-devel \
|
||||
gcc-c++ \
|
||||
pkgconfig \
|
||||
poppler-cpp-devel \
|
||||
python-devel \
|
||||
redhat-rpm-config
|
||||
cd /var/www/MISP
|
||||
$SUDO_WWW git clone https://github.com/MISP/misp-modules.git
|
||||
cd misp-modules
|
||||
$SUDO_WWW /usr/bin/scl enable rh-python36 "virtualenv -p python3 /var/www/MISP/venv"
|
||||
$SUDO_WWW /var/www/MISP/venv/bin/pip install -U -I -r REQUIREMENTS
|
||||
$SUDO_WWW /var/www/MISP/venv/bin/pip install -U .
|
||||
git clone https://github.com/MISP/misp-modules.git && cd misp-modules
|
||||
git submodule update --init
|
||||
poetry install
|
||||
~~~~
|
||||
|
||||
Create the service file /etc/systemd/system/misp-modules.service :
|
||||
Note that the dependencies will require a number of system packages installed. On Ubuntu these packages are `libpoppler-cpp-dev`, `libzbar0`, and `tesseract-ocr`. For an updated list, check the github action used to test the build inside `.github/workflows`.
|
||||
|
||||
|
||||
## Install the systemd unit
|
||||
|
||||
To run `misp-modules` as a service on a distribution based on systemd, you need to create the unit as follows and store it in a file `/etc/systemd/system/misp-modules.service`:
|
||||
|
||||
~~~~bash
|
||||
echo "[Unit]
|
||||
Description=MISP's modules
|
||||
After=misp-workers.service
|
||||
[Unit]
|
||||
Description=MISP modules
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=apache
|
||||
Group=apache
|
||||
ExecStart=/usr/bin/scl enable rh-python36 rh-ruby22 '/var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s'
|
||||
ExecStart='/path/to/venv/bin/misp-modules -l 127.0.0.1 -s'
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target" | sudo tee /etc/systemd/system/misp-modules.service
|
||||
WantedBy=multi-user.target
|
||||
~~~~
|
||||
|
||||
The `After=misp-workers.service` must be changed or removed if you have not created a misp-workers service.
|
||||
Then, enable the misp-modules service and start it:
|
||||
~~~~bash
|
||||
systemctl daemon-reload
|
||||
systemctl enable --now misp-modules
|
||||
~~~~
|
||||
|
||||
## How to use an MISP modules Docker container
|
||||
|
||||
### Docker build
|
||||
## Run the tests
|
||||
|
||||
To run tests you need to install misp-modules from the cloned repository, run the server, and then run the tests. You can do all these step with `poetry`.
|
||||
|
||||
~~~~bash
|
||||
docker build -t misp-modules \
|
||||
--build-arg BUILD_DATE=$(date -u +"%Y-%m-%d") \
|
||||
docker/
|
||||
poetry install
|
||||
poetry run misp-modules
|
||||
~~~~
|
||||
|
||||
### Docker run
|
||||
And in another terminal:
|
||||
|
||||
~~~~bash
|
||||
# Start Redis
|
||||
docker run --rm -d --name=misp-redis redis:alpine
|
||||
# Start MISP-modules
|
||||
docker run \
|
||||
--rm -d --name=misp-modules \
|
||||
-e REDIS_BACKEND=misp-redis \
|
||||
-e REDIS_PORT="6379" \
|
||||
-e REDIS_PW="" \
|
||||
-e REDIS_DATABASE="245" \
|
||||
-e MISP_MODULES_DEBUG="false" \
|
||||
dcso/misp-dockerized-misp-modules
|
||||
poetry run pytest
|
||||
~~~~
|
||||
|
||||
### Docker-compose
|
||||
|
||||
~~~~yml
|
||||
services:
|
||||
misp-modules:
|
||||
# https://hub.docker.com/r/dcso/misp-dockerized-misp-modules
|
||||
image: dcso/misp-dockerized-misp-modules:3
|
||||
## Build the documentation
|
||||
|
||||
# Local image:
|
||||
#image: misp-modules
|
||||
#build:
|
||||
# context: docker/
|
||||
To build the documentation you can use the provided `Makefile`.
|
||||
Inside you will find three targets:
|
||||
|
||||
environment:
|
||||
# Redis
|
||||
REDIS_BACKEND: misp-redis
|
||||
REDIS_PORT: "6379"
|
||||
REDIS_DATABASE: "245"
|
||||
# System PROXY (OPTIONAL)
|
||||
http_proxy:
|
||||
https_proxy:
|
||||
no_proxy: 0.0.0.0
|
||||
# Timezone (OPTIONAL)
|
||||
TZ: Europe/Berlin
|
||||
# MISP-Modules (OPTIONAL)
|
||||
MISP_MODULES_DEBUG: "false"
|
||||
# Logging options (OPTIONAL)
|
||||
LOG_SYSLOG_ENABLED: "no"
|
||||
misp-redis:
|
||||
# https://hub.docker.com/_/redis or alternative https://hub.docker.com/r/dcso/misp-dockerized-redis/
|
||||
image: redis:alpine
|
||||
- `generate_docs`: install the depdendency and generate the documentation.
|
||||
|
||||
- `generate_docs`: build the documentation using `mkdocs`.
|
||||
|
||||
- `deploy`: deploy the documentation using `mkdocs gh-deploy`.
|
||||
|
||||
- `test-docs`: run a local server exposing the newly built documentation.
|
||||
|
||||
Note that you can either run the targets using `poetry` (default), or using the `squidfunk/mkdocs-material` by setting the environment variable `USE_DOCKER=true`.
|
||||
|
||||
|
||||
## Run MISP modules
|
||||
|
||||
If you installed it using pip, you just need to execute the command `misp-modules` (source the virtual environment a second time to update the search paths). If you installed it from the cloned repository, just use poetry, i.e., `poetry run misp-modules`.
|
||||
|
||||
|
||||
## Run MISP modules in Docker
|
||||
|
||||
You can find an up-to-date container image and related documentation at the following repository:m https://github.com/MISP/misp-docker
|
||||
|
||||
|
||||
## Install misp-module on an offline instance
|
||||
|
||||
### If `misp-modules` is available on PyPI
|
||||
|
||||
Once `misp-modules` is available on PyPI, you can just download all the necessary packages:
|
||||
|
||||
~~~~bash
|
||||
mkdir wheels
|
||||
pip wheel misp-modules --no-cache-dir -w ./wheels
|
||||
~~~~
|
||||
|
||||
## Install misp-module on an offline instance.
|
||||
First, you need to grab all necessary packages for example like this :
|
||||
Move the `wheels` directory to the target system, and install them there:
|
||||
|
||||
Use pip wheel to create an archive
|
||||
~~~
|
||||
mkdir misp-modules-offline
|
||||
pip3 wheel -r REQUIREMENTS shodan --wheel-dir=./misp-modules-offline
|
||||
tar -cjvf misp-module-bundeled.tar.bz2 ./misp-modules-offline/*
|
||||
~~~
|
||||
On offline machine :
|
||||
~~~
|
||||
mkdir misp-modules-bundle
|
||||
tar xvf misp-module-bundeled.tar.bz2 -C misp-modules-bundle
|
||||
cd misp-modules-bundle
|
||||
ls -1|while read line; do sudo pip3 install --force-reinstall --ignore-installed --upgrade --no-index --no-deps ${line};done
|
||||
~~~
|
||||
Next you can follow standard install procedure.
|
||||
~~~~bash
|
||||
pip install --no-cache-dir --use-deprecated=legacy-resolver /wheels/*.whl
|
||||
~~~~
|
||||
|
||||
Once again, using a virtual environment is recommended.
|
||||
|
||||
### If `misp-modules` is not available on PyPI
|
||||
|
||||
You have two choices, the first approach uses `poetry export` to export the entire virtual environment so you can copy and run it on the target system; the second one uses `poetry bundle` to export a `requirements.txt` file.
|
||||
|
||||
#### Using `poetry bundle`
|
||||
|
||||
This is quite straightforward but it assumes your target system is relatively similar (same distribution, architecture, libaries).
|
||||
|
||||
~~~~bash
|
||||
poetry install
|
||||
poetry self add poetry-plugin-bundle
|
||||
poetry bundle venv /destination/path/
|
||||
~~~~
|
||||
|
||||
#### Using `poetry export`
|
||||
|
||||
This is a bit more convoluted and it is similar to how you would install `misp-modules` on an offline instance.
|
||||
|
||||
Just follow those instructions but replace the package `misp-modules` with `-r requirements.txt`.
|
||||
|
||||
Before doing so you need to generate the `requirements.txt` file. Due to the fact we are still supporting Python 3.8 and that Poetry still has some limitations (soon to be resolved) you need to need to replace the line `python = ">=3.8.*,<3.13"` inside `pyproject.toml` with your exact version (just run `python --version`).
|
||||
|
||||
Once you have done that, run the following commands to generate your very own `requirements.txt`.
|
||||
|
||||
~~~~bash
|
||||
poetry lock
|
||||
poetry install
|
||||
poetry self add poetry-plugin-export
|
||||
poetry export --without-hashes -f requirements.txt -o requirements.txt
|
||||
~~~~
|
||||
|
|
|
@ -175,28 +175,6 @@ Backscatter.io module to bring mass-scanning observations into MISP.
|
|||
|
||||
-----
|
||||
|
||||
#### [BGP Ranking](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/bgpranking.py)
|
||||
|
||||
Query BGP Ranking to get the ranking of an Autonomous System number.
|
||||
[[source code](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/bgpranking.py)]
|
||||
|
||||
- **features**:
|
||||
>The module takes an AS number attribute as input and displays its description as well as its ranking position in BGP Ranking for a given day.
|
||||
|
||||
- **input**:
|
||||
>Autonomous system number.
|
||||
|
||||
- **output**:
|
||||
>An asn object with its related bgp-ranking object.
|
||||
|
||||
- **references**:
|
||||
>https://github.com/D4-project/BGP-Ranking/
|
||||
|
||||
- **requirements**:
|
||||
>pybgpranking python library
|
||||
|
||||
-----
|
||||
|
||||
#### [BTC Scam Check](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/btc_scam_check.py)
|
||||
|
||||
<img src=logos/bitcoin.png height=60>
|
||||
|
@ -2843,32 +2821,6 @@ Module to enrich the information by making use of the Vysion API.
|
|||
|
||||
-----
|
||||
|
||||
#### [Whois Lookup](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/whois.py)
|
||||
|
||||
Module to query a local instance of uwhois (https://github.com/rafiot/uwhoisd).
|
||||
[[source code](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/whois.py)]
|
||||
|
||||
- **features**:
|
||||
>This module takes a domain or IP address attribute as input and queries a 'Univseral Whois proxy server' to get the correct details of the Whois query on the input value (check the references for more details about this whois server).
|
||||
|
||||
- **config**:
|
||||
> - server
|
||||
> - port
|
||||
|
||||
- **input**:
|
||||
>A domain or IP address attribute.
|
||||
|
||||
- **output**:
|
||||
>Text describing the result of a whois request for the input value.
|
||||
|
||||
- **references**:
|
||||
>https://github.com/rafiot/uwhoisd
|
||||
|
||||
- **requirements**:
|
||||
>uwhois: A whois python library
|
||||
|
||||
-----
|
||||
|
||||
#### [WhoisFreaks Lookup](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/whoisfreaks.py)
|
||||
|
||||
<img src=logos/whoisfreaks.png height=60>
|
||||
|
|
|
@ -1,96 +0,0 @@
|
|||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
import os
|
||||
|
||||
module_types = ['expansion', 'export_mod', 'import_mod']
|
||||
|
||||
moduleinfo_template = {
|
||||
'version': '1.0',
|
||||
'author': '',
|
||||
'module-type': [],
|
||||
'description': '',
|
||||
'logo': '',
|
||||
'requirements': [],
|
||||
'features': '',
|
||||
'references': [],
|
||||
'input': '',
|
||||
'output': ''
|
||||
}
|
||||
|
||||
if __name__ == '__main__':
|
||||
exit("This code was temporary and should not be run again. It was used to migrate the JSON documentation to the module files.")
|
||||
root_path = Path(__file__).resolve().parent.parent
|
||||
modules_path = root_path / 'misp_modules' / 'modules'
|
||||
|
||||
for module_type in module_types:
|
||||
files = sorted(os.listdir(modules_path / module_type))
|
||||
for python_filename in files:
|
||||
if not python_filename.endswith('.py') or '__init__' in python_filename:
|
||||
continue
|
||||
modulename = python_filename.split('.py')[0]
|
||||
json_filename = root_path / 'documentation' / 'website' / module_type / f'{modulename}.json'
|
||||
print(f"Processing type {module_type}:{modulename} in {python_filename} and {json_filename}")
|
||||
json_exists = json_filename.exists()
|
||||
if json_exists:
|
||||
print(" Found JSON file")
|
||||
with open(json_filename, 'rt') as f:
|
||||
json_content = json.loads(f.read())
|
||||
else:
|
||||
json_content = {}
|
||||
# if json does not exist, then still edit the python file and add the stub structure
|
||||
with open(modules_path / module_type / python_filename, 'r+t') as python_f:
|
||||
# read from python file, find moduleinfo and load it as python variable
|
||||
python_content = python_f.read()
|
||||
re_pattern = r'moduleinfo\s=\s{[^}]*}'
|
||||
m = re.search(re_pattern, python_content, re.MULTILINE | re.DOTALL)
|
||||
if not m:
|
||||
print(f" Moduleinfo not found in {python_filename}")
|
||||
continue
|
||||
s = m.group(0)
|
||||
moduleinfo = {}
|
||||
exec(s) # we now have a moduleinfo dict
|
||||
print(f" Moduleinfo found in {python_filename}: {moduleinfo}")
|
||||
# populate from template
|
||||
for k, v in moduleinfo_template.items():
|
||||
if k not in moduleinfo or moduleinfo.get(k) == '' or moduleinfo.get(k) == []:
|
||||
# print(f" Adding {k} = {v} to {python_filename}")
|
||||
moduleinfo[k] = v
|
||||
# populate from json
|
||||
for k, v in json_content.items():
|
||||
if k not in moduleinfo or moduleinfo.get(k) == '' or moduleinfo.get(k) == []:
|
||||
# print(f" Adding {k} = {v} to {python_filename}")
|
||||
moduleinfo[k] = v
|
||||
if json_content and json_content.get('description') != moduleinfo.get('description'):
|
||||
print(" WARNING: Description in JSON and Python file do not match:")
|
||||
print("")
|
||||
print(f" JSON: {json_content.get('description')}")
|
||||
print("")
|
||||
print(f" Python: {moduleinfo.get('description')}")
|
||||
print("")
|
||||
user_input = input("Which version do you want to use? Enter '[j]son' for JSON version or '[p]ython' for Python version, or any other text for a new description: ")
|
||||
|
||||
if user_input in ['json', 'j', 'JSON']:
|
||||
moduleinfo['description'] = json_content['description']
|
||||
elif user_input in ['python', 'p', 'PYTHON']:
|
||||
pass
|
||||
else:
|
||||
moduleinfo['description'] = user_input.strip()
|
||||
|
||||
# write back to python file
|
||||
new_moduleinfo_text = ['moduleinfo = {']
|
||||
for k, v in moduleinfo.items():
|
||||
v_updated = repr(v).replace('\\', '\\\\')
|
||||
new_moduleinfo_text.append(f" '{k}': {v_updated},")
|
||||
new_moduleinfo_text.append('}')
|
||||
|
||||
python_content_new, cnt = re.subn(re_pattern, '\n'.join(new_moduleinfo_text), python_content, re.MULTILINE | re.DOTALL)
|
||||
if cnt == 0:
|
||||
print(f" WARNING: Moduleinfo not replaced in {python_filename}")
|
||||
continue
|
||||
python_f.seek(0)
|
||||
python_f.write(python_content_new)
|
||||
python_f.truncate() # remove the rest of the file
|
||||
pass
|
||||
|
||||
pass
|
|
@ -1,3 +0,0 @@
|
|||
mkdocs
|
||||
mkdocs-material
|
||||
markdown_include
|
|
@ -172,28 +172,6 @@ Backscatter.io module to bring mass-scanning observations into MISP.
|
|||
|
||||
-----
|
||||
|
||||
#### [BGP Ranking](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/bgpranking.py)
|
||||
|
||||
Query BGP Ranking to get the ranking of an Autonomous System number.
|
||||
[[source code](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/bgpranking.py)]
|
||||
|
||||
- **features**:
|
||||
>The module takes an AS number attribute as input and displays its description as well as its ranking position in BGP Ranking for a given day.
|
||||
|
||||
- **input**:
|
||||
>Autonomous system number.
|
||||
|
||||
- **output**:
|
||||
>An asn object with its related bgp-ranking object.
|
||||
|
||||
- **references**:
|
||||
>https://github.com/D4-project/BGP-Ranking/
|
||||
|
||||
- **requirements**:
|
||||
>pybgpranking python library
|
||||
|
||||
-----
|
||||
|
||||
#### [BTC Scam Check](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/btc_scam_check.py)
|
||||
|
||||
<img src=../logos/bitcoin.png height=60>
|
||||
|
@ -2840,32 +2818,6 @@ Module to enrich the information by making use of the Vysion API.
|
|||
|
||||
-----
|
||||
|
||||
#### [Whois Lookup](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/whois.py)
|
||||
|
||||
Module to query a local instance of uwhois (https://github.com/rafiot/uwhoisd).
|
||||
[[source code](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/whois.py)]
|
||||
|
||||
- **features**:
|
||||
>This module takes a domain or IP address attribute as input and queries a 'Univseral Whois proxy server' to get the correct details of the Whois query on the input value (check the references for more details about this whois server).
|
||||
|
||||
- **config**:
|
||||
> - server
|
||||
> - port
|
||||
|
||||
- **input**:
|
||||
>A domain or IP address attribute.
|
||||
|
||||
- **output**:
|
||||
>Text describing the result of a whois request for the input value.
|
||||
|
||||
- **references**:
|
||||
>https://github.com/rafiot/uwhoisd
|
||||
|
||||
- **requirements**:
|
||||
>uwhois: A whois python library
|
||||
|
||||
-----
|
||||
|
||||
#### [WhoisFreaks Lookup](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/whoisfreaks.py)
|
||||
|
||||
<img src=../logos/whoisfreaks.png height=60>
|
||||
|
|
|
@ -21,7 +21,6 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj
|
|||
* [AssemblyLine Query](https://misp.github.io/misp-modules/expansion/#assemblyline-query) - A module tu query the AssemblyLine API with a submission ID to get the submission report and parse it.
|
||||
* [AssemblyLine Submit](https://misp.github.io/misp-modules/expansion/#assemblyline-submit) - A module to submit samples and URLs to AssemblyLine for advanced analysis, and return the link of the submission.
|
||||
* [Backscatter.io](https://misp.github.io/misp-modules/expansion/#backscatter.io) - Backscatter.io module to bring mass-scanning observations into MISP.
|
||||
* [BGP Ranking](https://misp.github.io/misp-modules/expansion/#bgp-ranking) - Query BGP Ranking to get the ranking of an Autonomous System number.
|
||||
* [BTC Scam Check](https://misp.github.io/misp-modules/expansion/#btc-scam-check) - An expansion hover module to query a special dns blacklist to check if a bitcoin address has been abused.
|
||||
* [BTC Steroids](https://misp.github.io/misp-modules/expansion/#btc-steroids) - An expansion hover module to get a blockchain balance from a BTC address in MISP.
|
||||
* [Censys Enrich](https://misp.github.io/misp-modules/expansion/#censys-enrich) - An expansion module to enrich attributes in MISP by quering the censys.io API
|
||||
|
@ -115,7 +114,6 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj
|
|||
* [Vulnerability Lookup](https://misp.github.io/misp-modules/expansion/#vulnerability-lookup) - An expansion module to query Vulnerability Lookup
|
||||
* [Vulners Lookup](https://misp.github.io/misp-modules/expansion/#vulners-lookup) - An expansion hover module to expand information about CVE id using Vulners API.
|
||||
* [Vysion Enrich](https://misp.github.io/misp-modules/expansion/#vysion-enrich) - Module to enrich the information by making use of the Vysion API.
|
||||
* [Whois Lookup](https://misp.github.io/misp-modules/expansion/#whois-lookup) - Module to query a local instance of uwhois (https://github.com/rafiot/uwhoisd).
|
||||
* [WhoisFreaks Lookup](https://misp.github.io/misp-modules/expansion/#whoisfreaks-lookup) - An expansion module for https://whoisfreaks.com/ that will provide an enriched analysis of the provided domain, including WHOIS and DNS information.
|
||||
* [Wikidata Lookup](https://misp.github.io/misp-modules/expansion/#wikidata-lookup) - An expansion hover module to extract information from Wikidata to have additional information about particular term for analysis.
|
||||
* [IBM X-Force Exchange Lookup](https://misp.github.io/misp-modules/expansion/#ibm-x-force-exchange-lookup) - An expansion module for IBM X-Force Exchange.
|
||||
|
|
|
@ -1,199 +1,159 @@
|
|||
## How to install and start MISP modules (in a Python virtualenv)? (recommended)
|
||||
## Install from pip
|
||||
|
||||
***Be sure to run the latest version of `pip`***. To install the latest version of pip, `pip install --upgrade pip` will do the job.
|
||||
It is strongly recommended to use a virtual environment (see here for instructions https://docs.python.org/3/tutorial/venv.html).
|
||||
|
||||
Once the virtual environment is loaded just use the command:
|
||||
|
||||
~~~~bash
|
||||
SUDO_WWW="sudo -u www-data"
|
||||
|
||||
sudo apt-get install -y \
|
||||
python3-dev \
|
||||
python3-pip \
|
||||
git \
|
||||
libpq5 \
|
||||
libjpeg-dev \
|
||||
tesseract-ocr \
|
||||
libpoppler-cpp-dev \
|
||||
imagemagick virtualenv \
|
||||
libopencv-dev \
|
||||
zbar-tools \
|
||||
libzbar0 \
|
||||
libzbar-dev \
|
||||
libfuzzy-dev \
|
||||
libcaca-dev \
|
||||
build-essential
|
||||
|
||||
# BEGIN with virtualenv:
|
||||
$SUDO_WWW virtualenv -p python3 /var/www/MISP/venv
|
||||
# END with virtualenv
|
||||
|
||||
cd /usr/local/src/
|
||||
# Ideally you add your user to the staff group and make /usr/local/src group writeable, below follows an example with user misp
|
||||
sudo adduser misp staff
|
||||
sudo chmod 2775 /usr/local/src
|
||||
sudo chown root:staff /usr/local/src
|
||||
git clone https://github.com/MISP/misp-modules.git
|
||||
git clone git://github.com/stricaud/faup.git faup
|
||||
git clone git://github.com/stricaud/gtcaca.git gtcaca
|
||||
|
||||
# Install gtcaca/faup
|
||||
cd gtcaca
|
||||
mkdir -p build
|
||||
cd build
|
||||
cmake .. && make
|
||||
sudo make install
|
||||
cd ../../faup
|
||||
mkdir -p build
|
||||
cd build
|
||||
cmake .. && make
|
||||
sudo make install
|
||||
sudo ldconfig
|
||||
|
||||
cd ../../misp-modules
|
||||
|
||||
# BEGIN with virtualenv:
|
||||
$SUDO_WWW /var/www/MISP/venv/bin/pip install -I -r REQUIREMENTS
|
||||
$SUDO_WWW /var/www/MISP/venv/bin/pip install .
|
||||
# END with virtualenv
|
||||
|
||||
# BEGIN without virtualenv:
|
||||
sudo pip install -I -r REQUIREMENTS
|
||||
sudo pip install .
|
||||
# END without virtualenv
|
||||
|
||||
# Start misp-modules as a service
|
||||
sudo cp etc/systemd/system/misp-modules.service /etc/systemd/system/
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl enable --now misp-modules
|
||||
sudo service misp-modules start # or
|
||||
/var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s & # to start the modules manually
|
||||
pip install misp-modules
|
||||
~~~~
|
||||
|
||||
## How to install and start MISP modules on RHEL-based distributions ?
|
||||
Note: this install method might not yet be available.
|
||||
|
||||
As of this writing, the official RHEL repositories only contain Ruby 2.0.0 and Ruby 2.1 or higher is required. As such, this guide installs Ruby 2.2 from the [SCL](https://access.redhat.com/documentation/en-us/red_hat_software_collections/3/html/3.2_release_notes/chap-installation#sect-Installation-Subscribe) repository.
|
||||
|
||||
## Install from cloned repository
|
||||
|
||||
In this case the only requirement is to install `poetry`. Normally you just need to run `pip install poetry`, but see here for more alternatives https://python-poetry.org/docs/#installation.
|
||||
|
||||
Once `poetry` is installed, you can clone the repository and install `misp-modules` as follows:
|
||||
|
||||
~~~~bash
|
||||
SUDO_WWW="sudo -u apache"
|
||||
sudo yum install \
|
||||
rh-python36 \
|
||||
rh-ruby22 \
|
||||
openjpeg-devel \
|
||||
rubygem-rouge \
|
||||
rubygem-asciidoctor \
|
||||
zbar-devel \
|
||||
opencv-devel \
|
||||
gcc-c++ \
|
||||
pkgconfig \
|
||||
poppler-cpp-devel \
|
||||
python-devel \
|
||||
redhat-rpm-config
|
||||
cd /var/www/MISP
|
||||
$SUDO_WWW git clone https://github.com/MISP/misp-modules.git
|
||||
cd misp-modules
|
||||
$SUDO_WWW /usr/bin/scl enable rh-python36 "virtualenv -p python3 /var/www/MISP/venv"
|
||||
$SUDO_WWW /var/www/MISP/venv/bin/pip install -U -I -r REQUIREMENTS
|
||||
$SUDO_WWW /var/www/MISP/venv/bin/pip install -U .
|
||||
git clone https://github.com/MISP/misp-modules.git && cd misp-modules
|
||||
git submodule update --init
|
||||
poetry install
|
||||
~~~~
|
||||
|
||||
Create the service file /etc/systemd/system/misp-modules.service :
|
||||
Note that the dependencies will require a number of system packages installed. On Ubuntu these packages are `libpoppler-cpp-dev`, `libzbar0`, and `tesseract-ocr`. For an updated list, check the github action used to test the build inside `.github/workflows`.
|
||||
|
||||
|
||||
## Install the systemd unit
|
||||
|
||||
To run `misp-modules` as a service on a distribution based on systemd, you need to create the unit as follows and store it in a file `/etc/systemd/system/misp-modules.service`:
|
||||
|
||||
~~~~bash
|
||||
echo "[Unit]
|
||||
Description=MISP's modules
|
||||
After=misp-workers.service
|
||||
[Unit]
|
||||
Description=MISP modules
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=apache
|
||||
Group=apache
|
||||
ExecStart=/usr/bin/scl enable rh-python36 rh-ruby22 '/var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s'
|
||||
ExecStart='/path/to/venv/bin/misp-modules -l 127.0.0.1 -s'
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target" | sudo tee /etc/systemd/system/misp-modules.service
|
||||
WantedBy=multi-user.target
|
||||
~~~~
|
||||
|
||||
The `After=misp-workers.service` must be changed or removed if you have not created a misp-workers service.
|
||||
Then, enable the misp-modules service and start it:
|
||||
~~~~bash
|
||||
systemctl daemon-reload
|
||||
systemctl enable --now misp-modules
|
||||
~~~~
|
||||
|
||||
## How to use an MISP modules Docker container
|
||||
|
||||
### Docker build
|
||||
## Run the tests
|
||||
|
||||
To run tests you need to install misp-modules from the cloned repository, run the server, and then run the tests. You can do all these step with `poetry`.
|
||||
|
||||
~~~~bash
|
||||
docker build -t misp-modules \
|
||||
--build-arg BUILD_DATE=$(date -u +"%Y-%m-%d") \
|
||||
docker/
|
||||
poetry install
|
||||
poetry run misp-modules
|
||||
~~~~
|
||||
|
||||
### Docker run
|
||||
And in another terminal:
|
||||
|
||||
~~~~bash
|
||||
# Start Redis
|
||||
docker run --rm -d --name=misp-redis redis:alpine
|
||||
# Start MISP-modules
|
||||
docker run \
|
||||
--rm -d --name=misp-modules \
|
||||
-e REDIS_BACKEND=misp-redis \
|
||||
-e REDIS_PORT="6379" \
|
||||
-e REDIS_PW="" \
|
||||
-e REDIS_DATABASE="245" \
|
||||
-e MISP_MODULES_DEBUG="false" \
|
||||
dcso/misp-dockerized-misp-modules
|
||||
poetry run pytest
|
||||
~~~~
|
||||
|
||||
### Docker-compose
|
||||
|
||||
~~~~yml
|
||||
services:
|
||||
misp-modules:
|
||||
# https://hub.docker.com/r/dcso/misp-dockerized-misp-modules
|
||||
image: dcso/misp-dockerized-misp-modules:3
|
||||
## Build the documentation
|
||||
|
||||
# Local image:
|
||||
#image: misp-modules
|
||||
#build:
|
||||
# context: docker/
|
||||
To build the documentation you can use the provided `Makefile`.
|
||||
Inside you will find three targets:
|
||||
|
||||
environment:
|
||||
# Redis
|
||||
REDIS_BACKEND: misp-redis
|
||||
REDIS_PORT: "6379"
|
||||
REDIS_DATABASE: "245"
|
||||
# System PROXY (OPTIONAL)
|
||||
http_proxy:
|
||||
https_proxy:
|
||||
no_proxy: 0.0.0.0
|
||||
# Timezone (OPTIONAL)
|
||||
TZ: Europe/Berlin
|
||||
# MISP-Modules (OPTIONAL)
|
||||
MISP_MODULES_DEBUG: "false"
|
||||
# Logging options (OPTIONAL)
|
||||
LOG_SYSLOG_ENABLED: "no"
|
||||
misp-redis:
|
||||
# https://hub.docker.com/_/redis or alternative https://hub.docker.com/r/dcso/misp-dockerized-redis/
|
||||
image: redis:alpine
|
||||
- `generate_docs`: install the depdendency and generate the documentation.
|
||||
|
||||
- `generate_docs`: build the documentation using `mkdocs`.
|
||||
|
||||
- `deploy`: deploy the documentation using `mkdocs gh-deploy`.
|
||||
|
||||
- `test-docs`: run a local server exposing the newly built documentation.
|
||||
|
||||
Note: you can either run the targets using `poetry` (default), or using the Docker image `squidfunk/mkdocs-material` by setting the environment variable `USE_DOCKER=true`.
|
||||
|
||||
|
||||
## Run MISP modules
|
||||
|
||||
If you installed it using pip, you just need to execute the command `misp-modules` (source the virtual environment a second time to update the search paths). If you installed it from the cloned repository, just use poetry, i.e., `poetry run misp-modules`.
|
||||
|
||||
|
||||
## Run MISP modules in Docker
|
||||
|
||||
You can find an up-to-date container image and related documentation at the following repository: https://github.com/MISP/misp-docker .
|
||||
|
||||
|
||||
## Install misp-module on an offline instance
|
||||
|
||||
### If `misp-modules` is available on PyPI
|
||||
|
||||
Once `misp-modules` is available on PyPI, you can just download all the necessary packages:
|
||||
|
||||
~~~~bash
|
||||
mkdir wheels
|
||||
pip wheel misp-modules --no-cache-dir -w ./wheels
|
||||
~~~~
|
||||
|
||||
## Install misp-module on an offline instance.
|
||||
First, you need to grab all necessary packages for example like this :
|
||||
Move the `wheels` directory to the target system, and install them there:
|
||||
|
||||
~~~~bash
|
||||
pip install --no-cache-dir --use-deprecated=legacy-resolver /wheels/*.whl
|
||||
~~~~
|
||||
|
||||
Once again, using a virtual environment is recommended.
|
||||
|
||||
### If `misp-modules` is not available on PyPI
|
||||
|
||||
You have two choices, the first approach uses `poetry export` to export the entire virtual environment so you can copy and run it on the target system; the second one uses `poetry bundle` to export a `requirements.txt` file.
|
||||
|
||||
#### Using `poetry bundle`
|
||||
|
||||
This is quite straightforward but it assumes your target system is relatively similar (same distribution, architecture, libaries).
|
||||
|
||||
~~~~bash
|
||||
poetry install
|
||||
poetry self add poetry-plugin-bundle
|
||||
poetry bundle venv /destination/path/
|
||||
~~~~
|
||||
|
||||
#### Using `poetry export`
|
||||
|
||||
This is a bit more convoluted and it is similar to how you would install `misp-modules` on an offline instance.
|
||||
|
||||
Just follow those instructions but replace the package `misp-modules` with `-r requirements.txt`.
|
||||
|
||||
Before doing so you need to generate the `requirements.txt` file. Due to the fact we are still supporting Python 3.8 and that Poetry still has some limitations (soon to be resolved) you need to need to replace the line `python = ">=3.8.*,<3.13"` inside `pyproject.toml` with your exact version (just run `python --version`).
|
||||
|
||||
The following `sed` command does everything for you.
|
||||
|
||||
~~~~bash
|
||||
sed -i "s/^python = .*/python = \"$(python -c 'import platform; print(platform.python_version())')\"/" pyproject.toml
|
||||
~~~~
|
||||
|
||||
Then, run the following commands to generate your very own `requirements.txt`.
|
||||
|
||||
~~~~bash
|
||||
poetry lock
|
||||
poetry install
|
||||
poetry self add poetry-plugin-export
|
||||
poetry export --without-hashes -f requirements.txt -o requirements.txt
|
||||
~~~~
|
||||
|
||||
Note that `misp-modules` will not be part of the `requirements.txt` file and you will need to create the wheel yourself:
|
||||
|
||||
~~~~bash
|
||||
poetry build --output ./wheels
|
||||
~~~~
|
||||
|
||||
Use pip wheel to create an archive
|
||||
~~~
|
||||
mkdir misp-modules-offline
|
||||
pip3 wheel -r REQUIREMENTS shodan --wheel-dir=./misp-modules-offline
|
||||
tar -cjvf misp-module-bundeled.tar.bz2 ./misp-modules-offline/*
|
||||
~~~
|
||||
On offline machine :
|
||||
~~~
|
||||
mkdir misp-modules-bundle
|
||||
tar xvf misp-module-bundeled.tar.bz2 -C misp-modules-bundle
|
||||
cd misp-modules-bundle
|
||||
ls -1|while read line; do sudo pip3 install --force-reinstall --ignore-installed --upgrade --no-index --no-deps ${line};done
|
||||
~~~
|
||||
Next you can follow standard install procedure.
|
||||
|
|
|
@ -1,14 +0,0 @@
|
|||
[Unit]
|
||||
Description=System-wide instance of the MISP Modules
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
User=www-data
|
||||
Group=www-data
|
||||
WorkingDirectory=/usr/local/src/misp-modules
|
||||
Environment="PATH=/var/www/MISP/venv/bin"
|
||||