MISP
/
misp-modules
mirror of https://github.com/MISP/misp-modules
2
0
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'main' of github.com:MISP/misp-modules into main

pull/555/head
Alexandre Dulaunoy 2022-02-14 09:38:14 +01:00
parent 3b7b2748e0 c62a911cd0
commit c100924eb6
No known key found for this signature in database
GPG Key ID: 09E2CD4944E6CBCD
7 changed files with 275 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/python-package.yml vendored
View File

@ -46,5 +46,7 @@ jobs:
# Run server in background # Run server in background
misp-modules -l 127.0.0.1 -s & misp-modules -l 127.0.0.1 -s &
sleep 5 sleep 5
# Check if modules are running
curl -sS localhost:6666/modules
# Run tests # Run tests
pytest tests pytest tests

24
Pipfile
View File

@ -17,9 +17,9 @@ passivetotal = "*"
pypdns = "*" pypdns = "*"
pypssl = "*" pypssl = "*"
pyeupi = "*" pyeupi = "*"
pymisp = { extras = ["fileobjects,openioc,pdfexport,email"], version = "*" } pymisp = { extras = ["fileobjects,openioc,pdfexport,email,url"], version = "*" }
pyonyphe = { editable = true, git = "https://github.com/sebdraven/pyonyphe" } pyonyphe = { git = "https://github.com/sebdraven/pyonyphe" }
pydnstrails = { editable = true, git = "https://github.com/sebdraven/pydnstrails" } pydnstrails = { git = "https://github.com/sebdraven/pydnstrails" }
pytesseract = "*" pytesseract = "*"
pygeoip = "*" pygeoip = "*"
beautifulsoup4 = "*" beautifulsoup4 = "*"
@ -31,20 +31,20 @@ maclookup = "*"
vulners = "*" vulners = "*"
blockchain = "*" blockchain = "*"
reportlab = "*" reportlab = "*"
pyintel471 = { editable = true, git = "https://github.com/MISP/PyIntel471.git" } pyintel471 = { git = "https://github.com/MISP/PyIntel471.git" }
shodan = "*" shodan = "*"
Pillow = ">=8.2.0" Pillow = ">=8.2.0"
Wand = "*" Wand = "*"
SPARQLWrapper = "*" SPARQLWrapper = "*"
domaintools_api = "*" domaintools_api = "*"
misp-modules = { editable = true, path = "." } misp-modules = { path = "." }
pybgpranking = { editable = true, git = "https://github.com/D4-project/BGP-Ranking.git/", subdirectory = "client" } pybgpranking = { git = "https://github.com/D4-project/BGP-Ranking.git/", subdirectory = "client", ref = "68de39f6c5196f796055c1ac34504054d688aa59" }
pyipasnhistory = { editable = true, git = "https://github.com/D4-project/IPASN-History.git/", subdirectory = "client" } pyipasnhistory = { git = "https://github.com/D4-project/IPASN-History.git/", subdirectory = "client", ref = "a2853c39265cecdd0c0d16850bd34621c0551b87" }
backscatter = "*" backscatter = "*"
pyzbar = "*" pyzbar = "*"
opencv-python = "*" opencv-python = "*"
np = "*" np = "*"
ODTReader = { editable = true, git = "https://github.com/cartertemm/ODTReader.git/" } ODTReader = { git = "https://github.com/cartertemm/ODTReader.git/" }
python-pptx = "*" python-pptx = "*"
python-docx = "*" python-docx = "*"
ezodf = "*" ezodf = "*"
@ -59,7 +59,7 @@ geoip2 = "*"
apiosintDS = "*" apiosintDS = "*"
assemblyline_client = "*" assemblyline_client = "*"
vt-graph-api = "*" vt-graph-api = "*"
trustar = { editable = true, git = "https://github.com/SteveClement/trustar-python.git" } trustar = { git = "https://github.com/SteveClement/trustar-python.git" }
markdownify = "==0.5.3" markdownify = "==0.5.3"
socialscan = "*" socialscan = "*"
dnsdb2 = "*" dnsdb2 = "*"
@ -67,6 +67,10 @@ clamd = "*"
aiohttp = ">=3.7.4" aiohttp = ">=3.7.4"
tau-clients = "*" tau-clients = "*"
vt-py = ">=0.7.1" vt-py = ">=0.7.1"
crowdstrike-falconpy = "0.9.0"
censys = "2.0.9"
mwdblib = "3.4.1"
ndjson = "0.3.1"
[requires] [requires]
python_version = "3.6" python_version = "3.7"

1
README.md
View File

@ -58,6 +58,7 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj
* [macvendors](misp_modules/modules/expansion/macvendors.py) - a hover module to retrieve mac vendor information. * [macvendors](misp_modules/modules/expansion/macvendors.py) - a hover module to retrieve mac vendor information.
* [MALWAREbazaar](misp_modules/modules/expansion/malwarebazaar.py) - an expansion module to query MALWAREbazaar with some payload. * [MALWAREbazaar](misp_modules/modules/expansion/malwarebazaar.py) - an expansion module to query MALWAREbazaar with some payload.
* [McAfee MVISION Insights](misp_modules/modules/expansion/mcafee_insights_enrich.py) - an expansion module enrich IOCs with McAfee MVISION Insights. * [McAfee MVISION Insights](misp_modules/modules/expansion/mcafee_insights_enrich.py) - an expansion module enrich IOCs with McAfee MVISION Insights.
* [Mmdb server lookup](misp_modules/modules/expansion/mmdb_lookup.py) - an expansion module to enrich an ip with geolocation information from an mmdb server such as ip.circl.lu.
* [ocr-enrich](misp_modules/modules/expansion/ocr_enrich.py) - an enrichment module to get OCRized data from images into MISP. * [ocr-enrich](misp_modules/modules/expansion/ocr_enrich.py) - an enrichment module to get OCRized data from images into MISP.
* [ods-enrich](misp_modules/modules/expansion/ods_enrich.py) - an enrichment module to get text out of OpenOffice spreadsheet document into MISP (using free-text parser). * [ods-enrich](misp_modules/modules/expansion/ods_enrich.py) - an enrichment module to get text out of OpenOffice spreadsheet document into MISP (using free-text parser).
* [odt-enrich](misp_modules/modules/expansion/odt_enrich.py) - an enrichment module to get text out of OpenOffice document into MISP (using free-text parser). * [odt-enrich](misp_modules/modules/expansion/odt_enrich.py) - an enrichment module to get text out of OpenOffice document into MISP (using free-text parser).

181
REQUIREMENTS
View File

@ -6,152 +6,165 @@
# #
-i https://pypi.org/simple -i https://pypi.org/simple
-e . .
-e git+https://github.com/D4-project/BGP-Ranking.git/@68de39f6c5196f796055c1ac34504054d688aa59#egg=pybgpranking&subdirectory=client aiohttp==3.8.1
-e git+https://github.com/D4-project/IPASN-History.git/@a2853c39265cecdd0c0d16850bd34621c0551b87#egg=pyipasnhistory&subdirectory=client aiosignal==1.2.0; python_version >= '3.6'
-e git+https://github.com/MISP/PyIntel471.git@917272fafa8e12102329faca52173e90c5256968#egg=pyintel471
-e git+https://github.com/cartertemm/ODTReader.git/@49d6938693f6faa3ff09998f86dba551ae3a996b#egg=odtreader
-e git+https://github.com/sebdraven/pydnstrails@48c1f740025c51289f43a24863d1845ff12fd21a#egg=pydnstrails
-e git+https://github.com/sebdraven/pyonyphe@1ce15581beebb13e841193a08a2eb6f967855fcb#egg=pyonyphe
git+https://github.com/SteveClement/trustar-python.git
aiohttp==3.7.4
antlr4-python3-runtime==4.8; python_version >= '3' antlr4-python3-runtime==4.8; python_version >= '3'
apiosintds==1.8.3 apiosintds==1.8.3
appdirs==1.4.4
argparse==1.4.0 argparse==1.4.0
assemblyline-client==4.1.0 assemblyline-client==4.2.2
async-timeout==3.0.1; python_full_version >= '3.5.3' async-timeout==4.0.2; python_version >= '3.6'
attrs==21.2.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' asynctest==0.13.0; python_version < '3.8'
attrs==21.4.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
backoff==1.11.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
backports.zoneinfo==0.2.1; python_version < '3.9' backports.zoneinfo==0.2.1; python_version < '3.9'
backscatter==0.2.4 backscatter==0.2.4
beautifulsoup4==4.9.3 beautifulsoup4==4.10.0
bidict==0.21.2; python_version >= '3.6' bidict==0.21.4; python_version >= '3.6'
blockchain==1.4.4 blockchain==1.4.4
certifi==2021.5.30 censys==2.1.2
censys==2.0.9 certifi==2021.10.8
cffi==1.14.6 cffi==1.15.0
#chardet==4.0.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' chardet==4.0.0
chardet charset-normalizer==2.0.11; python_version >= '3'
charset-normalizer==2.0.4; python_version >= '3'
clamd==1.0.2 clamd==1.0.2
click-plugins==1.1.1 click-plugins==1.1.1
click==8.0.1; python_version >= '3.6' click==8.0.3; python_version >= '3.6'
colorama==0.4.4; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' colorama==0.4.4; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
colorclass==2.2.0 colorclass==2.2.2; python_version >= '2.6'
commonmark==0.9.1
compressed-rtf==1.0.6 compressed-rtf==1.0.6
configparser==5.0.2; python_version >= '3.6' configparser==5.2.0; python_version >= '3.6'
crowdstrike-falconpy==0.9.0 crowdstrike-falconpy==1.0.0
cryptography==3.4.7; python_version >= '3.6' cryptography==36.0.1; python_version >= '3.6'
decorator==5.0.9; python_version >= '3.5' decorator==5.1.1; python_version >= '3.5'
deprecated==1.2.12; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' deprecated==1.2.13; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
dnsdb2==1.1.3 dnsdb2==1.1.3
dnspython==2.1.0 dnspython==2.2.0
domaintools-api==0.5.4 domaintools-api==0.6.1
easygui==0.98.2 easygui==0.98.2
ebcdic==1.1.1 ebcdic==1.1.1
enum-compat==0.0.3 enum-compat==0.0.3
extract-msg==0.28.7 extract-msg==0.28.7
ez-setup==0.9
ezodf==0.3.2 ezodf==0.3.2
filelock==3.0.12 filelock==3.4.2; python_version >= '3.7'
frozenlist==1.3.0; python_version >= '3.7'
future==0.18.2; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' future==0.18.2; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
futures==3.1.1 geoip2==4.5.0
geoip2==4.2.0 git+https://github.com/D4-project/BGP-Ranking.git/@68de39f6c5196f796055c1ac34504054d688aa59#egg=pybgpranking&subdirectory=client
httplib2==0.19.1 git+https://github.com/D4-project/IPASN-History.git/@a2853c39265cecdd0c0d16850bd34621c0551b87#egg=pyipasnhistory&subdirectory=client
git+https://github.com/MISP/PyIntel471.git@917272fafa8e12102329faca52173e90c5256968#egg=pyintel471
git+https://github.com/SteveClement/trustar-python.git@6954eae38e0c77eaeef26084b6c5fd033925c1c7#egg=trustar
git+https://github.com/cartertemm/ODTReader.git/@49d6938693f6faa3ff09998f86dba551ae3a996b#egg=odtreader
git+https://github.com/sebdraven/pydnstrails@48c1f740025c51289f43a24863d1845ff12fd21a#egg=pydnstrails
git+https://github.com/sebdraven/pyonyphe@aed008ee5a27e3a5e4afbb3e5cbfc47170108452#egg=pyonyphe
httplib2==0.20.4; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
idna-ssl==1.1.0; python_version < '3.7' idna-ssl==1.1.0; python_version < '3.7'
idna==3.2; python_version >= '3' idna==3.3; python_version >= '3'
imapclient==2.1.0 imapclient==2.1.0
isodate==0.6.0 importlib-metadata==4.10.1; python_version < '3.8'
isodate==0.6.1
itsdangerous==2.0.1; python_version >= '3.6' itsdangerous==2.0.1; python_version >= '3.6'
jbxapi==3.17.2 jbxapi==3.17.2
json-log-formatter==0.4.0 jeepney==0.7.1; sys_platform == 'linux'
json-log-formatter==0.5.1
jsonschema==3.2.0 jsonschema==3.2.0
lark-parser==0.11.3 keyring==23.5.0; python_version >= '3.7'
lark-parser==0.12.0
lief==0.11.5 lief==0.11.5
lxml==4.7.1 lxml==4.7.1
maclookup==1.0.3 maclookup==1.0.3
markdownify==0.5.3 markdownify==0.5.3
maxminddb==2.0.3; python_version >= '3.6' maxminddb==2.2.0; python_version >= '3.6'
more-itertools==8.8.0; python_version >= '3.5' more-itertools==8.12.0; python_version >= '3.5'
msoffcrypto-tool==4.12.0; python_version >= '3' and platform_python_implementation != 'PyPy' or (platform_system != 'Windows' and platform_system != 'Darwin') msoffcrypto-tool==5.0.0; python_version >= '3' and platform_python_implementation != 'PyPy' or (platform_system != 'Windows' and platform_system != 'Darwin')
multidict==5.1.0; python_version >= '3.6' multidict==6.0.2; python_version >= '3.7'
mwdblib==4.0.0
ndjson==0.3.1
np==1.0.2 np==1.0.2
numpy==1.21.2; python_version < '3.11' and python_version >= '3.7' numpy==1.21.5; python_version < '3.10' and platform_machine != 'aarch64' and platform_machine != 'arm64'
oauth2==1.9.0.post1 oauth2==1.9.0.post1
olefile==0.46; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' olefile==0.46; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
oletools==0.56.2 oletools==0.60
opencv-python==4.5.3.56 opencv-python==4.5.5.62
packaging==21.3; python_version >= '3.6'
pandas-ods-reader==0.1.2 pandas-ods-reader==0.1.2
pandas==1.3.5 pandas==1.3.5
passivetotal==2.5.4 passivetotal==2.5.8
pcodedmp==1.2.6 pcodedmp==1.2.6
pdftotext==2.2.0 pdftotext==2.2.2
pillow==8.3.2 pillow==9.0.1
progressbar2==3.53.1 progressbar2==4.0.0; python_version >= '3.7'
psutil==5.8.0; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' psutil==5.9.0; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
pycparser==2.20; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' pycparser==2.21
pycryptodome==3.10.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' pycryptodome==3.14.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
pycryptodomex==3.10.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' pycryptodomex==3.14.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
pydeep==0.4 pydeep==0.4
pyeupi==1.1 pyeupi==1.1
pyfaup==1.2
pygeoip==0.3.2 pygeoip==0.3.2
pymisp[email,fileobjects,openioc,pdfexport]==2.4.148 pygments==2.11.2; python_version >= '3.5'
pymisp[email,fileobjects,openioc,pdfexport,url]==2.4.152
pyparsing==2.4.7; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' pyparsing==2.4.7; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
pypdns==1.5.2 pypdns==1.5.2
pypssl==2.2 pypssl==2.2
pyrsistent==0.18.0; python_version >= '3.6' pyrsistent==0.18.1; python_version >= '3.7'
pytesseract==0.3.8 pytesseract==0.3.8
python-baseconv==1.2.2 python-baseconv==1.2.2
python-dateutil==2.8.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' python-dateutil==2.8.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
python-docx==0.8.11 python-docx==0.8.11
python-engineio==4.2.1; python_version >= '3.6' python-engineio==4.3.1; python_version >= '3.6'
python-magic==0.4.24 python-magic==0.4.25
python-pptx==0.6.19 python-pptx==0.6.21
python-socketio[client]==5.4.0; python_version >= '3.6' python-socketio[client]==5.5.1; python_version >= '3.6'
python-utils==2.5.6 python-utils==3.1.0; python_version >= '3.7'
pytz-deprecation-shim==0.1.0.post0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
pytz==2019.3 pytz==2019.3
pyyaml==5.4.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' pyyaml==6.0; python_version >= '3.6'
pyzbar==0.1.8 pyzbar==0.1.8
pyzipper==0.3.5; python_version >= '3.5' pyzipper==0.3.5; python_version >= '3.5'
rdflib==6.0.0; python_version >= '3.7' rdflib==6.1.1; python_version >= '3.7'
redis==3.5.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' redis==4.1.2; python_version >= '3.6'
reportlab==3.6.1 reportlab==3.6.6
requests-cache==0.6.4; python_version >= '3.6' requests-cache==0.6.4; python_version >= '3.6'
requests-file==1.5.1 requests-file==1.5.1
requests[security]==2.26.0 requests[security]==2.27.1
rich==11.1.0; python_full_version >= '3.6.2' and python_full_version < '4.0.0'
rtfde==0.0.2 rtfde==0.0.2
ruamel.yaml.clib==0.2.6; python_version < '3.10' and platform_python_implementation == 'CPython' secretstorage==3.3.1; sys_platform == 'linux'
ruamel.yaml==0.17.13; python_version >= '3' setuptools==60.7.1; python_version >= '3.7'
shodan==1.25.0 shodan==1.26.1
sigmatools==0.19.1 sigmatools==0.19.1
six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
socialscan==1.4.2 socialscan==1.4.2
socketio-client==0.5.7.4 socketio-client==0.5.7.4
soupsieve==2.2.1; python_version >= '3' soupsieve==2.3.1; python_version >= '3.6'
sparqlwrapper==1.8.5 sparqlwrapper==1.8.5
stix2==3.0.1 stix2==3.0.1
stix2-patterns==1.3.2 stix2-patterns==1.3.2
tabulate==0.8.9 tabulate==0.8.9
tau-clients==0.1.3 tau-clients==0.1.9
taxii2-client==2.3.0 taxii2-client==2.3.0
tldextract==3.1.0; python_version >= '3.5' tldextract==3.1.2; python_version >= '3.6'
tornado==6.1; python_version >= '3.5' tornado==6.1; python_version >= '3.5'
tqdm==4.62.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' tqdm==4.62.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
typing-extensions==3.10.0.0 typing-extensions==4.0.1; python_version < '3.8'
tzlocal==3.0; python_version >= '3.6' tzdata==2021.5; python_version >= '3.6'
tzlocal==4.1; python_version >= '3.6'
unicodecsv==0.14.1 unicodecsv==0.14.1
url-normalize==1.4.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' url-normalize==1.4.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
urlarchiver==0.2 urlarchiver==0.2
urllib3==1.26.6; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4' urllib3==1.26.8; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_full_version < '4.0.0'
validators==0.14.0 validators==0.14.0
vt-graph-api==1.1.2 vt-graph-api==1.1.3
vt-py==0.7.2 vt-py==0.13.1
vulners==1.5.12 vulners==2.0.0
wand==0.6.7 wand==0.6.7
websocket-client==1.2.1 websocket-client==1.2.3; python_version >= '3.6'
wrapt==1.12.1 wrapt==1.13.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
xlrd==2.0.1 xlrd==2.0.1
xlsxwriter==3.0.1; python_version >= '3.4' xlsxwriter==3.0.2; python_version >= '3.4'
yara-python==3.8.1 yara-python==3.8.1
yarl==1.6.3; python_version >= '3.6' yarl==1.7.2; python_version >= '3.6'
ndjson==0.3.1 zipp==3.7.0; python_version >= '3.7'
mwdblib==3.4.1

2
misp_modules/modules/expansion/__init__.py
View File

@ -18,7 +18,7 @@ __all__ = ['cuckoo_submit', 'vmray_submit', 'bgpranking', 'circl_passivedns', 'c
'assemblyline_submit', 'assemblyline_query', 'ransomcoindb', 'malwarebazaar', 'assemblyline_submit', 'assemblyline_query', 'ransomcoindb', 'malwarebazaar',
'lastline_query', 'lastline_submit', 'sophoslabs_intelix', 'cytomic_orion', 'censys_enrich', 'lastline_query', 'lastline_submit', 'sophoslabs_intelix', 'cytomic_orion', 'censys_enrich',
'trustar_enrich', 'recordedfuture', 'html_to_markdown', 'socialscan', 'passive-ssh', 'trustar_enrich', 'recordedfuture', 'html_to_markdown', 'socialscan', 'passive-ssh',
'qintel_qsentry', 'mwdb', 'hashlookup'] 'qintel_qsentry', 'mwdb', 'hashlookup', 'mmdb_lookup']
minimum_required_fields = ('type', 'uuid', 'value') minimum_required_fields = ('type', 'uuid', 'value')

34
misp_modules/modules/expansion/apivoid.py
View File

@ -4,8 +4,8 @@ from . import check_input_attribute, standard_error_message
from pymisp import MISPAttribute, MISPEvent, MISPObject from pymisp import MISPAttribute, MISPEvent, MISPObject
misperrors = {'error': 'Error'} misperrors = {'error': 'Error'}
mispattributes = {'input': ['domain', 'hostname'], 'format': 'misp_standard'} mispattributes = {'input': ['domain', 'hostname', 'email'], 'format': 'misp_standard'}
moduleinfo = {'version': '0.1', 'author': 'Christian Studer', moduleinfo = {'version': '0.2', 'author': 'Christian Studer',
'description': 'On demand query API for APIVoid.', 'description': 'On demand query API for APIVoid.',
'module-type': ['expansion', 'hover']} 'module-type': ['expansion', 'hover']}
moduleconfig = ['apikey'] moduleconfig = ['apikey']
@ -43,6 +43,31 @@ class APIVoidParser():
ssl = requests.get(f'{self.url.format("sslinfo", apikey)}host={self.attribute.value}').json() ssl = requests.get(f'{self.url.format("sslinfo", apikey)}host={self.attribute.value}').json()
self._parse_ssl_certificate(ssl['data']['certificate']) self._parse_ssl_certificate(ssl['data']['certificate'])
def handle_email(self, apikey):
feature = 'emailverify'
if requests.get(f'{self.url.format(feature, apikey)}stats').json()['credits_remained'] < 0.06:
self.result = {'error': 'You do not have enough APIVoid credits to proceed your request.'}
return
emaillookup = requests.get(f'{self.url.format(feature, apikey)}email={self.attribute.value}').json()
email_verification = MISPObject('apivoid-email-verification')
boolean_attributes = ['valid_format', 'suspicious_username', 'suspicious_email', 'dirty_words_username',
'suspicious_email', 'valid_tld', 'disposable', 'has_a_records', 'has_mx_records',
'has_spf_records', 'is_spoofable', 'dmarc_configured', 'dmarc_enforced', 'free_email',
'russian_free_email', 'china_free_email', 'suspicious_domain', 'dirty_words_domain',
'domain_popular', 'risky_tld', 'police_domain', 'government_domain', 'educational_domain',
'should_block']
for boolean_attribute in boolean_attributes:
email_verification.add_attribute(boolean_attribute,
**{'type': 'boolean', 'value': emaillookup['data'][boolean_attribute]})
email_verification.add_attribute('email', **{'type': 'email', 'value': emaillookup['data']['email']})
email_verification.add_attribute('username', **{'type': 'text', 'value': emaillookup['data']['username']})
email_verification.add_attribute('role_address',
**{'type': 'boolean', 'value': emaillookup['data']['role_address']})
email_verification.add_attribute('domain', **{'type': 'domain', 'value': emaillookup['data']['domain']})
email_verification.add_attribute('score', **{'type': 'float', 'value': emaillookup['data']['score']})
email_verification.add_reference(self.attribute['uuid'], 'related-to')
self.misp_event.add_object(email_verification)
def _handle_dns_record(self, item, record_type, relationship): def _handle_dns_record(self, item, record_type, relationship):
dns_record = MISPObject('dns-record') dns_record = MISPObject('dns-record')
dns_record.add_attribute('queried-domain', type='domain', value=item['host']) dns_record.add_attribute('queried-domain', type='domain', value=item['host'])
@ -82,7 +107,10 @@ def handler(q=False):
return {'error': 'Unsupported attribute type.'} return {'error': 'Unsupported attribute type.'}
apikey = request['config']['apikey'] apikey = request['config']['apikey']
apivoid_parser = APIVoidParser(attribute) apivoid_parser = APIVoidParser(attribute)
apivoid_parser.parse_domain(apikey) if attribute['type'] in ['domain', 'hostname']:
apivoid_parser.parse_domain(apikey)
else:
apivoid_parser.handle_email(apikey)
return apivoid_parser.get_results() return apivoid_parser.get_results()

129
misp_modules/modules/expansion/mmdb_lookup.py Normal file
View File

@ -0,0 +1,129 @@
import json
import requests
from . import check_input_attribute, standard_error_message
from pymisp import MISPEvent, MISPObject
misperrors = {'error': 'Error'}
mispattributes = {'input': ['ip-src', 'ip-src|port', 'ip-dst', 'ip-dst|port'], 'format': 'misp_standard'}
moduleinfo = {'version': '1', 'author': 'Jeroen Pinoy',
'description': "An expansion module to enrich an ip with geolocation information from an mmdb server "
"such as ip.circl.lu.",
'module-type': ['expansion', 'hover']}
moduleconfig = ["custom_API", "db_source_filter"]
mmdblookup_url = 'https://ip.circl.lu/'
class MmdbLookupParser():
def __init__(self, attribute, mmdblookupresult, api_url):
self.attribute = attribute
self.mmdblookupresult = mmdblookupresult
self.api_url = api_url
self.misp_event = MISPEvent()
self.misp_event.add_attribute(**attribute)
def get_result(self):
event = json.loads(self.misp_event.to_json())
results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])}
return {'results': results}
def parse_mmdblookup_information(self):
# There is a chance some db's have a hit while others don't so we have to check if entry is empty each time
for result_entry in self.mmdblookupresult:
if result_entry['country_info']:
mmdblookup_object = MISPObject('geolocation')
mmdblookup_object.add_attribute('country',
**{'type': 'text', 'value': result_entry['country_info']['Country']})
mmdblookup_object.add_attribute('countrycode',
**{'type': 'text', 'value': result_entry['country']['iso_code']})
mmdblookup_object.add_attribute('latitude',
**{'type': 'float',
'value': result_entry['country_info']['Latitude (average)']})
mmdblookup_object.add_attribute('longitude',
**{'type': 'float',
'value': result_entry['country_info']['Longitude (average)']})
mmdblookup_object.add_attribute('text',
**{'type': 'text',
'value': 'db_source: {}. build_db: {}. Latitude and longitude are country average.'.format(
result_entry['meta']['db_source'],
result_entry['meta']['build_db'])})
mmdblookup_object.add_reference(self.attribute['uuid'], 'related-to')
self.misp_event.add_object(mmdblookup_object)
if 'AutonomousSystemNumber' in result_entry['country']:
mmdblookup_object_asn = MISPObject('asn')
mmdblookup_object_asn.add_attribute('asn',
**{'type': 'text',
'value': result_entry['country'][
'AutonomousSystemNumber']})
mmdblookup_object_asn.add_attribute('description',
**{'type': 'text',
'value': 'ASNOrganization: {}. db_source: {}. build_db: {}.'.format(
result_entry['country'][
'AutonomousSystemOrganization'],
result_entry['meta']['db_source'],
result_entry['meta']['build_db'])})
mmdblookup_object_asn.add_reference(self.attribute['uuid'], 'related-to')
self.misp_event.add_object(mmdblookup_object_asn)
def check_url(url):
return "{}/".format(url) if not url.endswith('/') else url
def handler(q=False):
if q is False:
return False
request = json.loads(q)
if not request.get('attribute') or not check_input_attribute(request['attribute']):
return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'}
attribute = request['attribute']
if attribute.get('type') == 'ip-src':
toquery = attribute['value']
elif attribute.get('type') == 'ip-src|port':
toquery = attribute['value'].split('|')[0]
elif attribute.get('type') == 'ip-dst':
toquery = attribute['value']
elif attribute.get('type') == 'ip-dst|port':
toquery = attribute['value'].split('|')[0]
else:
misperrors['error'] = 'There is no attribute of type ip-src or ip-dst provided as input'
return misperrors
api_url = check_url(request['config']['custom_API']) if 'config' in request and request['config'].get(
'custom_API') else mmdblookup_url
r = requests.get("{}/geolookup/{}".format(api_url, toquery))
if r.status_code == 200:
mmdblookupresult = r.json()
if not mmdblookupresult or len(mmdblookupresult) == 0:
misperrors['error'] = 'Empty result returned by server'
return misperrors
if 'config' in request and request['config'].get('db_source_filter'):
db_source_filter = request['config'].get('db_source_filter')
mmdblookupresult = [entry for entry in mmdblookupresult if entry['meta']['db_source'] == db_source_filter]
if not mmdblookupresult or len(mmdblookupresult) == 0:
misperrors['error'] = 'There was no result with the selected db_source'
return misperrors
# Server might return one or multiple entries which could all be empty, we check if there is at least one
# non-empty result below
empty_result = True
for lookup_result_entry in mmdblookupresult:
if lookup_result_entry['country_info']:
empty_result = False
break
if empty_result:
misperrors['error'] = 'Empty result returned by server'
return misperrors
else:
misperrors['error'] = 'API not accessible - http status code {} was returned'.format(r.status_code)
return misperrors
parser = MmdbLookupParser(attribute, mmdblookupresult, api_url)
parser.parse_mmdblookup_information()
result = parser.get_result()
return result
def introspection():
return mispattributes
def version():
moduleinfo['config'] = moduleconfig
return moduleinfo