mirror of https://github.com/MISP/misp-modules
Merge branch 'main' into main
GitHub
commit
beb463bdab
|
|
@ -46,5 +46,7 @@ jobs:
|
|||
# Run server in background
|
||||
misp-modules -l 127.0.0.1 -s &
|
||||
sleep 5
|
||||
# Check if modules are running
|
||||
curl -sS localhost:6666/modules
|
||||
# Run tests
|
||||
pytest tests
|
||||
|
|
|
|||
|
|
@ -0,0 +1,4 @@
|
|||
[submodule "misp_modules/lib/misp-objects"]
|
||||
path = misp_modules/lib/misp-objects
|
||||
url = https://github.com/MISP/misp-objects.git
|
||||
branch = main
|
||||
24
Pipfile
24
Pipfile
|
|
@ -17,9 +17,9 @@ passivetotal = "*"
|
|||
pypdns = "*"
|
||||
pypssl = "*"
|
||||
pyeupi = "*"
|
||||
pymisp = { extras = ["fileobjects,openioc,pdfexport,email"], version = "*" }
|
||||
pyonyphe = { editable = true, git = "https://github.com/sebdraven/pyonyphe" }
|
||||
pydnstrails = { editable = true, git = "https://github.com/sebdraven/pydnstrails" }
|
||||
pymisp = { extras = ["fileobjects,openioc,pdfexport,email,url"], version = "*" }
|
||||
pyonyphe = { git = "https://github.com/sebdraven/pyonyphe" }
|
||||
pydnstrails = { git = "https://github.com/sebdraven/pydnstrails" }
|
||||
pytesseract = "*"
|
||||
pygeoip = "*"
|
||||
beautifulsoup4 = "*"
|
||||
|
|
@ -31,20 +31,20 @@ maclookup = "*"
|
|||
vulners = "*"
|
||||
blockchain = "*"
|
||||
reportlab = "*"
|
||||
pyintel471 = { editable = true, git = "https://github.com/MISP/PyIntel471.git" }
|
||||
pyintel471 = { git = "https://github.com/MISP/PyIntel471.git" }
|
||||
shodan = "*"
|
||||
Pillow = ">=8.2.0"
|
||||
Wand = "*"
|
||||
SPARQLWrapper = "*"
|
||||
domaintools_api = "*"
|
||||
misp-modules = { editable = true, path = "." }
|
||||
pybgpranking = { editable = true, git = "https://github.com/D4-project/BGP-Ranking.git/", subdirectory = "client" }
|
||||
pyipasnhistory = { editable = true, git = "https://github.com/D4-project/IPASN-History.git/", subdirectory = "client" }
|
||||
misp-modules = { path = "." }
|
||||
pybgpranking = { git = "https://github.com/D4-project/BGP-Ranking.git/", subdirectory = "client", ref = "68de39f6c5196f796055c1ac34504054d688aa59" }
|
||||
pyipasnhistory = { git = "https://github.com/D4-project/IPASN-History.git/", subdirectory = "client", ref = "a2853c39265cecdd0c0d16850bd34621c0551b87" }
|
||||
backscatter = "*"
|
||||
pyzbar = "*"
|
||||
opencv-python = "*"
|
||||
np = "*"
|
||||
ODTReader = { editable = true, git = "https://github.com/cartertemm/ODTReader.git/" }
|
||||
ODTReader = { git = "https://github.com/cartertemm/ODTReader.git/" }
|
||||
python-pptx = "*"
|
||||
python-docx = "*"
|
||||
ezodf = "*"
|
||||
|
|
@ -59,7 +59,7 @@ geoip2 = "*"
|
|||
apiosintDS = "*"
|
||||
assemblyline_client = "*"
|
||||
vt-graph-api = "*"
|
||||
trustar = { editable = true, git = "https://github.com/SteveClement/trustar-python.git" }
|
||||
trustar = { git = "https://github.com/SteveClement/trustar-python.git" }
|
||||
markdownify = "==0.5.3"
|
||||
socialscan = "*"
|
||||
dnsdb2 = "*"
|
||||
|
|
@ -67,6 +67,10 @@ clamd = "*"
|
|||
aiohttp = ">=3.7.4"
|
||||
tau-clients = "*"
|
||||
vt-py = ">=0.7.1"
|
||||
crowdstrike-falconpy = "0.9.0"
|
||||
censys = "2.0.9"
|
||||
mwdblib = "3.4.1"
|
||||
ndjson = "0.3.1"
|
||||
|
||||
[requires]
|
||||
python_version = "3.6"
|
||||
python_version = "3.7"
|
||||
|
|
|
|||
|
|
@ -58,6 +58,7 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj
|
|||
* [macvendors](misp_modules/modules/expansion/macvendors.py) - a hover module to retrieve mac vendor information.
|
||||
* [MALWAREbazaar](misp_modules/modules/expansion/malwarebazaar.py) - an expansion module to query MALWAREbazaar with some payload.
|
||||
* [McAfee MVISION Insights](misp_modules/modules/expansion/mcafee_insights_enrich.py) - an expansion module enrich IOCs with McAfee MVISION Insights.
|
||||
* [Mmdb server lookup](misp_modules/modules/expansion/mmdb_lookup.py) - an expansion module to enrich an ip with geolocation information from an mmdb server such as ip.circl.lu.
|
||||
* [ocr-enrich](misp_modules/modules/expansion/ocr_enrich.py) - an enrichment module to get OCRized data from images into MISP.
|
||||
* [ods-enrich](misp_modules/modules/expansion/ods_enrich.py) - an enrichment module to get text out of OpenOffice spreadsheet document into MISP (using free-text parser).
|
||||
* [odt-enrich](misp_modules/modules/expansion/odt_enrich.py) - an enrichment module to get text out of OpenOffice document into MISP (using free-text parser).
|
||||
|
|
|
|||
183
REQUIREMENTS
183
REQUIREMENTS
|
|
@ -6,150 +6,165 @@
|
|||
#
|
||||
|
||||
-i https://pypi.org/simple
|
||||
-e .
|
||||
-e git+https://github.com/D4-project/BGP-Ranking.git/@68de39f6c5196f796055c1ac34504054d688aa59#egg=pybgpranking&subdirectory=client
|
||||
-e git+https://github.com/D4-project/IPASN-History.git/@a2853c39265cecdd0c0d16850bd34621c0551b87#egg=pyipasnhistory&subdirectory=client
|
||||
-e git+https://github.com/MISP/PyIntel471.git@917272fafa8e12102329faca52173e90c5256968#egg=pyintel471
|
||||
-e git+https://github.com/cartertemm/ODTReader.git/@49d6938693f6faa3ff09998f86dba551ae3a996b#egg=odtreader
|
||||
-e git+https://github.com/sebdraven/pydnstrails@48c1f740025c51289f43a24863d1845ff12fd21a#egg=pydnstrails
|
||||
-e git+https://github.com/sebdraven/pyonyphe@1ce15581beebb13e841193a08a2eb6f967855fcb#egg=pyonyphe
|
||||
git+https://github.com/SteveClement/trustar-python.git
|
||||
aiohttp==3.7.4
|
||||
.
|
||||
aiohttp==3.8.1
|
||||
aiosignal==1.2.0; python_version >= '3.6'
|
||||
antlr4-python3-runtime==4.8; python_version >= '3'
|
||||
apiosintds==1.8.3
|
||||
appdirs==1.4.4
|
||||
argparse==1.4.0
|
||||
assemblyline-client==4.1.0
|
||||
async-timeout==3.0.1; python_full_version >= '3.5.3'
|
||||
attrs==21.2.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
|
||||
assemblyline-client==4.2.2
|
||||
async-timeout==4.0.2; python_version >= '3.6'
|
||||
asynctest==0.13.0; python_version < '3.8'
|
||||
attrs==21.4.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
|
||||
backoff==1.11.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
|
||||
backports.zoneinfo==0.2.1; python_version < '3.9'
|
||||
backscatter==0.2.4
|
||||
beautifulsoup4==4.9.3
|
||||
bidict==0.21.2; python_version >= '3.6'
|
||||
beautifulsoup4==4.10.0
|
||||
bidict==0.21.4; python_version >= '3.6'
|
||||
blockchain==1.4.4
|
||||
certifi==2021.5.30
|
||||
censys==2.0.9
|
||||
cffi==1.14.6
|
||||
#chardet==4.0.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
|
||||
chardet
|
||||
charset-normalizer==2.0.4; python_version >= '3'
|
||||
censys==2.1.2
|
||||
certifi==2021.10.8
|
||||
cffi==1.15.0
|
||||
chardet==4.0.0
|
||||
charset-normalizer==2.0.11; python_version >= '3'
|
||||
clamd==1.0.2
|
||||
click-plugins==1.1.1
|
||||
click==8.0.1; python_version >= '3.6'
|
||||
click==8.0.3; python_version >= '3.6'
|
||||
colorama==0.4.4; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
|
||||
colorclass==2.2.0
|
||||
colorclass==2.2.2; python_version >= '2.6'
|
||||
commonmark==0.9.1
|
||||
compressed-rtf==1.0.6
|
||||
configparser==5.0.2; python_version >= '3.6'
|
||||
crowdstrike-falconpy==0.9.0
|
||||
cryptography==3.4.7; python_version >= '3.6'
|
||||
decorator==5.0.9; python_version >= '3.5'
|
||||
deprecated==1.2.12; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||
configparser==5.2.0; python_version >= '3.6'
|
||||
crowdstrike-falconpy==1.0.0
|
||||
cryptography==36.0.1; python_version >= '3.6'
|
||||
decorator==5.1.1; python_version >= '3.5'
|
||||
deprecated==1.2.13; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||
dnsdb2==1.1.3
|
||||
dnspython==2.1.0
|
||||
domaintools-api==0.5.4
|
||||
dnspython==2.2.0
|
||||
domaintools-api==0.6.1
|
||||
easygui==0.98.2
|
||||
ebcdic==1.1.1
|
||||
enum-compat==0.0.3
|
||||
extract-msg==0.28.7
|
||||
ez-setup==0.9
|
||||
ezodf==0.3.2
|
||||
filelock==3.0.12
|
||||
filelock==3.4.2; python_version >= '3.7'
|
||||
frozenlist==1.3.0; python_version >= '3.7'
|
||||
future==0.18.2; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||
futures==3.1.1
|
||||
geoip2==4.2.0
|
||||
httplib2==0.19.1
|
||||
geoip2==4.5.0
|
||||
git+https://github.com/D4-project/BGP-Ranking.git/@68de39f6c5196f796055c1ac34504054d688aa59#egg=pybgpranking&subdirectory=client
|
||||
git+https://github.com/D4-project/IPASN-History.git/@a2853c39265cecdd0c0d16850bd34621c0551b87#egg=pyipasnhistory&subdirectory=client
|
||||
git+https://github.com/MISP/PyIntel471.git@917272fafa8e12102329faca52173e90c5256968#egg=pyintel471
|
||||
git+https://github.com/SteveClement/trustar-python.git@6954eae38e0c77eaeef26084b6c5fd033925c1c7#egg=trustar
|
||||
git+https://github.com/cartertemm/ODTReader.git/@49d6938693f6faa3ff09998f86dba551ae3a996b#egg=odtreader
|
||||
git+https://github.com/sebdraven/pydnstrails@48c1f740025c51289f43a24863d1845ff12fd21a#egg=pydnstrails
|
||||
git+https://github.com/sebdraven/pyonyphe@aed008ee5a27e3a5e4afbb3e5cbfc47170108452#egg=pyonyphe
|
||||
httplib2==0.20.4; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||
idna-ssl==1.1.0; python_version < '3.7'
|
||||
idna==3.2; python_version >= '3'
|
||||
idna==3.3; python_version >= '3'
|
||||
imapclient==2.1.0
|
||||
isodate==0.6.0
|
||||
importlib-metadata==4.10.1; python_version < '3.8'
|
||||
isodate==0.6.1
|
||||
itsdangerous==2.0.1; python_version >= '3.6'
|
||||
jbxapi==3.17.2
|
||||
json-log-formatter==0.4.0
|
||||
jeepney==0.7.1; sys_platform == 'linux'
|
||||
json-log-formatter==0.5.1
|
||||
jsonschema==3.2.0
|
||||
lark-parser==0.11.3
|
||||
keyring==23.5.0; python_version >= '3.7'
|
||||
lark-parser==0.12.0
|
||||
lief==0.11.5
|
||||
lxml==4.7.1
|
||||
maclookup==1.0.3
|
||||
markdownify==0.5.3
|
||||
maxminddb==2.0.3; python_version >= '3.6'
|
||||
more-itertools==8.8.0; python_version >= '3.5'
|
||||
msoffcrypto-tool==4.12.0; python_version >= '3' and platform_python_implementation != 'PyPy' or (platform_system != 'Windows' and platform_system != 'Darwin')
|
||||
multidict==5.1.0; python_version >= '3.6'
|
||||
maxminddb==2.2.0; python_version >= '3.6'
|
||||
more-itertools==8.12.0; python_version >= '3.5'
|
||||
msoffcrypto-tool==5.0.0; python_version >= '3' and platform_python_implementation != 'PyPy' or (platform_system != 'Windows' and platform_system != 'Darwin')
|
||||
multidict==6.0.2; python_version >= '3.7'
|
||||
mwdblib==4.0.0
|
||||
ndjson==0.3.1
|
||||
np==1.0.2
|
||||
numpy==1.21.2; python_version < '3.11' and python_version >= '3.7'
|
||||
numpy==1.21.5; python_version < '3.10' and platform_machine != 'aarch64' and platform_machine != 'arm64'
|
||||
oauth2==1.9.0.post1
|
||||
olefile==0.46; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||
oletools==0.56.2
|
||||
opencv-python==4.5.3.56
|
||||
oletools==0.60
|
||||
opencv-python==4.5.5.62
|
||||
packaging==21.3; python_version >= '3.6'
|
||||
pandas-ods-reader==0.1.2
|
||||
pandas==1.3.5
|
||||
passivetotal==2.5.4
|
||||
passivetotal==2.5.8
|
||||
pcodedmp==1.2.6
|
||||
pdftotext==2.2.0
|
||||
pillow==8.3.2
|
||||
progressbar2==3.53.1
|
||||
psutil==5.8.0; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||
pycparser==2.20; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||
pycryptodome==3.10.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
|
||||
pycryptodomex==3.10.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
|
||||
pdftotext==2.2.2
|
||||
pillow==9.0.1
|
||||
progressbar2==4.0.0; python_version >= '3.7'
|
||||
psutil==5.9.0; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||
pycparser==2.21
|
||||
pycryptodome==3.14.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
|
||||
pycryptodomex==3.14.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
|
||||
pydeep==0.4
|
||||
pyeupi==1.1
|
||||
pyfaup==1.2
|
||||
pygeoip==0.3.2
|
||||
pymisp[email,fileobjects,openioc,pdfexport]==2.4.148
|
||||
pygments==2.11.2; python_version >= '3.5'
|
||||
pymisp[email,fileobjects,openioc,pdfexport,url]==2.4.152
|
||||
pyparsing==2.4.7; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||
pypdns==1.5.2
|
||||
pypssl==2.2
|
||||
pyrsistent==0.18.0; python_version >= '3.6'
|
||||
pyrsistent==0.18.1; python_version >= '3.7'
|
||||
pytesseract==0.3.8
|
||||
python-baseconv==1.2.2
|
||||
python-dateutil==2.8.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||
python-docx==0.8.11
|
||||
python-engineio==4.2.1; python_version >= '3.6'
|
||||
python-magic==0.4.24
|
||||
python-pptx==0.6.19
|
||||
python-socketio[client]==5.4.0; python_version >= '3.6'
|
||||
python-utils==2.5.6
|
||||
python-engineio==4.3.1; python_version >= '3.6'
|
||||
python-magic==0.4.25
|
||||
python-pptx==0.6.21
|
||||
python-socketio[client]==5.5.1; python_version >= '3.6'
|
||||
python-utils==3.1.0; python_version >= '3.7'
|
||||
pytz-deprecation-shim==0.1.0.post0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
|
||||
pytz==2019.3
|
||||
pyyaml==5.4.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
|
||||
pyyaml==6.0; python_version >= '3.6'
|
||||
pyzbar==0.1.8
|
||||
pyzipper==0.3.5; python_version >= '3.5'
|
||||
rdflib==6.0.0; python_version >= '3.7'
|
||||
redis==3.5.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
|
||||
reportlab==3.6.1
|
||||
rdflib==6.1.1; python_version >= '3.7'
|
||||
redis==4.1.2; python_version >= '3.6'
|
||||
reportlab==3.6.6
|
||||
requests-cache==0.6.4; python_version >= '3.6'
|
||||
requests-file==1.5.1
|
||||
requests[security]==2.26.0
|
||||
requests[security]==2.27.1
|
||||
rich==11.1.0; python_full_version >= '3.6.2' and python_full_version < '4.0.0'
|
||||
rtfde==0.0.2
|
||||
ruamel.yaml.clib==0.2.6; python_version < '3.10' and platform_python_implementation == 'CPython'
|
||||
ruamel.yaml==0.17.13; python_version >= '3'
|
||||
shodan==1.25.0
|
||||
secretstorage==3.3.1; sys_platform == 'linux'
|
||||
setuptools==60.7.1; python_version >= '3.7'
|
||||
shodan==1.26.1
|
||||
sigmatools==0.19.1
|
||||
six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||
socialscan==1.4.2
|
||||
socketio-client==0.5.7.4
|
||||
soupsieve==2.2.1; python_version >= '3'
|
||||
soupsieve==2.3.1; python_version >= '3.6'
|
||||
sparqlwrapper==1.8.5
|
||||
stix2==3.0.1
|
||||
stix2-patterns==1.3.2
|
||||
tabulate==0.8.9
|
||||
tau-clients==0.1.3
|
||||
tldextract==3.1.0; python_version >= '3.5'
|
||||
tau-clients==0.1.9
|
||||
taxii2-client==2.3.0
|
||||
tldextract==3.1.2; python_version >= '3.6'
|
||||
tornado==6.1; python_version >= '3.5'
|
||||
tqdm==4.62.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||
typing-extensions==3.10.0.0
|
||||
tzlocal==3.0; python_version >= '3.6'
|
||||
tqdm==4.62.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||
typing-extensions==4.0.1; python_version < '3.8'
|
||||
tzdata==2021.5; python_version >= '3.6'
|
||||
tzlocal==4.1; python_version >= '3.6'
|
||||
unicodecsv==0.14.1
|
||||
url-normalize==1.4.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
|
||||
urlarchiver==0.2
|
||||
urllib3==1.26.6; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'
|
||||
urllib3==1.26.8; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_full_version < '4.0.0'
|
||||
validators==0.14.0
|
||||
vt-graph-api==1.1.2
|
||||
vt-py==0.7.2
|
||||
vulners==1.5.12
|
||||
vt-graph-api==1.1.3
|
||||
vt-py==0.13.1
|
||||
vulners==2.0.0
|
||||
wand==0.6.7
|
||||
websocket-client==1.2.1
|
||||
wrapt==1.12.1
|
||||
websocket-client==1.2.3; python_version >= '3.6'
|
||||
wrapt==1.13.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
|
||||
xlrd==2.0.1
|
||||
xlsxwriter==3.0.1; python_version >= '3.4'
|
||||
xlsxwriter==3.0.2; python_version >= '3.4'
|
||||
yara-python==3.8.1
|
||||
yarl==1.6.3; python_version >= '3.6'
|
||||
ndjson==0.3.1
|
||||
mwdblib==3.4.1
|
||||
yarl==1.7.2; python_version >= '3.6'
|
||||
zipp==3.7.0; python_version >= '3.7'
|
||||
|
|
|
|||
|
|
@ -2045,6 +2045,25 @@ Module to export a structured CSV file for uploading to ThreatConnect.
|
|||
|
||||
-----
|
||||
|
||||
#### [virustotal_collections](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/export_mod/virustotal_collections.py)
|
||||
|
||||
<img src=logos/virustotal.png height=60>
|
||||
|
||||
Creates a VT Collection from an event iocs.
|
||||
- **features**:
|
||||
>This export module which takes advantage of a new endpoint in VT APIv3 to create VT Collections from IOCs contained in a MISP event. With this module users will be able to create a collection just using the Download as... button.
|
||||
- **input**:
|
||||
>A domain, hash (md5, sha1, sha256 or sha512), hostname, url or IP address attribute.
|
||||
- **output**:
|
||||
>A VirusTotal collection in VT.
|
||||
- **references**:
|
||||
> - https://www.virustotal.com/
|
||||
> - https://blog.virustotal.com/2021/11/introducing-virustotal-collections.html
|
||||
- **requirements**:
|
||||
>An access to the VirusTotal API (apikey).
|
||||
|
||||
-----
|
||||
|
||||
#### [vt_graph](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/export_mod/vt_graph.py)
|
||||
|
||||
<img src=logos/virustotal.png height=60>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,14 @@
|
|||
{
|
||||
"description": "Creates a VT Collection from an event iocs.",
|
||||
"logo": "virustotal.png",
|
||||
"requirements": [
|
||||
"An access to the VirusTotal API (apikey)."
|
||||
],
|
||||
"input": "A domain, hash (md5, sha1, sha256 or sha512), hostname, url or IP address attribute.",
|
||||
"output": "A VirusTotal collection in VT.",
|
||||
"references": [
|
||||
"https://www.virustotal.com/",
|
||||
"https://blog.virustotal.com/2021/11/introducing-virustotal-collections.html"
|
||||
],
|
||||
"features": "This export module which takes advantage of a new endpoint in VT APIv3 to create VT Collections from IOCs contained in a MISP event. With this module users will be able to create a collection just using the Download as... button."
|
||||
}
|
||||
|
|
@ -0,0 +1 @@
|
|||
Subproject commit 9dc7e3578f2165e32a3b7cdd09e9e552f2d98d36
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,460 @@
|
|||
################################################################################
|
||||
# ATTRIBUTES AND OBJECTS MAPPING #
|
||||
################################################################################
|
||||
|
||||
attributes_mapping = {
|
||||
'filename': '_parse_name',
|
||||
'ip-src': '_parse_value',
|
||||
'ip-dst': '_parse_value',
|
||||
'hostname': '_parse_value',
|
||||
'domain': '_parse_value',
|
||||
'domain|ip': '_parse_domain_ip_attribute',
|
||||
'email-src': '_parse_value',
|
||||
'email-dst': '_parse_value',
|
||||
'email-attachment': '_parse_name',
|
||||
'url': '_parse_value',
|
||||
'regkey': '_parse_regkey_attribute',
|
||||
'regkey|value': '_parse_regkey_value',
|
||||
'malware-sample': '_parse_malware_sample',
|
||||
'mutex': '_parse_name',
|
||||
'uri': '_parse_value',
|
||||
'port': '_parse_port',
|
||||
'ip-dst|port': '_parse_network_attribute',
|
||||
'ip-src|port': '_parse_network_attribute',
|
||||
'hostname|port': '_parse_network_attribute',
|
||||
'email-reply-to': '_parse_email_reply_to',
|
||||
'attachment': '_parse_attachment',
|
||||
'mac-address': '_parse_value',
|
||||
'AS': '_parse_number'
|
||||
}
|
||||
|
||||
attributes_type_mapping = {
|
||||
'md5': '_parse_hash',
|
||||
'sha1': '_parse_hash',
|
||||
'sha256': '_parse_hash',
|
||||
'filename|md5': '_parse_filename_hash',
|
||||
'filename|sha1': '_parse_filename_hash',
|
||||
'filename|sha256': '_parse_filename_hash',
|
||||
'email-subject': '_parse_email_message',
|
||||
'email-body': '_parse_email_message',
|
||||
'authentihash': '_parse_hash',
|
||||
'ssdeep': '_parse_hash',
|
||||
'imphash': '_parse_hash',
|
||||
'pehash': '_parse_hash',
|
||||
'impfuzzy': '_parse_hash',
|
||||
'sha224': '_parse_hash',
|
||||
'sha384': '_parse_hash',
|
||||
'sha512': '_parse_hash',
|
||||
'sha512/224': '_parse_hash',
|
||||
'sha512/256': '_parse_hash',
|
||||
'tlsh': '_parse_hash',
|
||||
'cdhash': '_parse_hash',
|
||||
'filename|authentihash': '_parse_filename_hash',
|
||||
'filename|ssdeep': '_parse_filename_hash',
|
||||
'filename|imphash': '_parse_filename_hash',
|
||||
'filename|impfuzzy': '_parse_filename_hash',
|
||||
'filename|pehash': '_parse_filename_hash',
|
||||
'filename|sha224': '_parse_filename_hash',
|
||||
'filename|sha384': '_parse_filename_hash',
|
||||
'filename|sha512': '_parse_filename_hash',
|
||||
'filename|sha512/224': '_parse_filename_hash',
|
||||
'filename|sha512/256': '_parse_filename_hash',
|
||||
'filename|tlsh': '_parse_filename_hash',
|
||||
'x509-fingerprint-md5': '_parse_x509_attribute',
|
||||
'x509-fingerprint-sha1': '_parse_x509_attribute',
|
||||
'x509-fingerprint-sha256': '_parse_x509_attribute'
|
||||
}
|
||||
|
||||
objects_mapping = {
|
||||
'asn': {
|
||||
'observable': 'parse_asn_observable',
|
||||
'pattern': 'parse_asn_pattern'},
|
||||
'credential': {
|
||||
'observable': 'parse_credential_observable',
|
||||
'pattern': 'parse_credential_pattern'},
|
||||
'domain-ip': {
|
||||
'observable': 'parse_domain_ip_observable',
|
||||
'pattern': 'parse_domain_ip_pattern'},
|
||||
'email': {
|
||||
'observable': 'parse_email_observable',
|
||||
'pattern': 'parse_email_pattern'},
|
||||
'file': {
|
||||
'observable': 'parse_file_observable',
|
||||
'pattern': 'parse_file_pattern'},
|
||||
'ip-port': {
|
||||
'observable': 'parse_ip_port_observable',
|
||||
'pattern': 'parse_ip_port_pattern'},
|
||||
'network-connection': {
|
||||
'observable': 'parse_network_connection_observable',
|
||||
'pattern': 'parse_network_connection_pattern'},
|
||||
'network-socket': {
|
||||
'observable': 'parse_network_socket_observable',
|
||||
'pattern': 'parse_network_socket_pattern'},
|
||||
'process': {
|
||||
'observable': 'parse_process_observable',
|
||||
'pattern': 'parse_process_pattern'},
|
||||
'registry-key': {
|
||||
'observable': 'parse_regkey_observable',
|
||||
'pattern': 'parse_regkey_pattern'},
|
||||
'url': {
|
||||
'observable': 'parse_url_observable',
|
||||
'pattern': 'parse_url_pattern'},
|
||||
'user-account': {
|
||||
'observable': 'parse_user_account_observable',
|
||||
'pattern': 'parse_user_account_pattern'},
|
||||
'WindowsPEBinaryFile': {
|
||||
'observable': 'parse_pe_observable',
|
||||
'pattern': 'parse_pe_pattern'},
|
||||
'x509': {
|
||||
'observable': 'parse_x509_observable',
|
||||
'pattern': 'parse_x509_pattern'}
|
||||
}
|
||||
|
||||
observable_mapping = {
|
||||
('artifact', 'file'): 'parse_file_observable',
|
||||
('artifact', 'directory', 'file'): 'parse_file_observable',
|
||||
('artifact', 'email-addr', 'email-message', 'file'): 'parse_email_observable',
|
||||
('autonomous-system',): 'parse_asn_observable',
|
||||
('autonomous-system', 'ipv4-addr'): 'parse_asn_observable',
|
||||
('autonomous-system', 'ipv6-addr'): 'parse_asn_observable',
|
||||
('autonomous-system', 'ipv4-addr', 'ipv6-addr'): 'parse_asn_observable',
|
||||
('directory', 'file'): 'parse_file_observable',
|
||||
('domain-name',): 'parse_domain_ip_observable',
|
||||
('domain-name', 'ipv4-addr'): 'parse_domain_ip_observable',
|
||||
('domain-name', 'ipv6-addr'): 'parse_domain_ip_observable',
|
||||
('domain-name', 'ipv4-addr', 'ipv6-addr'): 'parse_domain_ip_observable',
|
||||
('domain-name', 'ipv4-addr', 'network-traffic'): 'parse_domain_ip_network_traffic_observable',
|
||||
('domain-name', 'ipv6-addr', 'network-traffic'): 'parse_domain_ip_network_traffic_observable',
|
||||
('domain-name', 'ipv4-addr', 'ipv6-addr', 'network-traffic'): 'parse_domain_ip_network_traffic_observable',
|
||||
('domain-name', 'network-traffic'): 'parse_domain_network_traffic_observable',
|
||||
('domain-name', 'network-traffic', 'url'): 'parse_url_observable',
|
||||
('email-addr',): 'parse_email_address_observable',
|
||||
('email-addr', 'email-message'): 'parse_email_observable',
|
||||
('email-addr', 'email-message', 'file'): 'parse_email_observable',
|
||||
('email-message',): 'parse_email_observable',
|
||||
('file',): 'parse_file_observable',
|
||||
('file', 'process'): 'parse_process_observable',
|
||||
('ipv4-addr',): 'parse_ip_address_observable',
|
||||
('ipv6-addr',): 'parse_ip_address_observable',
|
||||
('ipv4-addr', 'network-traffic'): 'parse_ip_network_traffic_observable',
|
||||
('ipv6-addr', 'network-traffic'): 'parse_ip_network_traffic_observable',
|
||||
('ipv4-addr', 'ipv6-addr', 'network-traffic'): 'parse_ip_network_traffic_observable',
|
||||
('mac-addr',): 'parse_mac_address_observable',
|
||||
('mutex',): 'parse_mutex_observable',
|
||||
('process',): 'parse_process_observable',
|
||||
('x509-certificate',): 'parse_x509_observable',
|
||||
('url',): 'parse_url_observable',
|
||||
('user-account',): 'parse_user_account_observable',
|
||||
('windows-registry-key',): 'parse_regkey_observable'
|
||||
}
|
||||
|
||||
pattern_mapping = {
|
||||
('artifact', 'file'): 'parse_file_pattern',
|
||||
('artifact', 'directory', 'file'): 'parse_file_pattern',
|
||||
('autonomous-system', ): 'parse_as_pattern',
|
||||
('autonomous-system', 'ipv4-addr'): 'parse_as_pattern',
|
||||
('autonomous-system', 'ipv6-addr'): 'parse_as_pattern',
|
||||
('autonomous-system', 'ipv4-addr', 'ipv6-addr'): 'parse_as_pattern',
|
||||
('directory',): 'parse_file_pattern',
|
||||
('directory', 'file'): 'parse_file_pattern',
|
||||
('domain-name',): 'parse_domain_ip_port_pattern',
|
||||
('domain-name', 'ipv4-addr'): 'parse_domain_ip_port_pattern',
|
||||
('domain-name', 'ipv6-addr'): 'parse_domain_ip_port_pattern',
|
||||
('domain-name', 'ipv4-addr', 'ipv6-addr'): 'parse_domain_ip_port_pattern',
|
||||
('domain-name', 'ipv4-addr', 'url'): 'parse_url_pattern',
|
||||
('domain-name', 'ipv6-addr', 'url'): 'parse_url_pattern',
|
||||
('domain-name', 'ipv4-addr', 'ipv6-addr', 'url'): 'parse_url_pattern',
|
||||
('domain-name', 'network-traffic'): 'parse_domain_ip_port_pattern',
|
||||
('domain-name', 'network-traffic', 'url'): 'parse_url_pattern',
|
||||
('email-addr',): 'parse_email_address_pattern',
|
||||
('email-message',): 'parse_email_message_pattern',
|
||||
('file',): 'parse_file_pattern',
|
||||
('ipv4-addr',): 'parse_ip_address_pattern',
|
||||
('ipv6-addr',): 'parse_ip_address_pattern',
|
||||
('ipv4-addr', 'ipv6-addr'): 'parse_ip_address_pattern',
|
||||
('mac-addr',): 'parse_mac_address_pattern',
|
||||
('mutex',): 'parse_mutex_pattern',
|
||||
('network-traffic',): 'parse_network_traffic_pattern',
|
||||
('process',): 'parse_process_pattern',
|
||||
('url',): 'parse_url_pattern',
|
||||
('user-account',): 'parse_user_account_pattern',
|
||||
('windows-registry-key',): 'parse_regkey_pattern',
|
||||
('x509-certificate',): 'parse_x509_pattern'
|
||||
}
|
||||
|
||||
pattern_forbidden_relations = (' LIKE ', ' FOLLOWEDBY ', ' MATCHES ', ' ISSUBSET ', ' ISSUPERSET ', ' REPEATS ')
|
||||
single_attribute_fields = ('type', 'value', 'to_ids')
|
||||
|
||||
|
||||
################################################################################
|
||||
# OBSERVABLE OBJECTS AND PATTERNS MAPPING. #
|
||||
################################################################################
|
||||
|
||||
address_family_attribute_mapping = {'type': 'text','object_relation': 'address-family'}
|
||||
as_number_attribute_mapping = {'type': 'AS', 'object_relation': 'asn'}
|
||||
description_attribute_mapping = {'type': 'text', 'object_relation': 'description'}
|
||||
asn_subnet_attribute_mapping = {'type': 'ip-src', 'object_relation': 'subnet-announced'}
|
||||
cc_attribute_mapping = {'type': 'email-dst', 'object_relation': 'cc'}
|
||||
credential_attribute_mapping = {'type': 'text', 'object_relation': 'password'}
|
||||
data_attribute_mapping = {'type': 'text', 'object_relation': 'data'}
|
||||
data_type_attribute_mapping = {'type': 'text', 'object_relation': 'data-type'}
|
||||
domain_attribute_mapping = {'type': 'domain', 'object_relation': 'domain'}
|
||||
domain_family_attribute_mapping = {'type': 'text', 'object_relation': 'domain-family'}
|
||||
dst_port_attribute_mapping = {'type': 'port', 'object_relation': 'dst-port'}
|
||||
email_attachment_attribute_mapping = {'type': 'email-attachment', 'object_relation': 'attachment'}
|
||||
email_date_attribute_mapping = {'type': 'datetime', 'object_relation': 'send-date'}
|
||||
email_subject_attribute_mapping = {'type': 'email-subject', 'object_relation': 'subject'}
|
||||
encoding_attribute_mapping = {'type': 'text', 'object_relation': 'file-encoding'}
|
||||
end_datetime_attribute_mapping = {'type': 'datetime', 'object_relation': 'last-seen'}
|
||||
entropy_mapping = {'type': 'float', 'object_relation': 'entropy'}
|
||||
filename_attribute_mapping = {'type': 'filename', 'object_relation': 'filename'}
|
||||
from_attribute_mapping = {'type': 'email-src', 'object_relation': 'from'}
|
||||
imphash_mapping = {'type': 'imphash', 'object_relation': 'imphash'}
|
||||
id_attribute_mapping = {'type': 'text', 'object_relation': 'id'}
|
||||
ip_attribute_mapping = {'type': 'ip-dst', 'object_relation': 'ip'}
|
||||
issuer_attribute_mapping = {'type': 'text', 'object_relation': 'issuer'}
|
||||
key_attribute_mapping = {'type': 'regkey', 'object_relation': 'key'}
|
||||
malware_sample_attribute_mapping = {'type': 'malware-sample', 'object_relation': 'malware-sample'}
|
||||
mime_type_attribute_mapping = {'type': 'mime-type', 'object_relation': 'mimetype'}
|
||||
modified_attribute_mapping = {'type': 'datetime', 'object_relation': 'last-modified'}
|
||||
name_attribute_mapping = {'type': 'text', 'object_relation': 'name'}
|
||||
network_traffic_ip = {'type': 'ip-{}', 'object_relation': 'ip-{}'}
|
||||
number_sections_mapping = {'type': 'counter', 'object_relation': 'number-sections'}
|
||||
password_mapping = {'type': 'text', 'object_relation': 'password'}
|
||||
path_attribute_mapping = {'type': 'text', 'object_relation': 'path'}
|
||||
pe_type_mapping = {'type': 'text', 'object_relation': 'type'}
|
||||
pid_attribute_mapping = {'type': 'text', 'object_relation': 'pid'}
|
||||
process_command_line_mapping = {'type': 'text', 'object_relation': 'command-line'}
|
||||
process_creation_time_mapping = {'type': 'datetime', 'object_relation': 'creation-time'}
|
||||
process_image_mapping = {'type': 'filename', 'object_relation': 'image'}
|
||||
process_name_mapping = {'type': 'text', 'object_relation': 'name'}
|
||||
regkey_name_attribute_mapping = {'type': 'text', 'object_relation': 'name'}
|
||||
references_attribute_mapping = {'type': 'link', 'object_relation': 'references'}
|
||||
reply_to_attribute_mapping = {'type': 'email-reply-to', 'object_relation': 'reply-to'}
|
||||
screenshot_attribute_mapping = {'type': 'attachment', 'object_relation': 'screenshot'}
|
||||
section_name_mapping = {'type': 'text', 'object_relation': 'name'}
|
||||
serial_number_attribute_mapping = {'type': 'text', 'object_relation': 'serial-number'}
|
||||
size_attribute_mapping = {'type': 'size-in-bytes', 'object_relation': 'size-in-bytes'}
|
||||
src_port_attribute_mapping = {'type': 'port', 'object_relation': 'src-port'}
|
||||
start_datetime_attribute_mapping = {'type': 'datetime', 'object_relation': 'first-seen'}
|
||||
state_attribute_mapping = {'type': 'text', 'object_relation': 'state'}
|
||||
summary_attribute_mapping = {'type': 'text', 'object_relation': 'summary'}
|
||||
to_attribute_mapping = {'type': 'email-dst', 'object_relation': 'to'}
|
||||
url_attribute_mapping = {'type': 'url', 'object_relation': 'url'}
|
||||
url_port_attribute_mapping = {'type': 'port', 'object_relation': 'port'}
|
||||
user_id_mapping = {'type': 'text', 'object_relation': 'username'}
|
||||
x_mailer_attribute_mapping = {'type': 'email-x-mailer', 'object_relation': 'x-mailer'}
|
||||
x509_md5_attribute_mapping = {'type': 'x509-fingerprint-md5', 'object_relation': 'x509-fingerprint-md5'}
|
||||
x509_sha1_attribute_mapping = {'type': 'x509-fingerprint-sha1', 'object_relation': 'x509-fingerprint-sha1'}
|
||||
x509_sha256_attribute_mapping = {'type': 'x509-fingerprint-sha256', 'object_relation': 'x509-fingerprint-sha256'}
|
||||
x509_spka_attribute_mapping = {'type': 'text', 'object_relation': 'pubkey-info-algorithm'} # x509 subject public key algorithm
|
||||
x509_spke_attribute_mapping = {'type': 'text', 'object_relation': 'pubkey-info-exponent'} # x509 subject public key exponent
|
||||
x509_spkm_attribute_mapping = {'type': 'text', 'object_relation': 'pubkey-info-modulus'} # x509 subject public key modulus
|
||||
x509_subject_attribute_mapping = {'type': 'text', 'object_relation': 'subject'}
|
||||
x509_version_attribute_mapping = {'type': 'text', 'object_relation': 'version'}
|
||||
x509_vna_attribute_mapping = {'type': 'datetime', 'object_relation': 'validity-not-after'} # x509 validity not after
|
||||
x509_vnb_attribute_mapping = {'type': 'datetime', 'object_relation': 'validity-not-before'} # x509 validity not before
|
||||
|
||||
asn_mapping = {'number': as_number_attribute_mapping,
|
||||
'autonomous-system:number': as_number_attribute_mapping,
|
||||
'name': description_attribute_mapping,
|
||||
'autonomous-system:name': description_attribute_mapping,
|
||||
'ipv4-addr': asn_subnet_attribute_mapping,
|
||||
'ipv6-addr': asn_subnet_attribute_mapping,
|
||||
'ipv4-addr:value': asn_subnet_attribute_mapping,
|
||||
'ipv6-addr:value': asn_subnet_attribute_mapping}
|
||||
|
||||
attack_pattern_mapping = {'name': name_attribute_mapping,
|
||||
'description': summary_attribute_mapping}
|
||||
|
||||
attack_pattern_references_mapping = {'mitre-attack': references_attribute_mapping,
|
||||
'capec': id_attribute_mapping}
|
||||
|
||||
course_of_action_mapping = {'description': description_attribute_mapping,
|
||||
'name': name_attribute_mapping}
|
||||
|
||||
credential_mapping = {'credential': credential_attribute_mapping,
|
||||
'user-account:credential': credential_attribute_mapping,
|
||||
'user_id': user_id_mapping,
|
||||
'user-account:user_id': user_id_mapping}
|
||||
|
||||
domain_ip_mapping = {'domain-name': domain_attribute_mapping,
|
||||
'domain-name:value': domain_attribute_mapping,
|
||||
'ipv4-addr': ip_attribute_mapping,
|
||||
'ipv6-addr': ip_attribute_mapping,
|
||||
'ipv4-addr:value': ip_attribute_mapping,
|
||||
'ipv6-addr:value': ip_attribute_mapping,
|
||||
'domain-name:resolves_to_refs[*].value': ip_attribute_mapping,
|
||||
'network-traffic:dst_port': dst_port_attribute_mapping,
|
||||
'network-traffic:src_port': src_port_attribute_mapping}
|
||||
|
||||
email_mapping = {'date': email_date_attribute_mapping,
|
||||
'email-message:date': email_date_attribute_mapping,
|
||||
'email-message:to_refs[*].value': to_attribute_mapping,
|
||||
'email-message:cc_refs[*].value': cc_attribute_mapping,
|
||||
'subject': email_subject_attribute_mapping,
|
||||
'email-message:subject': email_subject_attribute_mapping,
|
||||
'X-Mailer': x_mailer_attribute_mapping,
|
||||
'email-message:additional_header_fields.x_mailer': x_mailer_attribute_mapping,
|
||||
'Reply-To': reply_to_attribute_mapping,
|
||||
'email-message:additional_header_fields.reply_to': reply_to_attribute_mapping,
|
||||
'email-message:from_ref.value': from_attribute_mapping,
|
||||
'email-addr:value': to_attribute_mapping}
|
||||
|
||||
email_references_mapping = {'attachment': email_attachment_attribute_mapping,
|
||||
'cc_refs': cc_attribute_mapping,
|
||||
'from_ref': from_attribute_mapping,
|
||||
'screenshot': screenshot_attribute_mapping,
|
||||
'to_refs': to_attribute_mapping}
|
||||
|
||||
file_mapping = {'artifact:mime_type': mime_type_attribute_mapping,
|
||||
'file:content_ref.mime_type': mime_type_attribute_mapping,
|
||||
'mime_type': mime_type_attribute_mapping,
|
||||
'file:mime_type': mime_type_attribute_mapping,
|
||||
'name': filename_attribute_mapping,
|
||||
'file:name': filename_attribute_mapping,
|
||||
'name_enc': encoding_attribute_mapping,
|
||||
'file:name_enc': encoding_attribute_mapping,
|
||||
'file:parent_directory_ref.path': path_attribute_mapping,
|
||||
'directory:path': path_attribute_mapping,
|
||||
'size': size_attribute_mapping,
|
||||
'file:size': size_attribute_mapping}
|
||||
|
||||
network_traffic_mapping = {'dst_port':dst_port_attribute_mapping,
|
||||
'src_port': src_port_attribute_mapping,
|
||||
'network-traffic:dst_port': dst_port_attribute_mapping,
|
||||
'network-traffic:src_port': src_port_attribute_mapping}
|
||||
|
||||
ip_port_mapping = {'value': domain_attribute_mapping,
|
||||
'domain-name:value': domain_attribute_mapping,
|
||||
'network-traffic:dst_ref.value': {'type': 'ip-dst', 'object_relation': 'ip-dst'},
|
||||
'network-traffic:src_ref.value': {'type': 'ip-src', 'object_relation': 'ip-src'}}
|
||||
ip_port_mapping.update(network_traffic_mapping)
|
||||
|
||||
ip_port_references_mapping = {'domain-name': domain_attribute_mapping,
|
||||
'ipv4-addr': network_traffic_ip,
|
||||
'ipv6-addr': network_traffic_ip}
|
||||
|
||||
network_socket_extension_mapping = {'address_family': address_family_attribute_mapping,
|
||||
"network-traffic:extensions.'socket-ext'.address_family": address_family_attribute_mapping,
|
||||
'protocol_family': domain_family_attribute_mapping,
|
||||
"network-traffic:extensions.'socket-ext'.protocol_family": domain_family_attribute_mapping,
|
||||
'is_blocking': state_attribute_mapping,
|
||||
"network-traffic:extensions.'socket-ext'.is_blocking": state_attribute_mapping,
|
||||
'is_listening': state_attribute_mapping,
|
||||
"network-traffic:extensions.'socket-ext'.is_listening": state_attribute_mapping}
|
||||
|
||||
network_traffic_references_mapping = {'domain-name': {'type': 'hostname', 'object_relation': 'hostname-{}'},
|
||||
'ipv4-addr': network_traffic_ip,
|
||||
'ipv6-addr': network_traffic_ip}
|
||||
|
||||
pe_mapping = {'pe_type': pe_type_mapping, 'number_of_sections': number_sections_mapping, 'imphash': imphash_mapping}
|
||||
|
||||
pe_section_mapping = {'name': section_name_mapping, 'size': size_attribute_mapping, 'entropy': entropy_mapping}
|
||||
|
||||
hash_types = ('MD5', 'SHA-1', 'SHA-256', 'SHA-224', 'SHA-384', 'SHA-512', 'ssdeep', 'tlsh')
|
||||
for hash_type in hash_types:
|
||||
misp_hash_type = hash_type.replace('-', '').lower()
|
||||
attribute = {'type': misp_hash_type, 'object_relation': misp_hash_type}
|
||||
file_mapping[hash_type] = attribute
|
||||
file_mapping.update({f"file:hashes.'{feature}'": attribute for feature in (hash_type, misp_hash_type)})
|
||||
file_mapping.update({f"file:hashes.{feature}": attribute for feature in (hash_type, misp_hash_type)})
|
||||
pe_section_mapping[hash_type] = attribute
|
||||
pe_section_mapping[misp_hash_type] = attribute
|
||||
|
||||
process_mapping = {'name': process_name_mapping,
|
||||
'process:name': process_name_mapping,
|
||||
'pid': pid_attribute_mapping,
|
||||
'process:pid': pid_attribute_mapping,
|
||||
'created': process_creation_time_mapping,
|
||||
'process:created': process_creation_time_mapping,
|
||||
'command_line': process_command_line_mapping,
|
||||
'process:command_line': process_command_line_mapping,
|
||||
'process:parent_ref.pid': {'type': 'text', 'object_relation': 'parent-pid'},
|
||||
'process:child_refs[*].pid': {'type': 'text', 'object_relation': 'child-pid'},
|
||||
'process:binary_ref.name': process_image_mapping}
|
||||
|
||||
child_process_reference_mapping = {'pid': {'type': 'text', 'object_relation': 'child-pid'}}
|
||||
|
||||
parent_process_reference_mapping = {'command_line': {'type': 'text', 'object_relation': 'parent-command-line'},
|
||||
'pid': {'type': 'text', 'object_relation': 'parent-pid'},
|
||||
'process-name': {'type': 'text', 'object_relation': 'parent-process-name'}}
|
||||
|
||||
regkey_mapping = {'data': data_attribute_mapping,
|
||||
'windows-registry-key:values.data': data_attribute_mapping,
|
||||
'data_type': data_type_attribute_mapping,
|
||||
'windows-registry-key:values.data_type': data_type_attribute_mapping,
|
||||
'modified': modified_attribute_mapping,
|
||||
'windows-registry-key:modified': modified_attribute_mapping,
|
||||
'name': regkey_name_attribute_mapping,
|
||||
'windows-registry-key:values.name': regkey_name_attribute_mapping,
|
||||
'key': key_attribute_mapping,
|
||||
'windows-registry-key:key': key_attribute_mapping,
|
||||
'windows-registry-key:value': {'type': 'text', 'object_relation': 'hive'}
|
||||
}
|
||||
|
||||
url_mapping = {'url': url_attribute_mapping,
|
||||
'url:value': url_attribute_mapping,
|
||||
'domain-name': domain_attribute_mapping,
|
||||
'domain-name:value': domain_attribute_mapping,
|
||||
'network-traffic': url_port_attribute_mapping,
|
||||
'network-traffic:dst_port': url_port_attribute_mapping,
|
||||
'ipv4-addr:value': ip_attribute_mapping,
|
||||
'ipv6-addr:value': ip_attribute_mapping
|
||||
}
|
||||
|
||||
user_account_mapping = {'account_created': {'type': 'datetime', 'object_relation': 'created'},
|
||||
'account_expires': {'type': 'datetime', 'object_relation': 'expires'},
|
||||
'account_first_login': {'type': 'datetime', 'object_relation': 'first_login'},
|
||||
'account_last_login': {'type': 'datetime', 'object_relation': 'last_login'},
|
||||
'account_login': user_id_mapping,
|
||||
'account_type': {'type': 'text', 'object_relation': 'account-type'},
|
||||
'can_escalate_privs': {'type': 'boolean', 'object_relation': 'can_escalate_privs'},
|
||||
'credential': credential_attribute_mapping,
|
||||
'credential_last_changed': {'type': 'datetime', 'object_relation': 'password_last_changed'},
|
||||
'display_name': {'type': 'text', 'object_relation': 'display-name'},
|
||||
'gid': {'type': 'text', 'object_relation': 'group-id'},
|
||||
'home_dir': {'type': 'text', 'object_relation': 'home_dir'},
|
||||
'is_disabled': {'type': 'boolean', 'object_relation': 'disabled'},
|
||||
'is_privileged': {'type': 'boolean', 'object_relation': 'privileged'},
|
||||
'is_service_account': {'type': 'boolean', 'object_relation': 'is_service_account'},
|
||||
'shell': {'type': 'text', 'object_relation': 'shell'},
|
||||
'user_id': {'type': 'text', 'object_relation': 'user-id'}}
|
||||
|
||||
vulnerability_mapping = {'name': id_attribute_mapping,
|
||||
'description': summary_attribute_mapping}
|
||||
|
||||
x509_mapping = {'issuer': issuer_attribute_mapping,
|
||||
'x509-certificate:issuer': issuer_attribute_mapping,
|
||||
'serial_number': serial_number_attribute_mapping,
|
||||
'x509-certificate:serial_number': serial_number_attribute_mapping,
|
||||
'subject': x509_subject_attribute_mapping,
|
||||
'x509-certificate:subject': x509_subject_attribute_mapping,
|
||||
'subject_public_key_algorithm': x509_spka_attribute_mapping,
|
||||
'x509-certificate:subject_public_key_algorithm': x509_spka_attribute_mapping,
|
||||
'subject_public_key_exponent': x509_spke_attribute_mapping,
|
||||
'x509-certificate:subject_public_key_exponent': x509_spke_attribute_mapping,
|
||||
'subject_public_key_modulus': x509_spkm_attribute_mapping,
|
||||
'x509-certificate:subject_public_key_modulus': x509_spkm_attribute_mapping,
|
||||
'validity_not_before': x509_vnb_attribute_mapping,
|
||||
'x509-certificate:validity_not_before': x509_vnb_attribute_mapping,
|
||||
'validity_not_after': x509_vna_attribute_mapping,
|
||||
'x509-certificate:validity_not_after': x509_vna_attribute_mapping,
|
||||
'version': x509_version_attribute_mapping,
|
||||
'x509-certificate:version': x509_version_attribute_mapping,
|
||||
'SHA-1': x509_sha1_attribute_mapping,
|
||||
"x509-certificate:hashes.'sha1'": x509_sha1_attribute_mapping,
|
||||
'SHA-256': x509_sha256_attribute_mapping,
|
||||
"x509-certificate:hashes.'sha256'": x509_sha256_attribute_mapping,
|
||||
'MD5': x509_md5_attribute_mapping,
|
||||
"x509-certificate:hashes.'md5'": x509_md5_attribute_mapping,
|
||||
}
|
||||
|
||||
attachment_types = ('file:content_ref.name', 'file:content_ref.payload_bin',
|
||||
'artifact:x_misp_text_name', 'artifact:payload_bin',
|
||||
"file:hashes.'MD5'", "file:content_ref.hashes.'MD5'",
|
||||
'file:name')
|
||||
|
||||
connection_protocols = {"IP": "3", "ICMP": "3", "ARP": "3",
|
||||
"TCP": "4", "UDP": "4",
|
||||
"HTTP": "7", "HTTPS": "7", "FTP": "7"}
|
||||
File diff suppressed because one or more lines are too long
|
|
@ -4,8 +4,8 @@ from . import check_input_attribute, standard_error_message
|
|||
from pymisp import MISPAttribute, MISPEvent, MISPObject
|
||||
|
||||
misperrors = {'error': 'Error'}
|
||||
mispattributes = {'input': ['domain', 'hostname'], 'format': 'misp_standard'}
|
||||
moduleinfo = {'version': '0.1', 'author': 'Christian Studer',
|
||||
mispattributes = {'input': ['domain', 'hostname', 'email'], 'format': 'misp_standard'}
|
||||
moduleinfo = {'version': '0.2', 'author': 'Christian Studer',
|
||||
'description': 'On demand query API for APIVoid.',
|
||||
'module-type': ['expansion', 'hover']}
|
||||
moduleconfig = ['apikey']
|
||||
|
|
@ -43,6 +43,31 @@ class APIVoidParser():
|
|||
ssl = requests.get(f'{self.url.format("sslinfo", apikey)}host={self.attribute.value}').json()
|
||||
self._parse_ssl_certificate(ssl['data']['certificate'])
|
||||
|
||||
def handle_email(self, apikey):
|
||||
feature = 'emailverify'
|
||||
if requests.get(f'{self.url.format(feature, apikey)}stats').json()['credits_remained'] < 0.06:
|
||||
self.result = {'error': 'You do not have enough APIVoid credits to proceed your request.'}
|
||||
return
|
||||
emaillookup = requests.get(f'{self.url.format(feature, apikey)}email={self.attribute.value}').json()
|
||||
email_verification = MISPObject('apivoid-email-verification')
|
||||
boolean_attributes = ['valid_format', 'suspicious_username', 'suspicious_email', 'dirty_words_username',
|
||||
'suspicious_email', 'valid_tld', 'disposable', 'has_a_records', 'has_mx_records',
|
||||
'has_spf_records', 'is_spoofable', 'dmarc_configured', 'dmarc_enforced', 'free_email',
|
||||
'russian_free_email', 'china_free_email', 'suspicious_domain', 'dirty_words_domain',
|
||||
'domain_popular', 'risky_tld', 'police_domain', 'government_domain', 'educational_domain',
|
||||
'should_block']
|
||||
for boolean_attribute in boolean_attributes:
|
||||
email_verification.add_attribute(boolean_attribute,
|
||||
**{'type': 'boolean', 'value': emaillookup['data'][boolean_attribute]})
|
||||
email_verification.add_attribute('email', **{'type': 'email', 'value': emaillookup['data']['email']})
|
||||
email_verification.add_attribute('username', **{'type': 'text', 'value': emaillookup['data']['username']})
|
||||
email_verification.add_attribute('role_address',
|
||||
**{'type': 'boolean', 'value': emaillookup['data']['role_address']})
|
||||
email_verification.add_attribute('domain', **{'type': 'domain', 'value': emaillookup['data']['domain']})
|
||||
email_verification.add_attribute('score', **{'type': 'float', 'value': emaillookup['data']['score']})
|
||||
email_verification.add_reference(self.attribute['uuid'], 'related-to')
|
||||
self.misp_event.add_object(email_verification)
|
||||
|
||||
def _handle_dns_record(self, item, record_type, relationship):
|
||||
dns_record = MISPObject('dns-record')
|
||||
dns_record.add_attribute('queried-domain', type='domain', value=item['host'])
|
||||
|
|
@ -82,7 +107,10 @@ def handler(q=False):
|
|||
return {'error': 'Unsupported attribute type.'}
|
||||
apikey = request['config']['apikey']
|
||||
apivoid_parser = APIVoidParser(attribute)
|
||||
apivoid_parser.parse_domain(apikey)
|
||||
if attribute['type'] in ['domain', 'hostname']:
|
||||
apivoid_parser.parse_domain(apikey)
|
||||
else:
|
||||
apivoid_parser.handle_email(apikey)
|
||||
return apivoid_parser.get_results()
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,129 @@
|
|||
import json
|
||||
import requests
|
||||
from . import check_input_attribute, standard_error_message
|
||||
from pymisp import MISPEvent, MISPObject
|
||||
|
||||
misperrors = {'error': 'Error'}
|
||||
mispattributes = {'input': ['ip-src', 'ip-src|port', 'ip-dst', 'ip-dst|port'], 'format': 'misp_standard'}
|
||||
moduleinfo = {'version': '1', 'author': 'Jeroen Pinoy',
|
||||
'description': "An expansion module to enrich an ip with geolocation information from an mmdb server "
|
||||
"such as ip.circl.lu.",
|
||||
'module-type': ['expansion', 'hover']}
|
||||
moduleconfig = ["custom_API", "db_source_filter"]
|
||||
mmdblookup_url = 'https://ip.circl.lu/'
|
||||
|
||||
|
||||
class MmdbLookupParser():
|
||||
def __init__(self, attribute, mmdblookupresult, api_url):
|
||||
self.attribute = attribute
|
||||
self.mmdblookupresult = mmdblookupresult
|
||||
self.api_url = api_url
|
||||
self.misp_event = MISPEvent()
|
||||
self.misp_event.add_attribute(**attribute)
|
||||
|
||||
def get_result(self):
|
||||
event = json.loads(self.misp_event.to_json())
|
||||
results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])}
|
||||
return {'results': results}
|
||||
|
||||
def parse_mmdblookup_information(self):
|
||||
# There is a chance some db's have a hit while others don't so we have to check if entry is empty each time
|
||||
for result_entry in self.mmdblookupresult:
|
||||
if result_entry['country_info']:
|
||||
mmdblookup_object = MISPObject('geolocation')
|
||||
mmdblookup_object.add_attribute('country',
|
||||
**{'type': 'text', 'value': result_entry['country_info']['Country']})
|
||||
mmdblookup_object.add_attribute('countrycode',
|
||||
**{'type': 'text', 'value': result_entry['country']['iso_code']})
|
||||
mmdblookup_object.add_attribute('latitude',
|
||||
**{'type': 'float',
|
||||
'value': result_entry['country_info']['Latitude (average)']})
|
||||
mmdblookup_object.add_attribute('longitude',
|
||||
**{'type': 'float',
|
||||
'value': result_entry['country_info']['Longitude (average)']})
|
||||
mmdblookup_object.add_attribute('text',
|
||||
**{'type': 'text',
|
||||
'value': 'db_source: {}. build_db: {}. Latitude and longitude are country average.'.format(
|
||||
result_entry['meta']['db_source'],
|
||||
result_entry['meta']['build_db'])})
|
||||
mmdblookup_object.add_reference(self.attribute['uuid'], 'related-to')
|
||||
self.misp_event.add_object(mmdblookup_object)
|
||||
if 'AutonomousSystemNumber' in result_entry['country']:
|
||||
mmdblookup_object_asn = MISPObject('asn')
|
||||
mmdblookup_object_asn.add_attribute('asn',
|
||||
**{'type': 'text',
|
||||
'value': result_entry['country'][
|
||||
'AutonomousSystemNumber']})
|
||||
mmdblookup_object_asn.add_attribute('description',
|
||||
**{'type': 'text',
|
||||
'value': 'ASNOrganization: {}. db_source: {}. build_db: {}.'.format(
|
||||
result_entry['country'][
|
||||
'AutonomousSystemOrganization'],
|
||||
result_entry['meta']['db_source'],
|
||||
result_entry['meta']['build_db'])})
|
||||
mmdblookup_object_asn.add_reference(self.attribute['uuid'], 'related-to')
|
||||
self.misp_event.add_object(mmdblookup_object_asn)
|
||||
|
||||
|
||||
def check_url(url):
|
||||
return "{}/".format(url) if not url.endswith('/') else url
|
||||
|
||||
|
||||
def handler(q=False):
|
||||
if q is False:
|
||||
return False
|
||||
request = json.loads(q)
|
||||
if not request.get('attribute') or not check_input_attribute(request['attribute']):
|
||||
return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'}
|
||||
attribute = request['attribute']
|
||||
if attribute.get('type') == 'ip-src':
|
||||
toquery = attribute['value']
|
||||
elif attribute.get('type') == 'ip-src|port':
|
||||
toquery = attribute['value'].split('|')[0]
|
||||
elif attribute.get('type') == 'ip-dst':
|
||||
toquery = attribute['value']
|
||||
elif attribute.get('type') == 'ip-dst|port':
|
||||
toquery = attribute['value'].split('|')[0]
|
||||
else:
|
||||
misperrors['error'] = 'There is no attribute of type ip-src or ip-dst provided as input'
|
||||
return misperrors
|
||||
api_url = check_url(request['config']['custom_API']) if 'config' in request and request['config'].get(
|
||||
'custom_API') else mmdblookup_url
|
||||
r = requests.get("{}/geolookup/{}".format(api_url, toquery))
|
||||
if r.status_code == 200:
|
||||
mmdblookupresult = r.json()
|
||||
if not mmdblookupresult or len(mmdblookupresult) == 0:
|
||||
misperrors['error'] = 'Empty result returned by server'
|
||||
return misperrors
|
||||
if 'config' in request and request['config'].get('db_source_filter'):
|
||||
db_source_filter = request['config'].get('db_source_filter')
|
||||
mmdblookupresult = [entry for entry in mmdblookupresult if entry['meta']['db_source'] == db_source_filter]
|
||||
if not mmdblookupresult or len(mmdblookupresult) == 0:
|
||||
misperrors['error'] = 'There was no result with the selected db_source'
|
||||
return misperrors
|
||||
# Server might return one or multiple entries which could all be empty, we check if there is at least one
|
||||
# non-empty result below
|
||||
empty_result = True
|
||||
for lookup_result_entry in mmdblookupresult:
|
||||
if lookup_result_entry['country_info']:
|
||||
empty_result = False
|
||||
break
|
||||
if empty_result:
|
||||
misperrors['error'] = 'Empty result returned by server'
|
||||
return misperrors
|
||||
else:
|
||||
misperrors['error'] = 'API not accessible - http status code {} was returned'.format(r.status_code)
|
||||
return misperrors
|
||||
parser = MmdbLookupParser(attribute, mmdblookupresult, api_url)
|
||||
parser.parse_mmdblookup_information()
|
||||
result = parser.get_result()
|
||||
return result
|
||||
|
||||
|
||||
def introspection():
|
||||
return mispattributes
|
||||
|
||||
|
||||
def version():
|
||||
moduleinfo['config'] = moduleconfig
|
||||
return moduleinfo
|
||||
|
|
@ -1,2 +1,3 @@
|
|||
__all__ = ['cef_export', 'mass_eql_export', 'liteexport', 'goamlexport', 'threat_connect_export', 'pdfexport',
|
||||
'threatStream_misp_export', 'osqueryexport', 'nexthinkexport', 'vt_graph', 'defender_endpoint_export']
|
||||
'threatStream_misp_export', 'osqueryexport', 'nexthinkexport', 'vt_graph', 'defender_endpoint_export',
|
||||
'virustotal_collections']
|
||||
|
|
|
|||
|
|
@ -0,0 +1,134 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
# Copyright 2022 Google Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Creates a VT Collection with indicators present in a given event."""
|
||||
|
||||
import base64
|
||||
import json
|
||||
import requests
|
||||
|
||||
misperrors = {
|
||||
'error': 'Error'
|
||||
}
|
||||
|
||||
mispattributes = {
|
||||
'input': [
|
||||
'hostname',
|
||||
'domain',
|
||||
'ip-src',
|
||||
'ip-dst',
|
||||
'md5',
|
||||
'sha1',
|
||||
'sha256',
|
||||
'url'
|
||||
],
|
||||
'format': 'misp_standard',
|
||||
'responseType': 'application/txt',
|
||||
'outputFileExtension': 'txt',
|
||||
}
|
||||
|
||||
moduleinfo = {
|
||||
'version': '1.0',
|
||||
'author': 'VirusTotal',
|
||||
'description': 'Creates a VT Collection from an event iocs.',
|
||||
'module-type': ['export']
|
||||
}
|
||||
|
||||
moduleconfig = [
|
||||
'vt_api_key',
|
||||
'proxy_host',
|
||||
'proxy_port',
|
||||
'proxy_username',
|
||||
'proxy_password'
|
||||
]
|
||||
|
||||
|
||||
class VTError(Exception):
|
||||
"Exception class to map vt api response errors."
|
||||
pass
|
||||
|
||||
|
||||
def create_collection(api_key, event_data):
|
||||
headers = {
|
||||
'x-apikey': api_key,
|
||||
'content-type': 'application/json',
|
||||
'x-tool': 'MISPModuleVirusTotalCollectionExport',
|
||||
}
|
||||
|
||||
response = requests.post('https://www.virustotal.com/api/v3/integrations/misp/collections',
|
||||
headers=headers,
|
||||
json=event_data)
|
||||
|
||||
uuid = event_data['Event']['uuid']
|
||||
response_data = response.json()
|
||||
|
||||
if response.status_code == 200:
|
||||
link = response_data['data']['links']['self']
|
||||
return f'{uuid}: {link}'
|
||||
|
||||
error = response_data['error']['message']
|
||||
if response.status_code == 400:
|
||||
return f'{uuid}: {error}'
|
||||
else:
|
||||
misperrors['error'] = error
|
||||
raise VTError(error)
|
||||
|
||||
|
||||
def normalize_misp_data(data):
|
||||
normalized_data = {'Event': data.pop('Event', {})}
|
||||
for attr_key in data:
|
||||
if isinstance(data[attr_key], list) or isinstance(data[attr_key], dict):
|
||||
if attr_key == 'EventTag':
|
||||
normalized_data['Event']['Tag'] = [tag['Tag'] for tag in data[attr_key]]
|
||||
else:
|
||||
normalized_data['Event'][attr_key] = data[attr_key]
|
||||
|
||||
return normalized_data
|
||||
|
||||
|
||||
def handler(q=False):
|
||||
request = json.loads(q)
|
||||
|
||||
if not request.get('config') or not request['config'].get('vt_api_key'):
|
||||
misperrors['error'] = 'A VirusTotal api key is required for this module.'
|
||||
return misperrors
|
||||
|
||||
config = request['config']
|
||||
data = request['data']
|
||||
responses = []
|
||||
|
||||
try:
|
||||
for event_data in data:
|
||||
normalized_event = normalize_misp_data(event_data)
|
||||
responses.append(create_collection(config.get('vt_api_key'),
|
||||
normalized_event))
|
||||
|
||||
output = '\n'.join(responses)
|
||||
return {
|
||||
"response": [],
|
||||
"data": str(base64.b64encode(bytes(output, 'utf-8')), 'utf-8'),
|
||||
}
|
||||
except VTError:
|
||||
return misperrors
|
||||
|
||||
|
||||
|
||||
def introspection():
|
||||
return mispattributes
|
||||
|
||||
|
||||
def version():
|
||||
moduleinfo['config'] = moduleconfig
|
||||
return moduleinfo
|
||||
|
|
@ -15,4 +15,5 @@ __all__ = [
|
|||
'csvimport',
|
||||
'cof2misp',
|
||||
'joe_import',
|
||||
'taxii21'
|
||||
]
|
||||
|
|
|
|||
|
|
@ -0,0 +1,373 @@
|
|||
"""
|
||||
Import content from a TAXII 2.1 server.
|
||||
"""
|
||||
import collections
|
||||
import itertools
|
||||
import json
|
||||
import misp_modules.lib.stix2misp
|
||||
from pathlib import Path
|
||||
import re
|
||||
import stix2.v20
|
||||
import taxii2client
|
||||
import taxii2client.exceptions
|
||||
import requests
|
||||
|
||||
|
||||
class ConfigError(Exception):
|
||||
"""
|
||||
Represents an error in the config settings for one invocation of this
|
||||
module.
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
misperrors = {'error': 'Error'}
|
||||
|
||||
moduleinfo = {'version': '0.1', 'author': 'Abc',
|
||||
'description': 'Import content from a TAXII 2.1 server',
|
||||
'module-type': ['import']}
|
||||
|
||||
mispattributes = {
|
||||
'inputSource': [],
|
||||
'output': ['MISP objects'],
|
||||
'format': 'misp_standard',
|
||||
}
|
||||
|
||||
|
||||
userConfig = {
|
||||
"url": {
|
||||
"type": "String",
|
||||
"message": "A TAXII 2.1 collection URL",
|
||||
},
|
||||
"added_after": {
|
||||
"type": "String",
|
||||
"message": "Lower bound on time the object was uploaded to the TAXII server"
|
||||
},
|
||||
"stix_id": {
|
||||
"type": "String",
|
||||
"message": "STIX ID(s) of objects"
|
||||
},
|
||||
"spec_version": { # TAXII 2.1 specific
|
||||
"type": "String",
|
||||
"message": "STIX version(s) of objects"
|
||||
},
|
||||
"type": {
|
||||
"type": "String",
|
||||
"message": "STIX type(s) of objects"
|
||||
},
|
||||
"version": {
|
||||
"type": "String",
|
||||
"message": 'Version timestamp(s), or "first"/"last"/"all"'
|
||||
},
|
||||
# Should we give some user control over this? It will not be allowed to
|
||||
# exceed the admin setting.
|
||||
"STIX object limit": {
|
||||
"type": "Integer",
|
||||
"message": "Maximum number of STIX objects to process"
|
||||
},
|
||||
"username": {
|
||||
"type": "String",
|
||||
"message": "Username for TAXII server authentication, if necessary"
|
||||
},
|
||||
"password": {
|
||||
"type": "String",
|
||||
"message": "Password for TAXII server authentication, if necessary"
|
||||
}
|
||||
}
|
||||
|
||||
# Paging will be handled transparently by this module, so user-defined
|
||||
# paging-related filtering parameters will not be supported.
|
||||
|
||||
|
||||
# This module will not process more than this number of STIX objects in total
|
||||
# from a TAXII server in one module invocation (across all pages), to limit
|
||||
# resource consumption.
|
||||
moduleconfig = [
|
||||
"stix_object_limit"
|
||||
]
|
||||
|
||||
|
||||
# In case there is neither an admin nor user setting given.
|
||||
_DEFAULT_STIX_OBJECT_LIMIT = 1000
|
||||
|
||||
|
||||
# Page size to use when paging TAXII results. Trades off the amount of
|
||||
# hammering on TAXII servers and overhead of repeated requests, with the
|
||||
# resource consumption of a single page. (Should be an admin setting too?)
|
||||
_PAGE_SIZE = 100
|
||||
|
||||
|
||||
_synonymsToTagNames_path = Path(__file__).parent / "../../lib/synonymsToTagNames.json"
|
||||
|
||||
|
||||
# Collects module config information necessary to perform the TAXII query.
|
||||
Config = collections.namedtuple("Config", [
|
||||
"url",
|
||||
"added_after",
|
||||
"id",
|
||||
"spec_version",
|
||||
"type",
|
||||
"version",
|
||||
"stix_object_limit",
|
||||
"username",
|
||||
"password"
|
||||
])
|
||||
|
||||
|
||||
def _pymisp_to_json_serializable(obj):
|
||||
"""
|
||||
Work around a possible bug with PyMISP's
|
||||
AbstractMisp.to_dict(json_format=True) method, which doesn't always produce
|
||||
a JSON-serializable value (i.e. a value which is serializable with the
|
||||
default JSON encoder).
|
||||
|
||||
:param obj: A PyMISP object
|
||||
:return: A JSON-serializable version of the object
|
||||
"""
|
||||
|
||||
# The workaround creates a JSON string and then parses it back to a
|
||||
# JSON-serializable value.
|
||||
json_ = obj.to_json()
|
||||
json_serializable = json.loads(json_)
|
||||
|
||||
return json_serializable
|
||||
|
||||
|
||||
def _normalize_multi_values(value):
|
||||
"""
|
||||
Some TAXII filters may contain multiple values separated by commas,
|
||||
without spaces around the commas. Maybe give MISP users a little more
|
||||
flexibility? This function normalizes a possible multi-valued value
|
||||
(e.g. multiple values delimited by commas or spaces, all in the same
|
||||
string) to TAXII-required format.
|
||||
|
||||
:param value: A MISP config value
|
||||
:return: A normalized value
|
||||
"""
|
||||
|
||||
if "," in value:
|
||||
value = re.sub(r"\s*,\s*", ",", value)
|
||||
else:
|
||||
# Assume space delimiting; replace spaces with commas.
|
||||
# I don't think we need to worry about spaces embedded in values.
|
||||
value = re.sub(r"\s+", ",", value)
|
||||
|
||||
value = value.strip(",")
|
||||
|
||||
return value
|
||||
|
||||
|
||||
def _get_config(config):
|
||||
"""
|
||||
Combine user, admin, and default config settings to produce a config
|
||||
object with all settings together.
|
||||
|
||||
:param config: The misp-modules request's "config" value.
|
||||
:return: A Config object
|
||||
:raises ConfigError: if any config errors are detected
|
||||
"""
|
||||
|
||||
# Strip whitespace from all config settings... except for password?
|
||||
for key, val in config.items():
|
||||
if isinstance(val, str) and key != "password":
|
||||
config[key] = val.strip()
|
||||
|
||||
url = config.get("url")
|
||||
added_after = config.get("added_after")
|
||||
id_ = config.get("stix_id")
|
||||
spec_version = config.get("spec_version")
|
||||
type_ = config.get("type")
|
||||
version_ = config.get("version")
|
||||
username = config.get("username")
|
||||
password = config.get("password")
|
||||
admin_stix_object_limit = config.get("stix_object_limit")
|
||||
user_stix_object_limit = config.get("STIX object limit")
|
||||
|
||||
if admin_stix_object_limit:
|
||||
admin_stix_object_limit = int(admin_stix_object_limit)
|
||||
else:
|
||||
admin_stix_object_limit = _DEFAULT_STIX_OBJECT_LIMIT
|
||||
|
||||
if user_stix_object_limit:
|
||||
user_stix_object_limit = int(user_stix_object_limit)
|
||||
stix_object_limit = min(user_stix_object_limit, admin_stix_object_limit)
|
||||
else:
|
||||
stix_object_limit = admin_stix_object_limit
|
||||
|
||||
# How much of this should we sanity-check here before passing it off to the
|
||||
# TAXII client (and thence, to the TAXII server)?
|
||||
|
||||
if not url:
|
||||
raise ConfigError("A TAXII 2.1 collection URL is required.")
|
||||
|
||||
if admin_stix_object_limit < 1:
|
||||
raise ConfigError(
|
||||
"Invalid admin object limit: must be positive: "
|
||||
+ str(admin_stix_object_limit)
|
||||
)
|
||||
|
||||
if stix_object_limit < 1:
|
||||
raise ConfigError(
|
||||
"Invalid object limit: must be positive: "
|
||||
+ str(stix_object_limit)
|
||||
)
|
||||
|
||||
if id_:
|
||||
id_ = _normalize_multi_values(id_)
|
||||
if spec_version:
|
||||
spec_version = _normalize_multi_values(spec_version)
|
||||
if type_:
|
||||
type_ = _normalize_multi_values(type_)
|
||||
if version_:
|
||||
version_ = _normalize_multi_values(version_)
|
||||
|
||||
# STIX->MISP converter currently only supports STIX 2.0, so let's force
|
||||
# spec_version="2.0".
|
||||
if not spec_version:
|
||||
spec_version = "2.0"
|
||||
elif spec_version != "2.0":
|
||||
raise ConfigError('Only spec_version="2.0" is supported for now.')
|
||||
|
||||
if (username and not password) or (not username and password):
|
||||
raise ConfigError(
|
||||
'Both or neither of "username" and "password" are required.'
|
||||
)
|
||||
|
||||
config_obj = Config(
|
||||
url, added_after, id_, spec_version, type_, version_, stix_object_limit,
|
||||
username, password
|
||||
)
|
||||
|
||||
return config_obj
|
||||
|
||||
|
||||
def _query_taxii(config):
|
||||
"""
|
||||
Query the TAXII server according to the given config, convert the STIX
|
||||
results to MISP, and return a standard misp-modules response.
|
||||
|
||||
:param config: Module config information as a Config object
|
||||
:return: A dict containing a misp-modules response
|
||||
"""
|
||||
|
||||
collection = taxii2client.Collection(
|
||||
config.url, user=config.username, password=config.password
|
||||
)
|
||||
|
||||
# No point in asking for more than our overall limit.
|
||||
page_size = min(_PAGE_SIZE, config.stix_object_limit)
|
||||
|
||||
kwargs = {
|
||||
"per_request": page_size
|
||||
}
|
||||
|
||||
if config.spec_version:
|
||||
kwargs["spec_version"] = config.spec_version
|
||||
if config.version:
|
||||
kwargs["version"] = config.version
|
||||
if config.id:
|
||||
kwargs["id"] = config.id
|
||||
if config.type:
|
||||
kwargs["type"] = config.type
|
||||
if config.added_after:
|
||||
kwargs["added_after"] = config.added_after
|
||||
|
||||
pages = taxii2client.as_pages(
|
||||
collection.get_objects,
|
||||
**kwargs
|
||||
)
|
||||
|
||||
# Chain all the objects from all pages together...
|
||||
all_stix_objects = itertools.chain.from_iterable(
|
||||
taxii_envelope.get("objects", [])
|
||||
for taxii_envelope in pages
|
||||
)
|
||||
|
||||
# And only take the first N objects from that.
|
||||
limited_stix_objects = itertools.islice(
|
||||
all_stix_objects, 0, config.stix_object_limit
|
||||
)
|
||||
|
||||
# Collect into a list. This is... unfortunate, but I don't think the
|
||||
# converter will work incrementally (will it?). It expects all objects to
|
||||
# be given at once.
|
||||
#
|
||||
# It may also be desirable to have all objects available at once so that
|
||||
# cross-references can be made where possible, but it results in increased
|
||||
# memory usage.
|
||||
stix_objects = list(limited_stix_objects)
|
||||
|
||||
# The STIX 2.0 converter wants a 2.0 bundle. (Hope the TAXII server isn't
|
||||
# returning 2.1 objects!)
|
||||
bundle20 = stix2.v20.Bundle(stix_objects, allow_custom=True)
|
||||
|
||||
converter = misp_modules.lib.stix2misp.ExternalStixParser()
|
||||
converter.handler(
|
||||
bundle20, None, [0, "event", str(_synonymsToTagNames_path)]
|
||||
)
|
||||
|
||||
attributes = [
|
||||
_pymisp_to_json_serializable(attr)
|
||||
for attr in converter.misp_event.attributes
|
||||
]
|
||||
|
||||
objects = [
|
||||
_pymisp_to_json_serializable(obj)
|
||||
for obj in converter.misp_event.objects
|
||||
]
|
||||
|
||||
tags = [
|
||||
_pymisp_to_json_serializable(tag)
|
||||
for tag in converter.misp_event.tags
|
||||
]
|
||||
|
||||
result = {
|
||||
"results": {
|
||||
"Attribute": attributes,
|
||||
"Object": objects,
|
||||
"Tag": tags
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def handler(q=False):
|
||||
if q is False:
|
||||
return False
|
||||
request = json.loads(q)
|
||||
|
||||
result = None
|
||||
config = None
|
||||
|
||||
try:
|
||||
config = _get_config(request["config"])
|
||||
except ConfigError as e:
|
||||
result = misperrors
|
||||
result["error"] = e.args[0]
|
||||
|
||||
if not result:
|
||||
try:
|
||||
result = _query_taxii(config)
|
||||
except taxii2client.exceptions.TAXIIServiceException as e:
|
||||
result = misperrors
|
||||
result["error"] = str(e)
|
||||
except requests.HTTPError as e:
|
||||
# Let's give a better error message for auth issues.
|
||||
if e.response.status_code in (401, 403):
|
||||
result = misperrors
|
||||
result["error"] = "Access was denied."
|
||||
else:
|
||||
raise
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def introspection():
|
||||
mispattributes["userConfig"] = userConfig
|
||||
return mispattributes
|
||||
|
||||
|
||||
def version():
|
||||
moduleinfo['config'] = moduleconfig
|
||||
return moduleinfo
|
||||
|
|
@ -509,7 +509,7 @@ class TestExpansions(unittest.TestCase):
|
|||
query = {"module": "sourcecache", "link": input_value}
|
||||
response = self.misp_modules_post(query)
|
||||
self.assertEqual(self.get_values(response), input_value)
|
||||
self.assertTrue(self.get_data(response).startswith('PCFET0NUWVBFIEhUTUw+CjwhLS0KCUFyY2FuYSBieSBIVE1MN'))
|
||||
self.assertTrue(self.get_data(response))
|
||||
|
||||
def test_stix2_pattern_validator(self):
|
||||
query = {"module": "stix2_pattern_syntax_validator", "stix2-pattern": "[ipv4-addr:value = '8.8.8.8']"}
|
||||
|
|
|
|||
Loading…
Reference in New Issue