Merge branch 'main' into main

pull/554/head
Alexandre Dulaunoy 2022-02-15 15:16:13 +01:00 committed by GitHub
commit beb463bdab
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 3366 additions and 99 deletions

View File

@ -46,5 +46,7 @@ jobs:
# Run server in background
misp-modules -l 127.0.0.1 -s &
sleep 5
# Check if modules are running
curl -sS localhost:6666/modules
# Run tests
pytest tests

4
.gitmodules vendored Normal file
View File

@ -0,0 +1,4 @@
[submodule "misp_modules/lib/misp-objects"]
path = misp_modules/lib/misp-objects
url = https://github.com/MISP/misp-objects.git
branch = main

24
Pipfile
View File

@ -17,9 +17,9 @@ passivetotal = "*"
pypdns = "*"
pypssl = "*"
pyeupi = "*"
pymisp = { extras = ["fileobjects,openioc,pdfexport,email"], version = "*" }
pyonyphe = { editable = true, git = "https://github.com/sebdraven/pyonyphe" }
pydnstrails = { editable = true, git = "https://github.com/sebdraven/pydnstrails" }
pymisp = { extras = ["fileobjects,openioc,pdfexport,email,url"], version = "*" }
pyonyphe = { git = "https://github.com/sebdraven/pyonyphe" }
pydnstrails = { git = "https://github.com/sebdraven/pydnstrails" }
pytesseract = "*"
pygeoip = "*"
beautifulsoup4 = "*"
@ -31,20 +31,20 @@ maclookup = "*"
vulners = "*"
blockchain = "*"
reportlab = "*"
pyintel471 = { editable = true, git = "https://github.com/MISP/PyIntel471.git" }
pyintel471 = { git = "https://github.com/MISP/PyIntel471.git" }
shodan = "*"
Pillow = ">=8.2.0"
Wand = "*"
SPARQLWrapper = "*"
domaintools_api = "*"
misp-modules = { editable = true, path = "." }
pybgpranking = { editable = true, git = "https://github.com/D4-project/BGP-Ranking.git/", subdirectory = "client" }
pyipasnhistory = { editable = true, git = "https://github.com/D4-project/IPASN-History.git/", subdirectory = "client" }
misp-modules = { path = "." }
pybgpranking = { git = "https://github.com/D4-project/BGP-Ranking.git/", subdirectory = "client", ref = "68de39f6c5196f796055c1ac34504054d688aa59" }
pyipasnhistory = { git = "https://github.com/D4-project/IPASN-History.git/", subdirectory = "client", ref = "a2853c39265cecdd0c0d16850bd34621c0551b87" }
backscatter = "*"
pyzbar = "*"
opencv-python = "*"
np = "*"
ODTReader = { editable = true, git = "https://github.com/cartertemm/ODTReader.git/" }
ODTReader = { git = "https://github.com/cartertemm/ODTReader.git/" }
python-pptx = "*"
python-docx = "*"
ezodf = "*"
@ -59,7 +59,7 @@ geoip2 = "*"
apiosintDS = "*"
assemblyline_client = "*"
vt-graph-api = "*"
trustar = { editable = true, git = "https://github.com/SteveClement/trustar-python.git" }
trustar = { git = "https://github.com/SteveClement/trustar-python.git" }
markdownify = "==0.5.3"
socialscan = "*"
dnsdb2 = "*"
@ -67,6 +67,10 @@ clamd = "*"
aiohttp = ">=3.7.4"
tau-clients = "*"
vt-py = ">=0.7.1"
crowdstrike-falconpy = "0.9.0"
censys = "2.0.9"
mwdblib = "3.4.1"
ndjson = "0.3.1"
[requires]
python_version = "3.6"
python_version = "3.7"

View File

@ -58,6 +58,7 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj
* [macvendors](misp_modules/modules/expansion/macvendors.py) - a hover module to retrieve mac vendor information.
* [MALWAREbazaar](misp_modules/modules/expansion/malwarebazaar.py) - an expansion module to query MALWAREbazaar with some payload.
* [McAfee MVISION Insights](misp_modules/modules/expansion/mcafee_insights_enrich.py) - an expansion module enrich IOCs with McAfee MVISION Insights.
* [Mmdb server lookup](misp_modules/modules/expansion/mmdb_lookup.py) - an expansion module to enrich an ip with geolocation information from an mmdb server such as ip.circl.lu.
* [ocr-enrich](misp_modules/modules/expansion/ocr_enrich.py) - an enrichment module to get OCRized data from images into MISP.
* [ods-enrich](misp_modules/modules/expansion/ods_enrich.py) - an enrichment module to get text out of OpenOffice spreadsheet document into MISP (using free-text parser).
* [odt-enrich](misp_modules/modules/expansion/odt_enrich.py) - an enrichment module to get text out of OpenOffice document into MISP (using free-text parser).

View File

@ -6,150 +6,165 @@
#
-i https://pypi.org/simple
-e .
-e git+https://github.com/D4-project/BGP-Ranking.git/@68de39f6c5196f796055c1ac34504054d688aa59#egg=pybgpranking&subdirectory=client
-e git+https://github.com/D4-project/IPASN-History.git/@a2853c39265cecdd0c0d16850bd34621c0551b87#egg=pyipasnhistory&subdirectory=client
-e git+https://github.com/MISP/PyIntel471.git@917272fafa8e12102329faca52173e90c5256968#egg=pyintel471
-e git+https://github.com/cartertemm/ODTReader.git/@49d6938693f6faa3ff09998f86dba551ae3a996b#egg=odtreader
-e git+https://github.com/sebdraven/pydnstrails@48c1f740025c51289f43a24863d1845ff12fd21a#egg=pydnstrails
-e git+https://github.com/sebdraven/pyonyphe@1ce15581beebb13e841193a08a2eb6f967855fcb#egg=pyonyphe
git+https://github.com/SteveClement/trustar-python.git
aiohttp==3.7.4
.
aiohttp==3.8.1
aiosignal==1.2.0; python_version >= '3.6'
antlr4-python3-runtime==4.8; python_version >= '3'
apiosintds==1.8.3
appdirs==1.4.4
argparse==1.4.0
assemblyline-client==4.1.0
async-timeout==3.0.1; python_full_version >= '3.5.3'
attrs==21.2.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
assemblyline-client==4.2.2
async-timeout==4.0.2; python_version >= '3.6'
asynctest==0.13.0; python_version < '3.8'
attrs==21.4.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
backoff==1.11.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
backports.zoneinfo==0.2.1; python_version < '3.9'
backscatter==0.2.4
beautifulsoup4==4.9.3
bidict==0.21.2; python_version >= '3.6'
beautifulsoup4==4.10.0
bidict==0.21.4; python_version >= '3.6'
blockchain==1.4.4
certifi==2021.5.30
censys==2.0.9
cffi==1.14.6
#chardet==4.0.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
chardet
charset-normalizer==2.0.4; python_version >= '3'
censys==2.1.2
certifi==2021.10.8
cffi==1.15.0
chardet==4.0.0
charset-normalizer==2.0.11; python_version >= '3'
clamd==1.0.2
click-plugins==1.1.1
click==8.0.1; python_version >= '3.6'
click==8.0.3; python_version >= '3.6'
colorama==0.4.4; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
colorclass==2.2.0
colorclass==2.2.2; python_version >= '2.6'
commonmark==0.9.1
compressed-rtf==1.0.6
configparser==5.0.2; python_version >= '3.6'
crowdstrike-falconpy==0.9.0
cryptography==3.4.7; python_version >= '3.6'
decorator==5.0.9; python_version >= '3.5'
deprecated==1.2.12; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
configparser==5.2.0; python_version >= '3.6'
crowdstrike-falconpy==1.0.0
cryptography==36.0.1; python_version >= '3.6'
decorator==5.1.1; python_version >= '3.5'
deprecated==1.2.13; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
dnsdb2==1.1.3
dnspython==2.1.0
domaintools-api==0.5.4
dnspython==2.2.0
domaintools-api==0.6.1
easygui==0.98.2
ebcdic==1.1.1
enum-compat==0.0.3
extract-msg==0.28.7
ez-setup==0.9
ezodf==0.3.2
filelock==3.0.12
filelock==3.4.2; python_version >= '3.7'
frozenlist==1.3.0; python_version >= '3.7'
future==0.18.2; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
futures==3.1.1
geoip2==4.2.0
httplib2==0.19.1
geoip2==4.5.0
git+https://github.com/D4-project/BGP-Ranking.git/@68de39f6c5196f796055c1ac34504054d688aa59#egg=pybgpranking&subdirectory=client
git+https://github.com/D4-project/IPASN-History.git/@a2853c39265cecdd0c0d16850bd34621c0551b87#egg=pyipasnhistory&subdirectory=client
git+https://github.com/MISP/PyIntel471.git@917272fafa8e12102329faca52173e90c5256968#egg=pyintel471
git+https://github.com/SteveClement/trustar-python.git@6954eae38e0c77eaeef26084b6c5fd033925c1c7#egg=trustar
git+https://github.com/cartertemm/ODTReader.git/@49d6938693f6faa3ff09998f86dba551ae3a996b#egg=odtreader
git+https://github.com/sebdraven/pydnstrails@48c1f740025c51289f43a24863d1845ff12fd21a#egg=pydnstrails
git+https://github.com/sebdraven/pyonyphe@aed008ee5a27e3a5e4afbb3e5cbfc47170108452#egg=pyonyphe
httplib2==0.20.4; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
idna-ssl==1.1.0; python_version < '3.7'
idna==3.2; python_version >= '3'
idna==3.3; python_version >= '3'
imapclient==2.1.0
isodate==0.6.0
importlib-metadata==4.10.1; python_version < '3.8'
isodate==0.6.1
itsdangerous==2.0.1; python_version >= '3.6'
jbxapi==3.17.2
json-log-formatter==0.4.0
jeepney==0.7.1; sys_platform == 'linux'
json-log-formatter==0.5.1
jsonschema==3.2.0
lark-parser==0.11.3
keyring==23.5.0; python_version >= '3.7'
lark-parser==0.12.0
lief==0.11.5
lxml==4.7.1
maclookup==1.0.3
markdownify==0.5.3
maxminddb==2.0.3; python_version >= '3.6'
more-itertools==8.8.0; python_version >= '3.5'
msoffcrypto-tool==4.12.0; python_version >= '3' and platform_python_implementation != 'PyPy' or (platform_system != 'Windows' and platform_system != 'Darwin')
multidict==5.1.0; python_version >= '3.6'
maxminddb==2.2.0; python_version >= '3.6'
more-itertools==8.12.0; python_version >= '3.5'
msoffcrypto-tool==5.0.0; python_version >= '3' and platform_python_implementation != 'PyPy' or (platform_system != 'Windows' and platform_system != 'Darwin')
multidict==6.0.2; python_version >= '3.7'
mwdblib==4.0.0
ndjson==0.3.1
np==1.0.2
numpy==1.21.2; python_version < '3.11' and python_version >= '3.7'
numpy==1.21.5; python_version < '3.10' and platform_machine != 'aarch64' and platform_machine != 'arm64'
oauth2==1.9.0.post1
olefile==0.46; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
oletools==0.56.2
opencv-python==4.5.3.56
oletools==0.60
opencv-python==4.5.5.62
packaging==21.3; python_version >= '3.6'
pandas-ods-reader==0.1.2
pandas==1.3.5
passivetotal==2.5.4
passivetotal==2.5.8
pcodedmp==1.2.6
pdftotext==2.2.0
pillow==8.3.2
progressbar2==3.53.1
psutil==5.8.0; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
pycparser==2.20; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
pycryptodome==3.10.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
pycryptodomex==3.10.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
pdftotext==2.2.2
pillow==9.0.1
progressbar2==4.0.0; python_version >= '3.7'
psutil==5.9.0; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
pycparser==2.21
pycryptodome==3.14.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
pycryptodomex==3.14.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
pydeep==0.4
pyeupi==1.1
pyfaup==1.2
pygeoip==0.3.2
pymisp[email,fileobjects,openioc,pdfexport]==2.4.148
pygments==2.11.2; python_version >= '3.5'
pymisp[email,fileobjects,openioc,pdfexport,url]==2.4.152
pyparsing==2.4.7; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
pypdns==1.5.2
pypssl==2.2
pyrsistent==0.18.0; python_version >= '3.6'
pyrsistent==0.18.1; python_version >= '3.7'
pytesseract==0.3.8
python-baseconv==1.2.2
python-dateutil==2.8.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
python-docx==0.8.11
python-engineio==4.2.1; python_version >= '3.6'
python-magic==0.4.24
python-pptx==0.6.19
python-socketio[client]==5.4.0; python_version >= '3.6'
python-utils==2.5.6
python-engineio==4.3.1; python_version >= '3.6'
python-magic==0.4.25
python-pptx==0.6.21
python-socketio[client]==5.5.1; python_version >= '3.6'
python-utils==3.1.0; python_version >= '3.7'
pytz-deprecation-shim==0.1.0.post0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
pytz==2019.3
pyyaml==5.4.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
pyyaml==6.0; python_version >= '3.6'
pyzbar==0.1.8
pyzipper==0.3.5; python_version >= '3.5'
rdflib==6.0.0; python_version >= '3.7'
redis==3.5.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
reportlab==3.6.1
rdflib==6.1.1; python_version >= '3.7'
redis==4.1.2; python_version >= '3.6'
reportlab==3.6.6
requests-cache==0.6.4; python_version >= '3.6'
requests-file==1.5.1
requests[security]==2.26.0
requests[security]==2.27.1
rich==11.1.0; python_full_version >= '3.6.2' and python_full_version < '4.0.0'
rtfde==0.0.2
ruamel.yaml.clib==0.2.6; python_version < '3.10' and platform_python_implementation == 'CPython'
ruamel.yaml==0.17.13; python_version >= '3'
shodan==1.25.0
secretstorage==3.3.1; sys_platform == 'linux'
setuptools==60.7.1; python_version >= '3.7'
shodan==1.26.1
sigmatools==0.19.1
six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
socialscan==1.4.2
socketio-client==0.5.7.4
soupsieve==2.2.1; python_version >= '3'
soupsieve==2.3.1; python_version >= '3.6'
sparqlwrapper==1.8.5
stix2==3.0.1
stix2-patterns==1.3.2
tabulate==0.8.9
tau-clients==0.1.3
tldextract==3.1.0; python_version >= '3.5'
tau-clients==0.1.9
taxii2-client==2.3.0
tldextract==3.1.2; python_version >= '3.6'
tornado==6.1; python_version >= '3.5'
tqdm==4.62.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
typing-extensions==3.10.0.0
tzlocal==3.0; python_version >= '3.6'
tqdm==4.62.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
typing-extensions==4.0.1; python_version < '3.8'
tzdata==2021.5; python_version >= '3.6'
tzlocal==4.1; python_version >= '3.6'
unicodecsv==0.14.1
url-normalize==1.4.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
urlarchiver==0.2
urllib3==1.26.6; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'
urllib3==1.26.8; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_full_version < '4.0.0'
validators==0.14.0
vt-graph-api==1.1.2
vt-py==0.7.2
vulners==1.5.12
vt-graph-api==1.1.3
vt-py==0.13.1
vulners==2.0.0
wand==0.6.7
websocket-client==1.2.1
wrapt==1.12.1
websocket-client==1.2.3; python_version >= '3.6'
wrapt==1.13.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
xlrd==2.0.1
xlsxwriter==3.0.1; python_version >= '3.4'
xlsxwriter==3.0.2; python_version >= '3.4'
yara-python==3.8.1
yarl==1.6.3; python_version >= '3.6'
ndjson==0.3.1
mwdblib==3.4.1
yarl==1.7.2; python_version >= '3.6'
zipp==3.7.0; python_version >= '3.7'

View File

@ -2045,6 +2045,25 @@ Module to export a structured CSV file for uploading to ThreatConnect.
-----
#### [virustotal_collections](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/export_mod/virustotal_collections.py)
<img src=logos/virustotal.png height=60>
Creates a VT Collection from an event iocs.
- **features**:
>This export module which takes advantage of a new endpoint in VT APIv3 to create VT Collections from IOCs contained in a MISP event. With this module users will be able to create a collection just using the Download as... button.
- **input**:
>A domain, hash (md5, sha1, sha256 or sha512), hostname, url or IP address attribute.
- **output**:
>A VirusTotal collection in VT.
- **references**:
> - https://www.virustotal.com/
> - https://blog.virustotal.com/2021/11/introducing-virustotal-collections.html
- **requirements**:
>An access to the VirusTotal API (apikey).
-----
#### [vt_graph](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/export_mod/vt_graph.py)
<img src=logos/virustotal.png height=60>

View File

@ -0,0 +1,14 @@
{
"description": "Creates a VT Collection from an event iocs.",
"logo": "virustotal.png",
"requirements": [
"An access to the VirusTotal API (apikey)."
],
"input": "A domain, hash (md5, sha1, sha256 or sha512), hostname, url or IP address attribute.",
"output": "A VirusTotal collection in VT.",
"references": [
"https://www.virustotal.com/",
"https://blog.virustotal.com/2021/11/introducing-virustotal-collections.html"
],
"features": "This export module which takes advantage of a new endpoint in VT APIv3 to create VT Collections from IOCs contained in a MISP event. With this module users will be able to create a collection just using the Download as... button."
}

@ -0,0 +1 @@
Subproject commit 9dc7e3578f2165e32a3b7cdd09e9e552f2d98d36

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,460 @@
################################################################################
# ATTRIBUTES AND OBJECTS MAPPING #
################################################################################
attributes_mapping = {
'filename': '_parse_name',
'ip-src': '_parse_value',
'ip-dst': '_parse_value',
'hostname': '_parse_value',
'domain': '_parse_value',
'domain|ip': '_parse_domain_ip_attribute',
'email-src': '_parse_value',
'email-dst': '_parse_value',
'email-attachment': '_parse_name',
'url': '_parse_value',
'regkey': '_parse_regkey_attribute',
'regkey|value': '_parse_regkey_value',
'malware-sample': '_parse_malware_sample',
'mutex': '_parse_name',
'uri': '_parse_value',
'port': '_parse_port',
'ip-dst|port': '_parse_network_attribute',
'ip-src|port': '_parse_network_attribute',
'hostname|port': '_parse_network_attribute',
'email-reply-to': '_parse_email_reply_to',
'attachment': '_parse_attachment',
'mac-address': '_parse_value',
'AS': '_parse_number'
}
attributes_type_mapping = {
'md5': '_parse_hash',
'sha1': '_parse_hash',
'sha256': '_parse_hash',
'filename|md5': '_parse_filename_hash',
'filename|sha1': '_parse_filename_hash',
'filename|sha256': '_parse_filename_hash',
'email-subject': '_parse_email_message',
'email-body': '_parse_email_message',
'authentihash': '_parse_hash',
'ssdeep': '_parse_hash',
'imphash': '_parse_hash',
'pehash': '_parse_hash',
'impfuzzy': '_parse_hash',
'sha224': '_parse_hash',
'sha384': '_parse_hash',
'sha512': '_parse_hash',
'sha512/224': '_parse_hash',
'sha512/256': '_parse_hash',
'tlsh': '_parse_hash',
'cdhash': '_parse_hash',
'filename|authentihash': '_parse_filename_hash',
'filename|ssdeep': '_parse_filename_hash',
'filename|imphash': '_parse_filename_hash',
'filename|impfuzzy': '_parse_filename_hash',
'filename|pehash': '_parse_filename_hash',
'filename|sha224': '_parse_filename_hash',
'filename|sha384': '_parse_filename_hash',
'filename|sha512': '_parse_filename_hash',
'filename|sha512/224': '_parse_filename_hash',
'filename|sha512/256': '_parse_filename_hash',
'filename|tlsh': '_parse_filename_hash',
'x509-fingerprint-md5': '_parse_x509_attribute',
'x509-fingerprint-sha1': '_parse_x509_attribute',
'x509-fingerprint-sha256': '_parse_x509_attribute'
}
objects_mapping = {
'asn': {
'observable': 'parse_asn_observable',
'pattern': 'parse_asn_pattern'},
'credential': {
'observable': 'parse_credential_observable',
'pattern': 'parse_credential_pattern'},
'domain-ip': {
'observable': 'parse_domain_ip_observable',
'pattern': 'parse_domain_ip_pattern'},
'email': {
'observable': 'parse_email_observable',
'pattern': 'parse_email_pattern'},
'file': {
'observable': 'parse_file_observable',
'pattern': 'parse_file_pattern'},
'ip-port': {
'observable': 'parse_ip_port_observable',
'pattern': 'parse_ip_port_pattern'},
'network-connection': {
'observable': 'parse_network_connection_observable',
'pattern': 'parse_network_connection_pattern'},
'network-socket': {
'observable': 'parse_network_socket_observable',
'pattern': 'parse_network_socket_pattern'},
'process': {
'observable': 'parse_process_observable',
'pattern': 'parse_process_pattern'},
'registry-key': {
'observable': 'parse_regkey_observable',
'pattern': 'parse_regkey_pattern'},
'url': {
'observable': 'parse_url_observable',
'pattern': 'parse_url_pattern'},
'user-account': {
'observable': 'parse_user_account_observable',
'pattern': 'parse_user_account_pattern'},
'WindowsPEBinaryFile': {
'observable': 'parse_pe_observable',
'pattern': 'parse_pe_pattern'},
'x509': {
'observable': 'parse_x509_observable',
'pattern': 'parse_x509_pattern'}
}
observable_mapping = {
('artifact', 'file'): 'parse_file_observable',
('artifact', 'directory', 'file'): 'parse_file_observable',
('artifact', 'email-addr', 'email-message', 'file'): 'parse_email_observable',
('autonomous-system',): 'parse_asn_observable',
('autonomous-system', 'ipv4-addr'): 'parse_asn_observable',
('autonomous-system', 'ipv6-addr'): 'parse_asn_observable',
('autonomous-system', 'ipv4-addr', 'ipv6-addr'): 'parse_asn_observable',
('directory', 'file'): 'parse_file_observable',
('domain-name',): 'parse_domain_ip_observable',
('domain-name', 'ipv4-addr'): 'parse_domain_ip_observable',
('domain-name', 'ipv6-addr'): 'parse_domain_ip_observable',
('domain-name', 'ipv4-addr', 'ipv6-addr'): 'parse_domain_ip_observable',
('domain-name', 'ipv4-addr', 'network-traffic'): 'parse_domain_ip_network_traffic_observable',
('domain-name', 'ipv6-addr', 'network-traffic'): 'parse_domain_ip_network_traffic_observable',
('domain-name', 'ipv4-addr', 'ipv6-addr', 'network-traffic'): 'parse_domain_ip_network_traffic_observable',
('domain-name', 'network-traffic'): 'parse_domain_network_traffic_observable',
('domain-name', 'network-traffic', 'url'): 'parse_url_observable',
('email-addr',): 'parse_email_address_observable',
('email-addr', 'email-message'): 'parse_email_observable',
('email-addr', 'email-message', 'file'): 'parse_email_observable',
('email-message',): 'parse_email_observable',
('file',): 'parse_file_observable',
('file', 'process'): 'parse_process_observable',
('ipv4-addr',): 'parse_ip_address_observable',
('ipv6-addr',): 'parse_ip_address_observable',
('ipv4-addr', 'network-traffic'): 'parse_ip_network_traffic_observable',
('ipv6-addr', 'network-traffic'): 'parse_ip_network_traffic_observable',
('ipv4-addr', 'ipv6-addr', 'network-traffic'): 'parse_ip_network_traffic_observable',
('mac-addr',): 'parse_mac_address_observable',
('mutex',): 'parse_mutex_observable',
('process',): 'parse_process_observable',
('x509-certificate',): 'parse_x509_observable',
('url',): 'parse_url_observable',
('user-account',): 'parse_user_account_observable',
('windows-registry-key',): 'parse_regkey_observable'
}
pattern_mapping = {
('artifact', 'file'): 'parse_file_pattern',
('artifact', 'directory', 'file'): 'parse_file_pattern',
('autonomous-system', ): 'parse_as_pattern',
('autonomous-system', 'ipv4-addr'): 'parse_as_pattern',
('autonomous-system', 'ipv6-addr'): 'parse_as_pattern',
('autonomous-system', 'ipv4-addr', 'ipv6-addr'): 'parse_as_pattern',
('directory',): 'parse_file_pattern',
('directory', 'file'): 'parse_file_pattern',
('domain-name',): 'parse_domain_ip_port_pattern',
('domain-name', 'ipv4-addr'): 'parse_domain_ip_port_pattern',
('domain-name', 'ipv6-addr'): 'parse_domain_ip_port_pattern',
('domain-name', 'ipv4-addr', 'ipv6-addr'): 'parse_domain_ip_port_pattern',
('domain-name', 'ipv4-addr', 'url'): 'parse_url_pattern',
('domain-name', 'ipv6-addr', 'url'): 'parse_url_pattern',
('domain-name', 'ipv4-addr', 'ipv6-addr', 'url'): 'parse_url_pattern',
('domain-name', 'network-traffic'): 'parse_domain_ip_port_pattern',
('domain-name', 'network-traffic', 'url'): 'parse_url_pattern',
('email-addr',): 'parse_email_address_pattern',
('email-message',): 'parse_email_message_pattern',
('file',): 'parse_file_pattern',
('ipv4-addr',): 'parse_ip_address_pattern',
('ipv6-addr',): 'parse_ip_address_pattern',
('ipv4-addr', 'ipv6-addr'): 'parse_ip_address_pattern',
('mac-addr',): 'parse_mac_address_pattern',
('mutex',): 'parse_mutex_pattern',
('network-traffic',): 'parse_network_traffic_pattern',
('process',): 'parse_process_pattern',
('url',): 'parse_url_pattern',
('user-account',): 'parse_user_account_pattern',
('windows-registry-key',): 'parse_regkey_pattern',
('x509-certificate',): 'parse_x509_pattern'
}
pattern_forbidden_relations = (' LIKE ', ' FOLLOWEDBY ', ' MATCHES ', ' ISSUBSET ', ' ISSUPERSET ', ' REPEATS ')
single_attribute_fields = ('type', 'value', 'to_ids')
################################################################################
# OBSERVABLE OBJECTS AND PATTERNS MAPPING. #
################################################################################
address_family_attribute_mapping = {'type': 'text','object_relation': 'address-family'}
as_number_attribute_mapping = {'type': 'AS', 'object_relation': 'asn'}
description_attribute_mapping = {'type': 'text', 'object_relation': 'description'}
asn_subnet_attribute_mapping = {'type': 'ip-src', 'object_relation': 'subnet-announced'}
cc_attribute_mapping = {'type': 'email-dst', 'object_relation': 'cc'}
credential_attribute_mapping = {'type': 'text', 'object_relation': 'password'}
data_attribute_mapping = {'type': 'text', 'object_relation': 'data'}
data_type_attribute_mapping = {'type': 'text', 'object_relation': 'data-type'}
domain_attribute_mapping = {'type': 'domain', 'object_relation': 'domain'}
domain_family_attribute_mapping = {'type': 'text', 'object_relation': 'domain-family'}
dst_port_attribute_mapping = {'type': 'port', 'object_relation': 'dst-port'}
email_attachment_attribute_mapping = {'type': 'email-attachment', 'object_relation': 'attachment'}
email_date_attribute_mapping = {'type': 'datetime', 'object_relation': 'send-date'}
email_subject_attribute_mapping = {'type': 'email-subject', 'object_relation': 'subject'}
encoding_attribute_mapping = {'type': 'text', 'object_relation': 'file-encoding'}
end_datetime_attribute_mapping = {'type': 'datetime', 'object_relation': 'last-seen'}
entropy_mapping = {'type': 'float', 'object_relation': 'entropy'}
filename_attribute_mapping = {'type': 'filename', 'object_relation': 'filename'}
from_attribute_mapping = {'type': 'email-src', 'object_relation': 'from'}
imphash_mapping = {'type': 'imphash', 'object_relation': 'imphash'}
id_attribute_mapping = {'type': 'text', 'object_relation': 'id'}
ip_attribute_mapping = {'type': 'ip-dst', 'object_relation': 'ip'}
issuer_attribute_mapping = {'type': 'text', 'object_relation': 'issuer'}
key_attribute_mapping = {'type': 'regkey', 'object_relation': 'key'}
malware_sample_attribute_mapping = {'type': 'malware-sample', 'object_relation': 'malware-sample'}
mime_type_attribute_mapping = {'type': 'mime-type', 'object_relation': 'mimetype'}
modified_attribute_mapping = {'type': 'datetime', 'object_relation': 'last-modified'}
name_attribute_mapping = {'type': 'text', 'object_relation': 'name'}
network_traffic_ip = {'type': 'ip-{}', 'object_relation': 'ip-{}'}
number_sections_mapping = {'type': 'counter', 'object_relation': 'number-sections'}
password_mapping = {'type': 'text', 'object_relation': 'password'}
path_attribute_mapping = {'type': 'text', 'object_relation': 'path'}
pe_type_mapping = {'type': 'text', 'object_relation': 'type'}
pid_attribute_mapping = {'type': 'text', 'object_relation': 'pid'}
process_command_line_mapping = {'type': 'text', 'object_relation': 'command-line'}
process_creation_time_mapping = {'type': 'datetime', 'object_relation': 'creation-time'}
process_image_mapping = {'type': 'filename', 'object_relation': 'image'}
process_name_mapping = {'type': 'text', 'object_relation': 'name'}
regkey_name_attribute_mapping = {'type': 'text', 'object_relation': 'name'}
references_attribute_mapping = {'type': 'link', 'object_relation': 'references'}
reply_to_attribute_mapping = {'type': 'email-reply-to', 'object_relation': 'reply-to'}
screenshot_attribute_mapping = {'type': 'attachment', 'object_relation': 'screenshot'}
section_name_mapping = {'type': 'text', 'object_relation': 'name'}
serial_number_attribute_mapping = {'type': 'text', 'object_relation': 'serial-number'}
size_attribute_mapping = {'type': 'size-in-bytes', 'object_relation': 'size-in-bytes'}
src_port_attribute_mapping = {'type': 'port', 'object_relation': 'src-port'}
start_datetime_attribute_mapping = {'type': 'datetime', 'object_relation': 'first-seen'}
state_attribute_mapping = {'type': 'text', 'object_relation': 'state'}
summary_attribute_mapping = {'type': 'text', 'object_relation': 'summary'}
to_attribute_mapping = {'type': 'email-dst', 'object_relation': 'to'}
url_attribute_mapping = {'type': 'url', 'object_relation': 'url'}
url_port_attribute_mapping = {'type': 'port', 'object_relation': 'port'}
user_id_mapping = {'type': 'text', 'object_relation': 'username'}
x_mailer_attribute_mapping = {'type': 'email-x-mailer', 'object_relation': 'x-mailer'}
x509_md5_attribute_mapping = {'type': 'x509-fingerprint-md5', 'object_relation': 'x509-fingerprint-md5'}
x509_sha1_attribute_mapping = {'type': 'x509-fingerprint-sha1', 'object_relation': 'x509-fingerprint-sha1'}
x509_sha256_attribute_mapping = {'type': 'x509-fingerprint-sha256', 'object_relation': 'x509-fingerprint-sha256'}
x509_spka_attribute_mapping = {'type': 'text', 'object_relation': 'pubkey-info-algorithm'} # x509 subject public key algorithm
x509_spke_attribute_mapping = {'type': 'text', 'object_relation': 'pubkey-info-exponent'} # x509 subject public key exponent
x509_spkm_attribute_mapping = {'type': 'text', 'object_relation': 'pubkey-info-modulus'} # x509 subject public key modulus
x509_subject_attribute_mapping = {'type': 'text', 'object_relation': 'subject'}
x509_version_attribute_mapping = {'type': 'text', 'object_relation': 'version'}
x509_vna_attribute_mapping = {'type': 'datetime', 'object_relation': 'validity-not-after'} # x509 validity not after
x509_vnb_attribute_mapping = {'type': 'datetime', 'object_relation': 'validity-not-before'} # x509 validity not before
asn_mapping = {'number': as_number_attribute_mapping,
'autonomous-system:number': as_number_attribute_mapping,
'name': description_attribute_mapping,
'autonomous-system:name': description_attribute_mapping,
'ipv4-addr': asn_subnet_attribute_mapping,
'ipv6-addr': asn_subnet_attribute_mapping,
'ipv4-addr:value': asn_subnet_attribute_mapping,
'ipv6-addr:value': asn_subnet_attribute_mapping}
attack_pattern_mapping = {'name': name_attribute_mapping,
'description': summary_attribute_mapping}
attack_pattern_references_mapping = {'mitre-attack': references_attribute_mapping,
'capec': id_attribute_mapping}
course_of_action_mapping = {'description': description_attribute_mapping,
'name': name_attribute_mapping}
credential_mapping = {'credential': credential_attribute_mapping,
'user-account:credential': credential_attribute_mapping,
'user_id': user_id_mapping,
'user-account:user_id': user_id_mapping}
domain_ip_mapping = {'domain-name': domain_attribute_mapping,
'domain-name:value': domain_attribute_mapping,
'ipv4-addr': ip_attribute_mapping,
'ipv6-addr': ip_attribute_mapping,
'ipv4-addr:value': ip_attribute_mapping,
'ipv6-addr:value': ip_attribute_mapping,
'domain-name:resolves_to_refs[*].value': ip_attribute_mapping,
'network-traffic:dst_port': dst_port_attribute_mapping,
'network-traffic:src_port': src_port_attribute_mapping}
email_mapping = {'date': email_date_attribute_mapping,
'email-message:date': email_date_attribute_mapping,
'email-message:to_refs[*].value': to_attribute_mapping,
'email-message:cc_refs[*].value': cc_attribute_mapping,
'subject': email_subject_attribute_mapping,
'email-message:subject': email_subject_attribute_mapping,
'X-Mailer': x_mailer_attribute_mapping,
'email-message:additional_header_fields.x_mailer': x_mailer_attribute_mapping,
'Reply-To': reply_to_attribute_mapping,
'email-message:additional_header_fields.reply_to': reply_to_attribute_mapping,
'email-message:from_ref.value': from_attribute_mapping,
'email-addr:value': to_attribute_mapping}
email_references_mapping = {'attachment': email_attachment_attribute_mapping,
'cc_refs': cc_attribute_mapping,
'from_ref': from_attribute_mapping,
'screenshot': screenshot_attribute_mapping,
'to_refs': to_attribute_mapping}
file_mapping = {'artifact:mime_type': mime_type_attribute_mapping,
'file:content_ref.mime_type': mime_type_attribute_mapping,
'mime_type': mime_type_attribute_mapping,
'file:mime_type': mime_type_attribute_mapping,
'name': filename_attribute_mapping,
'file:name': filename_attribute_mapping,
'name_enc': encoding_attribute_mapping,
'file:name_enc': encoding_attribute_mapping,
'file:parent_directory_ref.path': path_attribute_mapping,
'directory:path': path_attribute_mapping,
'size': size_attribute_mapping,
'file:size': size_attribute_mapping}
network_traffic_mapping = {'dst_port':dst_port_attribute_mapping,
'src_port': src_port_attribute_mapping,
'network-traffic:dst_port': dst_port_attribute_mapping,
'network-traffic:src_port': src_port_attribute_mapping}
ip_port_mapping = {'value': domain_attribute_mapping,
'domain-name:value': domain_attribute_mapping,
'network-traffic:dst_ref.value': {'type': 'ip-dst', 'object_relation': 'ip-dst'},
'network-traffic:src_ref.value': {'type': 'ip-src', 'object_relation': 'ip-src'}}
ip_port_mapping.update(network_traffic_mapping)
ip_port_references_mapping = {'domain-name': domain_attribute_mapping,
'ipv4-addr': network_traffic_ip,
'ipv6-addr': network_traffic_ip}
network_socket_extension_mapping = {'address_family': address_family_attribute_mapping,
"network-traffic:extensions.'socket-ext'.address_family": address_family_attribute_mapping,
'protocol_family': domain_family_attribute_mapping,
"network-traffic:extensions.'socket-ext'.protocol_family": domain_family_attribute_mapping,
'is_blocking': state_attribute_mapping,
"network-traffic:extensions.'socket-ext'.is_blocking": state_attribute_mapping,
'is_listening': state_attribute_mapping,
"network-traffic:extensions.'socket-ext'.is_listening": state_attribute_mapping}
network_traffic_references_mapping = {'domain-name': {'type': 'hostname', 'object_relation': 'hostname-{}'},
'ipv4-addr': network_traffic_ip,
'ipv6-addr': network_traffic_ip}
pe_mapping = {'pe_type': pe_type_mapping, 'number_of_sections': number_sections_mapping, 'imphash': imphash_mapping}
pe_section_mapping = {'name': section_name_mapping, 'size': size_attribute_mapping, 'entropy': entropy_mapping}
hash_types = ('MD5', 'SHA-1', 'SHA-256', 'SHA-224', 'SHA-384', 'SHA-512', 'ssdeep', 'tlsh')
for hash_type in hash_types:
misp_hash_type = hash_type.replace('-', '').lower()
attribute = {'type': misp_hash_type, 'object_relation': misp_hash_type}
file_mapping[hash_type] = attribute
file_mapping.update({f"file:hashes.'{feature}'": attribute for feature in (hash_type, misp_hash_type)})
file_mapping.update({f"file:hashes.{feature}": attribute for feature in (hash_type, misp_hash_type)})
pe_section_mapping[hash_type] = attribute
pe_section_mapping[misp_hash_type] = attribute
process_mapping = {'name': process_name_mapping,
'process:name': process_name_mapping,
'pid': pid_attribute_mapping,
'process:pid': pid_attribute_mapping,
'created': process_creation_time_mapping,
'process:created': process_creation_time_mapping,
'command_line': process_command_line_mapping,
'process:command_line': process_command_line_mapping,
'process:parent_ref.pid': {'type': 'text', 'object_relation': 'parent-pid'},
'process:child_refs[*].pid': {'type': 'text', 'object_relation': 'child-pid'},
'process:binary_ref.name': process_image_mapping}
child_process_reference_mapping = {'pid': {'type': 'text', 'object_relation': 'child-pid'}}
parent_process_reference_mapping = {'command_line': {'type': 'text', 'object_relation': 'parent-command-line'},
'pid': {'type': 'text', 'object_relation': 'parent-pid'},
'process-name': {'type': 'text', 'object_relation': 'parent-process-name'}}
regkey_mapping = {'data': data_attribute_mapping,
'windows-registry-key:values.data': data_attribute_mapping,
'data_type': data_type_attribute_mapping,
'windows-registry-key:values.data_type': data_type_attribute_mapping,
'modified': modified_attribute_mapping,
'windows-registry-key:modified': modified_attribute_mapping,
'name': regkey_name_attribute_mapping,
'windows-registry-key:values.name': regkey_name_attribute_mapping,
'key': key_attribute_mapping,
'windows-registry-key:key': key_attribute_mapping,
'windows-registry-key:value': {'type': 'text', 'object_relation': 'hive'}
}
url_mapping = {'url': url_attribute_mapping,
'url:value': url_attribute_mapping,
'domain-name': domain_attribute_mapping,
'domain-name:value': domain_attribute_mapping,
'network-traffic': url_port_attribute_mapping,
'network-traffic:dst_port': url_port_attribute_mapping,
'ipv4-addr:value': ip_attribute_mapping,
'ipv6-addr:value': ip_attribute_mapping
}
user_account_mapping = {'account_created': {'type': 'datetime', 'object_relation': 'created'},
'account_expires': {'type': 'datetime', 'object_relation': 'expires'},
'account_first_login': {'type': 'datetime', 'object_relation': 'first_login'},
'account_last_login': {'type': 'datetime', 'object_relation': 'last_login'},
'account_login': user_id_mapping,
'account_type': {'type': 'text', 'object_relation': 'account-type'},
'can_escalate_privs': {'type': 'boolean', 'object_relation': 'can_escalate_privs'},
'credential': credential_attribute_mapping,
'credential_last_changed': {'type': 'datetime', 'object_relation': 'password_last_changed'},
'display_name': {'type': 'text', 'object_relation': 'display-name'},
'gid': {'type': 'text', 'object_relation': 'group-id'},
'home_dir': {'type': 'text', 'object_relation': 'home_dir'},
'is_disabled': {'type': 'boolean', 'object_relation': 'disabled'},
'is_privileged': {'type': 'boolean', 'object_relation': 'privileged'},
'is_service_account': {'type': 'boolean', 'object_relation': 'is_service_account'},
'shell': {'type': 'text', 'object_relation': 'shell'},
'user_id': {'type': 'text', 'object_relation': 'user-id'}}
vulnerability_mapping = {'name': id_attribute_mapping,
'description': summary_attribute_mapping}
x509_mapping = {'issuer': issuer_attribute_mapping,
'x509-certificate:issuer': issuer_attribute_mapping,
'serial_number': serial_number_attribute_mapping,
'x509-certificate:serial_number': serial_number_attribute_mapping,
'subject': x509_subject_attribute_mapping,
'x509-certificate:subject': x509_subject_attribute_mapping,
'subject_public_key_algorithm': x509_spka_attribute_mapping,
'x509-certificate:subject_public_key_algorithm': x509_spka_attribute_mapping,
'subject_public_key_exponent': x509_spke_attribute_mapping,
'x509-certificate:subject_public_key_exponent': x509_spke_attribute_mapping,
'subject_public_key_modulus': x509_spkm_attribute_mapping,
'x509-certificate:subject_public_key_modulus': x509_spkm_attribute_mapping,
'validity_not_before': x509_vnb_attribute_mapping,
'x509-certificate:validity_not_before': x509_vnb_attribute_mapping,
'validity_not_after': x509_vna_attribute_mapping,
'x509-certificate:validity_not_after': x509_vna_attribute_mapping,
'version': x509_version_attribute_mapping,
'x509-certificate:version': x509_version_attribute_mapping,
'SHA-1': x509_sha1_attribute_mapping,
"x509-certificate:hashes.'sha1'": x509_sha1_attribute_mapping,
'SHA-256': x509_sha256_attribute_mapping,
"x509-certificate:hashes.'sha256'": x509_sha256_attribute_mapping,
'MD5': x509_md5_attribute_mapping,
"x509-certificate:hashes.'md5'": x509_md5_attribute_mapping,
}
attachment_types = ('file:content_ref.name', 'file:content_ref.payload_bin',
'artifact:x_misp_text_name', 'artifact:payload_bin',
"file:hashes.'MD5'", "file:content_ref.hashes.'MD5'",
'file:name')
connection_protocols = {"IP": "3", "ICMP": "3", "ARP": "3",
"TCP": "4", "UDP": "4",
"HTTP": "7", "HTTPS": "7", "FTP": "7"}

File diff suppressed because one or more lines are too long

View File

@ -4,8 +4,8 @@ from . import check_input_attribute, standard_error_message
from pymisp import MISPAttribute, MISPEvent, MISPObject
misperrors = {'error': 'Error'}
mispattributes = {'input': ['domain', 'hostname'], 'format': 'misp_standard'}
moduleinfo = {'version': '0.1', 'author': 'Christian Studer',
mispattributes = {'input': ['domain', 'hostname', 'email'], 'format': 'misp_standard'}
moduleinfo = {'version': '0.2', 'author': 'Christian Studer',
'description': 'On demand query API for APIVoid.',
'module-type': ['expansion', 'hover']}
moduleconfig = ['apikey']
@ -43,6 +43,31 @@ class APIVoidParser():
ssl = requests.get(f'{self.url.format("sslinfo", apikey)}host={self.attribute.value}').json()
self._parse_ssl_certificate(ssl['data']['certificate'])
def handle_email(self, apikey):
feature = 'emailverify'
if requests.get(f'{self.url.format(feature, apikey)}stats').json()['credits_remained'] < 0.06:
self.result = {'error': 'You do not have enough APIVoid credits to proceed your request.'}
return
emaillookup = requests.get(f'{self.url.format(feature, apikey)}email={self.attribute.value}').json()
email_verification = MISPObject('apivoid-email-verification')
boolean_attributes = ['valid_format', 'suspicious_username', 'suspicious_email', 'dirty_words_username',
'suspicious_email', 'valid_tld', 'disposable', 'has_a_records', 'has_mx_records',
'has_spf_records', 'is_spoofable', 'dmarc_configured', 'dmarc_enforced', 'free_email',
'russian_free_email', 'china_free_email', 'suspicious_domain', 'dirty_words_domain',
'domain_popular', 'risky_tld', 'police_domain', 'government_domain', 'educational_domain',
'should_block']
for boolean_attribute in boolean_attributes:
email_verification.add_attribute(boolean_attribute,
**{'type': 'boolean', 'value': emaillookup['data'][boolean_attribute]})
email_verification.add_attribute('email', **{'type': 'email', 'value': emaillookup['data']['email']})
email_verification.add_attribute('username', **{'type': 'text', 'value': emaillookup['data']['username']})
email_verification.add_attribute('role_address',
**{'type': 'boolean', 'value': emaillookup['data']['role_address']})
email_verification.add_attribute('domain', **{'type': 'domain', 'value': emaillookup['data']['domain']})
email_verification.add_attribute('score', **{'type': 'float', 'value': emaillookup['data']['score']})
email_verification.add_reference(self.attribute['uuid'], 'related-to')
self.misp_event.add_object(email_verification)
def _handle_dns_record(self, item, record_type, relationship):
dns_record = MISPObject('dns-record')
dns_record.add_attribute('queried-domain', type='domain', value=item['host'])
@ -82,7 +107,10 @@ def handler(q=False):
return {'error': 'Unsupported attribute type.'}
apikey = request['config']['apikey']
apivoid_parser = APIVoidParser(attribute)
apivoid_parser.parse_domain(apikey)
if attribute['type'] in ['domain', 'hostname']:
apivoid_parser.parse_domain(apikey)
else:
apivoid_parser.handle_email(apikey)
return apivoid_parser.get_results()

View File

@ -0,0 +1,129 @@
import json
import requests
from . import check_input_attribute, standard_error_message
from pymisp import MISPEvent, MISPObject
misperrors = {'error': 'Error'}
mispattributes = {'input': ['ip-src', 'ip-src|port', 'ip-dst', 'ip-dst|port'], 'format': 'misp_standard'}
moduleinfo = {'version': '1', 'author': 'Jeroen Pinoy',
'description': "An expansion module to enrich an ip with geolocation information from an mmdb server "
"such as ip.circl.lu.",
'module-type': ['expansion', 'hover']}
moduleconfig = ["custom_API", "db_source_filter"]
mmdblookup_url = 'https://ip.circl.lu/'
class MmdbLookupParser():
def __init__(self, attribute, mmdblookupresult, api_url):
self.attribute = attribute
self.mmdblookupresult = mmdblookupresult
self.api_url = api_url
self.misp_event = MISPEvent()
self.misp_event.add_attribute(**attribute)
def get_result(self):
event = json.loads(self.misp_event.to_json())
results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])}
return {'results': results}
def parse_mmdblookup_information(self):
# There is a chance some db's have a hit while others don't so we have to check if entry is empty each time
for result_entry in self.mmdblookupresult:
if result_entry['country_info']:
mmdblookup_object = MISPObject('geolocation')
mmdblookup_object.add_attribute('country',
**{'type': 'text', 'value': result_entry['country_info']['Country']})
mmdblookup_object.add_attribute('countrycode',
**{'type': 'text', 'value': result_entry['country']['iso_code']})
mmdblookup_object.add_attribute('latitude',
**{'type': 'float',
'value': result_entry['country_info']['Latitude (average)']})
mmdblookup_object.add_attribute('longitude',
**{'type': 'float',
'value': result_entry['country_info']['Longitude (average)']})
mmdblookup_object.add_attribute('text',
**{'type': 'text',
'value': 'db_source: {}. build_db: {}. Latitude and longitude are country average.'.format(
result_entry['meta']['db_source'],
result_entry['meta']['build_db'])})
mmdblookup_object.add_reference(self.attribute['uuid'], 'related-to')
self.misp_event.add_object(mmdblookup_object)
if 'AutonomousSystemNumber' in result_entry['country']:
mmdblookup_object_asn = MISPObject('asn')
mmdblookup_object_asn.add_attribute('asn',
**{'type': 'text',
'value': result_entry['country'][
'AutonomousSystemNumber']})
mmdblookup_object_asn.add_attribute('description',
**{'type': 'text',
'value': 'ASNOrganization: {}. db_source: {}. build_db: {}.'.format(
result_entry['country'][
'AutonomousSystemOrganization'],
result_entry['meta']['db_source'],
result_entry['meta']['build_db'])})
mmdblookup_object_asn.add_reference(self.attribute['uuid'], 'related-to')
self.misp_event.add_object(mmdblookup_object_asn)
def check_url(url):
return "{}/".format(url) if not url.endswith('/') else url
def handler(q=False):
if q is False:
return False
request = json.loads(q)
if not request.get('attribute') or not check_input_attribute(request['attribute']):
return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'}
attribute = request['attribute']
if attribute.get('type') == 'ip-src':
toquery = attribute['value']
elif attribute.get('type') == 'ip-src|port':
toquery = attribute['value'].split('|')[0]
elif attribute.get('type') == 'ip-dst':
toquery = attribute['value']
elif attribute.get('type') == 'ip-dst|port':
toquery = attribute['value'].split('|')[0]
else:
misperrors['error'] = 'There is no attribute of type ip-src or ip-dst provided as input'
return misperrors
api_url = check_url(request['config']['custom_API']) if 'config' in request and request['config'].get(
'custom_API') else mmdblookup_url
r = requests.get("{}/geolookup/{}".format(api_url, toquery))
if r.status_code == 200:
mmdblookupresult = r.json()
if not mmdblookupresult or len(mmdblookupresult) == 0:
misperrors['error'] = 'Empty result returned by server'
return misperrors
if 'config' in request and request['config'].get('db_source_filter'):
db_source_filter = request['config'].get('db_source_filter')
mmdblookupresult = [entry for entry in mmdblookupresult if entry['meta']['db_source'] == db_source_filter]
if not mmdblookupresult or len(mmdblookupresult) == 0:
misperrors['error'] = 'There was no result with the selected db_source'
return misperrors
# Server might return one or multiple entries which could all be empty, we check if there is at least one
# non-empty result below
empty_result = True
for lookup_result_entry in mmdblookupresult:
if lookup_result_entry['country_info']:
empty_result = False
break
if empty_result:
misperrors['error'] = 'Empty result returned by server'
return misperrors
else:
misperrors['error'] = 'API not accessible - http status code {} was returned'.format(r.status_code)
return misperrors
parser = MmdbLookupParser(attribute, mmdblookupresult, api_url)
parser.parse_mmdblookup_information()
result = parser.get_result()
return result
def introspection():
return mispattributes
def version():
moduleinfo['config'] = moduleconfig
return moduleinfo

View File

@ -1,2 +1,3 @@
__all__ = ['cef_export', 'mass_eql_export', 'liteexport', 'goamlexport', 'threat_connect_export', 'pdfexport',
'threatStream_misp_export', 'osqueryexport', 'nexthinkexport', 'vt_graph', 'defender_endpoint_export']
'threatStream_misp_export', 'osqueryexport', 'nexthinkexport', 'vt_graph', 'defender_endpoint_export',
'virustotal_collections']

View File

@ -0,0 +1,134 @@
#!/usr/bin/env python3
# Copyright 2022 Google Inc. All Rights Reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Creates a VT Collection with indicators present in a given event."""
import base64
import json
import requests
misperrors = {
'error': 'Error'
}
mispattributes = {
'input': [
'hostname',
'domain',
'ip-src',
'ip-dst',
'md5',
'sha1',
'sha256',
'url'
],
'format': 'misp_standard',
'responseType': 'application/txt',
'outputFileExtension': 'txt',
}
moduleinfo = {
'version': '1.0',
'author': 'VirusTotal',
'description': 'Creates a VT Collection from an event iocs.',
'module-type': ['export']
}
moduleconfig = [
'vt_api_key',
'proxy_host',
'proxy_port',
'proxy_username',
'proxy_password'
]
class VTError(Exception):
"Exception class to map vt api response errors."
pass
def create_collection(api_key, event_data):
headers = {
'x-apikey': api_key,
'content-type': 'application/json',
'x-tool': 'MISPModuleVirusTotalCollectionExport',
}
response = requests.post('https://www.virustotal.com/api/v3/integrations/misp/collections',
headers=headers,
json=event_data)
uuid = event_data['Event']['uuid']
response_data = response.json()
if response.status_code == 200:
link = response_data['data']['links']['self']
return f'{uuid}: {link}'
error = response_data['error']['message']
if response.status_code == 400:
return f'{uuid}: {error}'
else:
misperrors['error'] = error
raise VTError(error)
def normalize_misp_data(data):
normalized_data = {'Event': data.pop('Event', {})}
for attr_key in data:
if isinstance(data[attr_key], list) or isinstance(data[attr_key], dict):
if attr_key == 'EventTag':
normalized_data['Event']['Tag'] = [tag['Tag'] for tag in data[attr_key]]
else:
normalized_data['Event'][attr_key] = data[attr_key]
return normalized_data
def handler(q=False):
request = json.loads(q)
if not request.get('config') or not request['config'].get('vt_api_key'):
misperrors['error'] = 'A VirusTotal api key is required for this module.'
return misperrors
config = request['config']
data = request['data']
responses = []
try:
for event_data in data:
normalized_event = normalize_misp_data(event_data)
responses.append(create_collection(config.get('vt_api_key'),
normalized_event))
output = '\n'.join(responses)
return {
"response": [],
"data": str(base64.b64encode(bytes(output, 'utf-8')), 'utf-8'),
}
except VTError:
return misperrors
def introspection():
return mispattributes
def version():
moduleinfo['config'] = moduleconfig
return moduleinfo

View File

@ -15,4 +15,5 @@ __all__ = [
'csvimport',
'cof2misp',
'joe_import',
'taxii21'
]

View File

@ -0,0 +1,373 @@
"""
Import content from a TAXII 2.1 server.
"""
import collections
import itertools
import json
import misp_modules.lib.stix2misp
from pathlib import Path
import re
import stix2.v20
import taxii2client
import taxii2client.exceptions
import requests
class ConfigError(Exception):
"""
Represents an error in the config settings for one invocation of this
module.
"""
pass
misperrors = {'error': 'Error'}
moduleinfo = {'version': '0.1', 'author': 'Abc',
'description': 'Import content from a TAXII 2.1 server',
'module-type': ['import']}
mispattributes = {
'inputSource': [],
'output': ['MISP objects'],
'format': 'misp_standard',
}
userConfig = {
"url": {
"type": "String",
"message": "A TAXII 2.1 collection URL",
},
"added_after": {
"type": "String",
"message": "Lower bound on time the object was uploaded to the TAXII server"
},
"stix_id": {
"type": "String",
"message": "STIX ID(s) of objects"
},
"spec_version": { # TAXII 2.1 specific
"type": "String",
"message": "STIX version(s) of objects"
},
"type": {
"type": "String",
"message": "STIX type(s) of objects"
},
"version": {
"type": "String",
"message": 'Version timestamp(s), or "first"/"last"/"all"'
},
# Should we give some user control over this? It will not be allowed to
# exceed the admin setting.
"STIX object limit": {
"type": "Integer",
"message": "Maximum number of STIX objects to process"
},
"username": {
"type": "String",
"message": "Username for TAXII server authentication, if necessary"
},
"password": {
"type": "String",
"message": "Password for TAXII server authentication, if necessary"
}
}
# Paging will be handled transparently by this module, so user-defined
# paging-related filtering parameters will not be supported.
# This module will not process more than this number of STIX objects in total
# from a TAXII server in one module invocation (across all pages), to limit
# resource consumption.
moduleconfig = [
"stix_object_limit"
]
# In case there is neither an admin nor user setting given.
_DEFAULT_STIX_OBJECT_LIMIT = 1000
# Page size to use when paging TAXII results. Trades off the amount of
# hammering on TAXII servers and overhead of repeated requests, with the
# resource consumption of a single page. (Should be an admin setting too?)
_PAGE_SIZE = 100
_synonymsToTagNames_path = Path(__file__).parent / "../../lib/synonymsToTagNames.json"
# Collects module config information necessary to perform the TAXII query.
Config = collections.namedtuple("Config", [
"url",
"added_after",
"id",
"spec_version",
"type",
"version",
"stix_object_limit",
"username",
"password"
])
def _pymisp_to_json_serializable(obj):
"""
Work around a possible bug with PyMISP's
AbstractMisp.to_dict(json_format=True) method, which doesn't always produce
a JSON-serializable value (i.e. a value which is serializable with the
default JSON encoder).
:param obj: A PyMISP object
:return: A JSON-serializable version of the object
"""
# The workaround creates a JSON string and then parses it back to a
# JSON-serializable value.
json_ = obj.to_json()
json_serializable = json.loads(json_)
return json_serializable
def _normalize_multi_values(value):
"""
Some TAXII filters may contain multiple values separated by commas,
without spaces around the commas. Maybe give MISP users a little more
flexibility? This function normalizes a possible multi-valued value
(e.g. multiple values delimited by commas or spaces, all in the same
string) to TAXII-required format.
:param value: A MISP config value
:return: A normalized value
"""
if "," in value:
value = re.sub(r"\s*,\s*", ",", value)
else:
# Assume space delimiting; replace spaces with commas.
# I don't think we need to worry about spaces embedded in values.
value = re.sub(r"\s+", ",", value)
value = value.strip(",")
return value
def _get_config(config):
"""
Combine user, admin, and default config settings to produce a config
object with all settings together.
:param config: The misp-modules request's "config" value.
:return: A Config object
:raises ConfigError: if any config errors are detected
"""
# Strip whitespace from all config settings... except for password?
for key, val in config.items():
if isinstance(val, str) and key != "password":
config[key] = val.strip()
url = config.get("url")
added_after = config.get("added_after")
id_ = config.get("stix_id")
spec_version = config.get("spec_version")
type_ = config.get("type")
version_ = config.get("version")
username = config.get("username")
password = config.get("password")
admin_stix_object_limit = config.get("stix_object_limit")
user_stix_object_limit = config.get("STIX object limit")
if admin_stix_object_limit:
admin_stix_object_limit = int(admin_stix_object_limit)
else:
admin_stix_object_limit = _DEFAULT_STIX_OBJECT_LIMIT
if user_stix_object_limit:
user_stix_object_limit = int(user_stix_object_limit)
stix_object_limit = min(user_stix_object_limit, admin_stix_object_limit)
else:
stix_object_limit = admin_stix_object_limit
# How much of this should we sanity-check here before passing it off to the
# TAXII client (and thence, to the TAXII server)?
if not url:
raise ConfigError("A TAXII 2.1 collection URL is required.")
if admin_stix_object_limit < 1:
raise ConfigError(
"Invalid admin object limit: must be positive: "
+ str(admin_stix_object_limit)
)
if stix_object_limit < 1:
raise ConfigError(
"Invalid object limit: must be positive: "
+ str(stix_object_limit)
)
if id_:
id_ = _normalize_multi_values(id_)
if spec_version:
spec_version = _normalize_multi_values(spec_version)
if type_:
type_ = _normalize_multi_values(type_)
if version_:
version_ = _normalize_multi_values(version_)
# STIX->MISP converter currently only supports STIX 2.0, so let's force
# spec_version="2.0".
if not spec_version:
spec_version = "2.0"
elif spec_version != "2.0":
raise ConfigError('Only spec_version="2.0" is supported for now.')
if (username and not password) or (not username and password):
raise ConfigError(
'Both or neither of "username" and "password" are required.'
)
config_obj = Config(
url, added_after, id_, spec_version, type_, version_, stix_object_limit,
username, password
)
return config_obj
def _query_taxii(config):
"""
Query the TAXII server according to the given config, convert the STIX
results to MISP, and return a standard misp-modules response.
:param config: Module config information as a Config object
:return: A dict containing a misp-modules response
"""
collection = taxii2client.Collection(
config.url, user=config.username, password=config.password
)
# No point in asking for more than our overall limit.
page_size = min(_PAGE_SIZE, config.stix_object_limit)
kwargs = {
"per_request": page_size
}
if config.spec_version:
kwargs["spec_version"] = config.spec_version
if config.version:
kwargs["version"] = config.version
if config.id:
kwargs["id"] = config.id
if config.type:
kwargs["type"] = config.type
if config.added_after:
kwargs["added_after"] = config.added_after
pages = taxii2client.as_pages(
collection.get_objects,
**kwargs
)
# Chain all the objects from all pages together...
all_stix_objects = itertools.chain.from_iterable(
taxii_envelope.get("objects", [])
for taxii_envelope in pages
)
# And only take the first N objects from that.
limited_stix_objects = itertools.islice(
all_stix_objects, 0, config.stix_object_limit
)
# Collect into a list. This is... unfortunate, but I don't think the
# converter will work incrementally (will it?). It expects all objects to
# be given at once.
#
# It may also be desirable to have all objects available at once so that
# cross-references can be made where possible, but it results in increased
# memory usage.
stix_objects = list(limited_stix_objects)
# The STIX 2.0 converter wants a 2.0 bundle. (Hope the TAXII server isn't
# returning 2.1 objects!)
bundle20 = stix2.v20.Bundle(stix_objects, allow_custom=True)
converter = misp_modules.lib.stix2misp.ExternalStixParser()
converter.handler(
bundle20, None, [0, "event", str(_synonymsToTagNames_path)]
)
attributes = [
_pymisp_to_json_serializable(attr)
for attr in converter.misp_event.attributes
]
objects = [
_pymisp_to_json_serializable(obj)
for obj in converter.misp_event.objects
]
tags = [
_pymisp_to_json_serializable(tag)
for tag in converter.misp_event.tags
]
result = {
"results": {
"Attribute": attributes,
"Object": objects,
"Tag": tags
}
}
return result
def handler(q=False):
if q is False:
return False
request = json.loads(q)
result = None
config = None
try:
config = _get_config(request["config"])
except ConfigError as e:
result = misperrors
result["error"] = e.args[0]
if not result:
try:
result = _query_taxii(config)
except taxii2client.exceptions.TAXIIServiceException as e:
result = misperrors
result["error"] = str(e)
except requests.HTTPError as e:
# Let's give a better error message for auth issues.
if e.response.status_code in (401, 403):
result = misperrors
result["error"] = "Access was denied."
else:
raise
return result
def introspection():
mispattributes["userConfig"] = userConfig
return mispattributes
def version():
moduleinfo['config'] = moduleconfig
return moduleinfo

View File

@ -509,7 +509,7 @@ class TestExpansions(unittest.TestCase):
query = {"module": "sourcecache", "link": input_value}
response = self.misp_modules_post(query)
self.assertEqual(self.get_values(response), input_value)
self.assertTrue(self.get_data(response).startswith('PCFET0NUWVBFIEhUTUw+CjwhLS0KCUFyY2FuYSBieSBIVE1MN'))
self.assertTrue(self.get_data(response))
def test_stix2_pattern_validator(self):
query = {"module": "stix2_pattern_syntax_validator", "stix2-pattern": "[ipv4-addr:value = '8.8.8.8']"}