mirror of https://github.com/MISP/misp-modules
Merge branch 'main' of github.com:MISP/misp-modules into main
commit
8f3cc42082
|
@ -13,10 +13,11 @@ jobs:
|
||||||
strategy:
|
strategy:
|
||||||
fail-fast: false
|
fail-fast: false
|
||||||
matrix:
|
matrix:
|
||||||
python-version: ["3.7", "3.8", "3.9"]
|
python-version: ["3.7", "3.8", "3.9", "3.10"]
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- run: |
|
- name: Install packages
|
||||||
|
run: |
|
||||||
sudo apt-get install libpoppler-cpp-dev libzbar0 tesseract-ocr
|
sudo apt-get install libpoppler-cpp-dev libzbar0 tesseract-ocr
|
||||||
- uses: actions/checkout@v2
|
- uses: actions/checkout@v2
|
||||||
- name: Set up Python ${{ matrix.python-version }}
|
- name: Set up Python ${{ matrix.python-version }}
|
||||||
|
|
|
@ -140,7 +140,7 @@ sudo -u www-data /var/www/MISP/venv/bin/pip install .
|
||||||
sudo cp etc/systemd/system/misp-modules.service /etc/systemd/system/
|
sudo cp etc/systemd/system/misp-modules.service /etc/systemd/system/
|
||||||
sudo systemctl daemon-reload
|
sudo systemctl daemon-reload
|
||||||
sudo systemctl enable --now misp-modules
|
sudo systemctl enable --now misp-modules
|
||||||
/var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s & #to start the modules
|
/var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 & #to start the modules
|
||||||
~~~~
|
~~~~
|
||||||
|
|
||||||
## How to install and start MISP modules on RHEL-based distributions ?
|
## How to install and start MISP modules on RHEL-based distributions ?
|
||||||
|
@ -168,7 +168,7 @@ After=misp-workers.service
|
||||||
Type=simple
|
Type=simple
|
||||||
User=apache
|
User=apache
|
||||||
Group=apache
|
Group=apache
|
||||||
ExecStart=/usr/bin/scl enable rh-python36 rh-ruby22 '/var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s'
|
ExecStart=/usr/bin/scl enable rh-python36 rh-ruby22 '/var/www/MISP/venv/bin/misp-modules -l 127.0.0.1'
|
||||||
Restart=always
|
Restart=always
|
||||||
RestartSec=10
|
RestartSec=10
|
||||||
|
|
||||||
|
|
82
REQUIREMENTS
82
REQUIREMENTS
|
@ -9,45 +9,44 @@
|
||||||
.
|
.
|
||||||
aiohttp==3.8.1
|
aiohttp==3.8.1
|
||||||
aiosignal==1.2.0; python_version >= '3.6'
|
aiosignal==1.2.0; python_version >= '3.6'
|
||||||
antlr4-python3-runtime==4.8; python_version >= '3'
|
antlr4-python3-runtime==4.9.3; python_version >= '3'
|
||||||
apiosintds==1.8.3
|
apiosintds==1.8.3
|
||||||
appdirs==1.4.4
|
appdirs==1.4.4
|
||||||
argparse==1.4.0
|
argparse==1.4.0
|
||||||
assemblyline-client==4.2.2
|
assemblyline-client==4.3.5
|
||||||
async-timeout==4.0.2; python_version >= '3.6'
|
async-timeout==4.0.2; python_version >= '3.6'
|
||||||
asynctest==0.13.0; python_version < '3.8'
|
asynctest==0.13.0; python_version < '3.8'
|
||||||
attrs==21.4.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
|
attrs==21.4.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
|
||||||
backoff==1.11.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
|
backoff==1.11.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
|
||||||
backports.zoneinfo==0.2.1; python_version < '3.9'
|
backports.zoneinfo==0.2.1; python_version < '3.9'
|
||||||
backscatter==0.2.4
|
backscatter==0.2.4
|
||||||
beautifulsoup4==4.10.0
|
beautifulsoup4==4.11.1
|
||||||
bidict==0.21.4; python_version >= '3.6'
|
bidict==0.22.0; python_version >= '3.7'
|
||||||
blockchain==1.4.4
|
blockchain==1.4.4
|
||||||
censys==2.1.2
|
censys==2.1.3
|
||||||
certifi==2021.10.8
|
certifi==2021.10.8
|
||||||
cffi==1.15.0
|
cffi==1.15.0
|
||||||
chardet==4.0.0
|
chardet==4.0.0
|
||||||
charset-normalizer==2.0.12; python_version >= '3'
|
charset-normalizer==2.0.12; python_version >= '3'
|
||||||
clamd==1.0.2
|
clamd==1.0.2
|
||||||
click-plugins==1.1.1
|
click-plugins==1.1.1
|
||||||
click==8.0.4; python_version >= '3.6'
|
click==8.1.2; python_version >= '3.7'
|
||||||
colorama==0.4.4; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
|
colorama==0.4.4; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
|
||||||
colorclass==2.2.2; python_version >= '2.6'
|
colorclass==2.2.2; python_version >= '2.6'
|
||||||
commonmark==0.9.1
|
commonmark==0.9.1
|
||||||
compressed-rtf==1.0.6
|
compressed-rtf==1.0.6
|
||||||
configparser==5.2.0; python_version >= '3.6'
|
configparser==5.2.0; python_version >= '3.6'
|
||||||
crowdstrike-falconpy==1.0.5
|
crowdstrike-falconpy==1.0.8
|
||||||
cryptography==36.0.1; python_version >= '3.6'
|
cryptography==36.0.2; python_version >= '3.6'
|
||||||
decorator==5.1.1; python_version >= '3.5'
|
decorator==5.1.1; python_version >= '3.5'
|
||||||
deprecated==1.2.13; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
deprecated==1.2.13; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||||
dnsdb2==1.1.4
|
dnsdb2==1.1.4
|
||||||
dnspython==2.2.0
|
dnspython==2.2.1
|
||||||
dnspython3
|
domaintools-api==0.6.2
|
||||||
domaintools-api==0.6.1
|
easygui==0.98.3
|
||||||
easygui==0.98.2
|
|
||||||
ebcdic==1.1.1
|
ebcdic==1.1.1
|
||||||
enum-compat==0.0.3
|
enum-compat==0.0.3
|
||||||
extract-msg==0.30.8
|
extract-msg==0.30.10
|
||||||
ezodf==0.3.2
|
ezodf==0.3.2
|
||||||
filelock==3.6.0; python_version >= '3.7'
|
filelock==3.6.0; python_version >= '3.7'
|
||||||
frozenlist==1.3.0; python_version >= '3.7'
|
frozenlist==1.3.0; python_version >= '3.7'
|
||||||
|
@ -59,17 +58,17 @@ git+https://github.com/MISP/PyIntel471.git@917272fafa8e12102329faca52173e90c5256
|
||||||
git+https://github.com/SteveClement/trustar-python.git@6954eae38e0c77eaeef26084b6c5fd033925c1c7#egg=trustar
|
git+https://github.com/SteveClement/trustar-python.git@6954eae38e0c77eaeef26084b6c5fd033925c1c7#egg=trustar
|
||||||
git+https://github.com/cartertemm/ODTReader.git/@49d6938693f6faa3ff09998f86dba551ae3a996b#egg=odtreader
|
git+https://github.com/cartertemm/ODTReader.git/@49d6938693f6faa3ff09998f86dba551ae3a996b#egg=odtreader
|
||||||
git+https://github.com/sebdraven/pydnstrails@48c1f740025c51289f43a24863d1845ff12fd21a#egg=pydnstrails
|
git+https://github.com/sebdraven/pydnstrails@48c1f740025c51289f43a24863d1845ff12fd21a#egg=pydnstrails
|
||||||
git+https://github.com/sebdraven/pyonyphe@aed008ee5a27e3a5e4afbb3e5cbfc47170108452#egg=pyonyphe
|
git+https://github.com/sebdraven/pyonyphe@49381beb41c61c90d8997ad2d04868977411bbcc#egg=pyonyphe
|
||||||
httplib2==0.20.4; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
httplib2==0.20.4; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||||
idna-ssl==1.1.0; python_version < '3.7'
|
idna-ssl==1.1.0; python_version < '3.7'
|
||||||
idna==3.3; python_version >= '3'
|
idna==3.3; python_version >= '3'
|
||||||
imapclient==2.2.0
|
imapclient==2.2.0
|
||||||
importlib-metadata==4.11.2; python_version < '3.8'
|
importlib-metadata==4.11.3; python_version < '3.8'
|
||||||
importlib-resources==5.4.0; python_version < '3.9'
|
importlib-resources==5.6.0; python_version < '3.9'
|
||||||
isodate==0.6.1
|
isodate==0.6.1
|
||||||
itsdangerous==2.1.0; python_version >= '3.7'
|
itsdangerous==2.1.2; python_version >= '3.7'
|
||||||
jbxapi==3.17.2
|
jbxapi==3.17.2
|
||||||
jeepney==0.7.1; sys_platform == 'linux'
|
jeepney==0.8.0; sys_platform == 'linux'
|
||||||
json-log-formatter==0.5.1
|
json-log-formatter==0.5.1
|
||||||
jsonschema==4.4.0; python_version >= '3.7'
|
jsonschema==4.4.0; python_version >= '3.7'
|
||||||
keyring==23.5.0; python_version >= '3.7'
|
keyring==23.5.0; python_version >= '3.7'
|
||||||
|
@ -89,14 +88,14 @@ numpy==1.21.5; python_version < '3.10' and platform_machine != 'aarch64' and pla
|
||||||
oauth2==1.9.0.post1
|
oauth2==1.9.0.post1
|
||||||
olefile==0.46; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
olefile==0.46; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||||
oletools==0.60
|
oletools==0.60
|
||||||
opencv-python==4.5.5.62
|
opencv-python==4.5.5.64
|
||||||
packaging==21.3; python_version >= '3.6'
|
packaging==21.3; python_version >= '3.6'
|
||||||
pandas-ods-reader==0.1.2
|
pandas-ods-reader==0.1.2
|
||||||
pandas==1.3.5
|
pandas==1.3.5
|
||||||
passivetotal==2.5.8
|
passivetotal==2.5.9
|
||||||
pcodedmp==1.2.6
|
pcodedmp==1.2.6
|
||||||
pdftotext==2.2.2
|
pdftotext==2.2.2
|
||||||
pillow==9.0.1
|
pillow==9.1.0
|
||||||
progressbar2==4.0.0; python_version >= '3.7'
|
progressbar2==4.0.0; python_version >= '3.7'
|
||||||
psutil==5.9.0; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
psutil==5.9.0; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||||
pycparser==2.21
|
pycparser==2.21
|
||||||
|
@ -107,7 +106,7 @@ pyeupi==1.1
|
||||||
pyfaup==1.2
|
pyfaup==1.2
|
||||||
pygeoip==0.3.2
|
pygeoip==0.3.2
|
||||||
pygments==2.11.2; python_version >= '3.5'
|
pygments==2.11.2; python_version >= '3.5'
|
||||||
pymisp[email,fileobjects,openioc,pdfexport,url]==2.4.155.1
|
pymisp[email,fileobjects,openioc,pdfexport,url]==2.4.157
|
||||||
pyparsing==2.4.7; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
pyparsing==2.4.7; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||||
pypdns==1.5.2
|
pypdns==1.5.2
|
||||||
pypssl==2.2
|
pypssl==2.2
|
||||||
|
@ -124,51 +123,50 @@ python-utils==3.1.0; python_version >= '3.7'
|
||||||
pytz-deprecation-shim==0.1.0.post0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
|
pytz-deprecation-shim==0.1.0.post0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
|
||||||
pytz==2019.3
|
pytz==2019.3
|
||||||
pyyaml==6.0; python_version >= '3.6'
|
pyyaml==6.0; python_version >= '3.6'
|
||||||
pyzbar==0.1.8
|
pyzbar==0.1.9
|
||||||
pyzipper==0.3.5; python_version >= '3.5'
|
pyzipper==0.3.5; python_version >= '3.5'
|
||||||
rdflib==6.1.1; python_version >= '3.7'
|
rdflib==6.1.1; python_version >= '3.7'
|
||||||
redis==4.1.4; python_version >= '3.6'
|
redis==4.2.2; python_version >= '3.6'
|
||||||
reportlab==3.6.8
|
reportlab==3.6.9
|
||||||
requests-cache==0.6.4; python_version >= '3.6'
|
requests-cache==0.6.4; python_version >= '3.6'
|
||||||
requests-file==1.5.1
|
requests-file==1.5.1
|
||||||
requests==2.27.1
|
requests[security]==2.27.1
|
||||||
rich==11.2.0; python_version < '4.0' and python_full_version >= '3.6.2'
|
rich==12.2.0; python_version < '4.0' and python_full_version >= '3.6.3'
|
||||||
rtfde==0.0.2
|
rtfde==0.0.2
|
||||||
secretstorage==3.3.1; sys_platform == 'linux'
|
secretstorage==3.3.1; sys_platform == 'linux'
|
||||||
setuptools==60.9.3; python_version >= '3.7'
|
setuptools==62.1.0; python_version >= '3.7'
|
||||||
shodan==1.27.0
|
shodan==1.27.0
|
||||||
sigmatools==0.19.1
|
sigmatools==0.19.1
|
||||||
simplejson==3.17.6; python_version >= '2.5' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
simplejson==3.17.6; python_version >= '2.5' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||||
six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||||
socialscan==1.4.2
|
socialscan==1.4.2
|
||||||
socketio-client==0.5.7.4
|
socketio-client==0.5.7.4
|
||||||
soupsieve==2.3.1; python_version >= '3.6'
|
soupsieve==2.3.2; python_version >= '3.6'
|
||||||
sparqlwrapper==1.8.5
|
sparqlwrapper==2.0.0
|
||||||
stix2-patterns==1.3.2
|
stix2-patterns==2.0.0
|
||||||
stix2==3.0.1
|
stix2==3.0.1
|
||||||
tabulate==0.8.9
|
tabulate==0.8.9
|
||||||
tau-clients==0.2.1
|
tau-clients==0.2.5
|
||||||
taxii2-client==2.3.0
|
taxii2-client==2.3.0
|
||||||
tldextract==3.2.0; python_version >= '3.7'
|
tldextract==3.2.0; python_version >= '3.7'
|
||||||
tornado==6.1; python_version >= '3.5'
|
tornado==6.1; python_version >= '3.5'
|
||||||
tqdm==4.63.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
tqdm==4.64.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||||
typing-extensions==4.1.1; python_version < '3.8'
|
typing-extensions==4.1.1; python_version < '3.8'
|
||||||
tzdata==2021.5; python_version >= '3.6'
|
tzdata==2022.1; python_version >= '3.6'
|
||||||
tzlocal==4.1; python_version >= '3.6'
|
tzlocal==4.2; python_version >= '3.6'
|
||||||
unicodecsv==0.14.1
|
unicodecsv==0.14.1
|
||||||
url-normalize==1.4.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
|
url-normalize==1.4.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
|
||||||
urlarchiver==0.2
|
urlarchiver==0.2
|
||||||
urllib3==1.26.8; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4.0'
|
urllib3==1.26.9; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4.0'
|
||||||
validators==0.14.0
|
validators==0.14.0
|
||||||
vt-graph-api==1.1.3
|
vt-graph-api==2.0.0
|
||||||
vt-py==0.13.1
|
vt-py==0.14.0
|
||||||
vulners==2.0.2
|
vulners==2.0.2
|
||||||
wand==0.6.7
|
wand==0.6.7
|
||||||
websocket-client==1.3.1; python_version >= '3.6'
|
websocket-client==1.3.2; python_version >= '3.7'
|
||||||
wrapt==1.13.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
|
wrapt==1.14.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
|
||||||
xlrd==2.0.1
|
xlrd==2.0.1
|
||||||
xlsxwriter==3.0.3; python_version >= '3.4'
|
xlsxwriter==3.0.3; python_version >= '3.4'
|
||||||
yara-python==3.8.1
|
yara-python==3.8.1
|
||||||
yarl==1.7.2; python_version >= '3.6'
|
yarl==1.7.2; python_version >= '3.6'
|
||||||
zipp==3.7.0; python_version >= '3.7'
|
zipp==3.8.0; python_version >= '3.7'
|
||||||
|
|
||||||
|
|
|
@ -111,7 +111,7 @@ class JoeParser():
|
||||||
if network.get(protocol):
|
if network.get(protocol):
|
||||||
for packet in network[protocol]['packet']:
|
for packet in network[protocol]['packet']:
|
||||||
timestamp = datetime.strptime(self.parse_timestamp(packet['timestamp']), '%b %d, %Y %H:%M:%S.%f')
|
timestamp = datetime.strptime(self.parse_timestamp(packet['timestamp']), '%b %d, %Y %H:%M:%S.%f')
|
||||||
connections[tuple(packet[field] for field in network_behavior_fields)][protocol].add(timestamp)
|
connections[tuple(packet.get(field) for field in network_behavior_fields)][protocol].add(timestamp)
|
||||||
for connection, data in connections.items():
|
for connection, data in connections.items():
|
||||||
attributes = self.prefetch_attributes_data(connection)
|
attributes = self.prefetch_attributes_data(connection)
|
||||||
if len(data.keys()) == len(set(protocols[protocol] for protocol in data.keys())):
|
if len(data.keys()) == len(set(protocols[protocol] for protocol in data.keys())):
|
||||||
|
|
|
@ -21,7 +21,7 @@ def handler(q=False):
|
||||||
apiurl = request['config'].get('apiurl') or 'https://jbxcloud.joesecurity.org/api'
|
apiurl = request['config'].get('apiurl') or 'https://jbxcloud.joesecurity.org/api'
|
||||||
apikey = request['config'].get('apikey')
|
apikey = request['config'].get('apikey')
|
||||||
parser_config = {
|
parser_config = {
|
||||||
"import_pe": request["config"].get('import_executable', "false") == "true",
|
"import_executable": request["config"].get('import_executable', "false") == "true",
|
||||||
"mitre_attack": request["config"].get('import_mitre_attack', "false") == "true",
|
"mitre_attack": request["config"].get('import_mitre_attack', "false") == "true",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
import json
|
import json
|
||||||
import requests
|
|
||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
|
import vt
|
||||||
from . import check_input_attribute, standard_error_message
|
from . import check_input_attribute, standard_error_message
|
||||||
from pymisp import MISPAttribute, MISPEvent, MISPObject
|
from pymisp import MISPAttribute, MISPEvent, MISPObject
|
||||||
|
|
||||||
|
@ -9,20 +9,23 @@ mispattributes = {'input': ['hostname', 'domain', "ip-src", "ip-dst", "md5", "sh
|
||||||
'format': 'misp_standard'}
|
'format': 'misp_standard'}
|
||||||
|
|
||||||
# possible module-types: 'expansion', 'hover' or both
|
# possible module-types: 'expansion', 'hover' or both
|
||||||
moduleinfo = {'version': '4', 'author': 'Hannah Ward',
|
moduleinfo = {'version': '5', 'author': 'Hannah Ward',
|
||||||
'description': 'Get information from VirusTotal',
|
'description': 'Enrich observables with the VirusTotal v3 API',
|
||||||
'module-type': ['expansion']}
|
'module-type': ['expansion']}
|
||||||
|
|
||||||
# config fields that your code expects from the site admin
|
# config fields that your code expects from the site admin
|
||||||
moduleconfig = ["apikey", "event_limit", 'proxy_host', 'proxy_port', 'proxy_username', 'proxy_password']
|
moduleconfig = ["apikey", "event_limit", 'proxy_host', 'proxy_port', 'proxy_username', 'proxy_password']
|
||||||
|
|
||||||
|
|
||||||
class VirusTotalParser(object):
|
DEFAULT_RESULTS_LIMIT = 10
|
||||||
def __init__(self, apikey, limit):
|
|
||||||
self.apikey = apikey
|
|
||||||
self.limit = limit
|
class VirusTotalParser:
|
||||||
self.base_url = "https://www.virustotal.com/vtapi/v2/{}/report"
|
def __init__(self, client: vt.Client, limit: int) -> None:
|
||||||
|
self.client = client
|
||||||
|
self.limit = limit or DEFAULT_RESULTS_LIMIT
|
||||||
self.misp_event = MISPEvent()
|
self.misp_event = MISPEvent()
|
||||||
|
self.attribute = MISPAttribute()
|
||||||
self.parsed_objects = {}
|
self.parsed_objects = {}
|
||||||
self.input_types_mapping = {'ip-src': self.parse_ip, 'ip-dst': self.parse_ip,
|
self.input_types_mapping = {'ip-src': self.parse_ip, 'ip-dst': self.parse_ip,
|
||||||
'domain': self.parse_domain, 'hostname': self.parse_domain,
|
'domain': self.parse_domain, 'hostname': self.parse_domain,
|
||||||
|
@ -30,196 +33,187 @@ class VirusTotalParser(object):
|
||||||
'sha256': self.parse_hash, 'url': self.parse_url}
|
'sha256': self.parse_hash, 'url': self.parse_url}
|
||||||
self.proxies = None
|
self.proxies = None
|
||||||
|
|
||||||
def query_api(self, attribute):
|
@staticmethod
|
||||||
self.attribute = MISPAttribute()
|
def get_total_analysis(analysis: dict, known_distributors: dict = None) -> int:
|
||||||
self.attribute.from_dict(**attribute)
|
if not analysis:
|
||||||
return self.input_types_mapping[self.attribute.type](self.attribute.value, recurse=True)
|
return 0
|
||||||
|
count = sum([analysis['undetected'], analysis['suspicious'], analysis['harmless']])
|
||||||
|
return count if known_distributors else count + analysis['malicious']
|
||||||
|
|
||||||
def get_result(self):
|
def query_api(self, attribute: dict) -> None:
|
||||||
|
self.attribute.from_dict(**attribute)
|
||||||
|
self.input_types_mapping[self.attribute.type](self.attribute.value)
|
||||||
|
|
||||||
|
def get_result(self) -> dict:
|
||||||
event = json.loads(self.misp_event.to_json())
|
event = json.loads(self.misp_event.to_json())
|
||||||
results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])}
|
results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])}
|
||||||
return {'results': results}
|
return {'results': results}
|
||||||
|
|
||||||
|
def add_vt_report(self, report: vt.Object) -> str:
|
||||||
|
analysis = report.get('last_analysis_stats')
|
||||||
|
total = self.get_total_analysis(analysis, report.get('known_distributors'))
|
||||||
|
permalink = f'https://www.virustotal.com/gui/{report.type}/{report.id}'
|
||||||
|
|
||||||
|
vt_object = MISPObject('virustotal-report')
|
||||||
|
vt_object.add_attribute('permalink', type='link', value=permalink)
|
||||||
|
detection_ratio = f"{analysis['malicious']}/{total}" if analysis else '-/-'
|
||||||
|
vt_object.add_attribute('detection-ratio', type='text', value=detection_ratio, disable_correlation=True)
|
||||||
|
self.misp_event.add_object(**vt_object)
|
||||||
|
return vt_object.uuid
|
||||||
|
|
||||||
|
def create_misp_object(self, report: vt.Object) -> MISPObject:
|
||||||
|
misp_object = None
|
||||||
|
vt_uuid = self.add_vt_report(report)
|
||||||
|
if report.type == 'file':
|
||||||
|
misp_object = MISPObject('file')
|
||||||
|
for hash_type in ('md5', 'sha1', 'sha256'):
|
||||||
|
misp_object.add_attribute(**{'type': hash_type,
|
||||||
|
'object_relation': hash_type,
|
||||||
|
'value': report.get(hash_type)})
|
||||||
|
elif report.type == 'domain':
|
||||||
|
misp_object = MISPObject('domain-ip')
|
||||||
|
misp_object.add_attribute('domain', type='domain', value=report.id)
|
||||||
|
elif report.type == 'ip_address':
|
||||||
|
misp_object = MISPObject('domain-ip')
|
||||||
|
misp_object.add_attribute('ip', type='ip-dst', value=report.id)
|
||||||
|
elif report.type == 'url':
|
||||||
|
misp_object = MISPObject('url')
|
||||||
|
misp_object.add_attribute('url', type='url', value=report.url)
|
||||||
|
misp_object.add_reference(vt_uuid, 'analyzed-with')
|
||||||
|
return misp_object
|
||||||
|
|
||||||
################################################################################
|
################################################################################
|
||||||
#### Main parsing functions #### # noqa
|
#### Main parsing functions #### # noqa
|
||||||
################################################################################
|
################################################################################
|
||||||
|
|
||||||
def parse_domain(self, domain, recurse=False):
|
def parse_domain(self, domain: str) -> str:
|
||||||
req = requests.get(self.base_url.format('domain'), params={'apikey': self.apikey, 'domain': domain}, proxies=self.proxies)
|
domain_report = self.client.get_object(f'/domains/{domain}')
|
||||||
if req.status_code != 200:
|
|
||||||
return req.status_code
|
# DOMAIN
|
||||||
req = req.json()
|
domain_object = self.create_misp_object(domain_report)
|
||||||
hash_type = 'sha256'
|
|
||||||
whois = 'whois'
|
# WHOIS
|
||||||
feature_types = {'communicating': 'communicates-with',
|
if domain_report.whois:
|
||||||
'downloaded': 'downloaded-from',
|
whois_object = MISPObject('whois')
|
||||||
'referrer': 'referring'}
|
whois_object.add_attribute('text', type='text', value=domain_report.whois)
|
||||||
siblings = (self.parse_siblings(domain) for domain in req['domain_siblings'])
|
|
||||||
uuid = self.parse_resolutions(req['resolutions'], req['subdomains'] if 'subdomains' in req else None, siblings)
|
|
||||||
for feature_type, relationship in feature_types.items():
|
|
||||||
for feature in ('undetected_{}_samples', 'detected_{}_samples'):
|
|
||||||
for sample in req.get(feature.format(feature_type), [])[:self.limit]:
|
|
||||||
status_code = self.parse_hash(sample[hash_type], False, uuid, relationship)
|
|
||||||
if status_code != 200:
|
|
||||||
return status_code
|
|
||||||
if req.get(whois):
|
|
||||||
whois_object = MISPObject(whois)
|
|
||||||
whois_object.add_attribute('text', type='text', value=req[whois])
|
|
||||||
self.misp_event.add_object(**whois_object)
|
self.misp_event.add_object(**whois_object)
|
||||||
return self.parse_related_urls(req, recurse, uuid)
|
|
||||||
|
|
||||||
def parse_hash(self, sample, recurse=False, uuid=None, relationship=None):
|
# SIBLINGS AND SUBDOMAINS
|
||||||
req = requests.get(self.base_url.format('file'), params={'apikey': self.apikey, 'resource': sample}, proxies=self.proxies)
|
for relationship_name, misp_name in [('siblings', 'sibling-of'), ('subdomains', 'subdomain')]:
|
||||||
status_code = req.status_code
|
rel_iterator = self.client.iterator(f'/domains/{domain_report.id}/{relationship_name}', limit=self.limit)
|
||||||
if req.status_code == 200:
|
for item in rel_iterator:
|
||||||
req = req.json()
|
attr = MISPAttribute()
|
||||||
vt_uuid = self.parse_vt_object(req)
|
attr.from_dict(**dict(type='domain', value=item.id))
|
||||||
file_attributes = []
|
self.misp_event.add_attribute(**attr)
|
||||||
for hash_type in ('md5', 'sha1', 'sha256'):
|
domain_object.add_reference(attr.uuid, misp_name)
|
||||||
if req.get(hash_type):
|
|
||||||
file_attributes.append({'type': hash_type, 'object_relation': hash_type,
|
# RESOLUTIONS
|
||||||
'value': req[hash_type]})
|
resolutions_iterator = self.client.iterator(f'/domains/{domain_report.id}/resolutions', limit=self.limit)
|
||||||
if file_attributes:
|
for resolution in resolutions_iterator:
|
||||||
file_object = MISPObject('file')
|
domain_object.add_attribute('ip', type='ip-dst', value=resolution.ip_address)
|
||||||
for attribute in file_attributes:
|
|
||||||
file_object.add_attribute(**attribute)
|
# COMMUNICATING, DOWNLOADED AND REFERRER FILES
|
||||||
file_object.add_reference(vt_uuid, 'analyzed-with')
|
for relationship_name, misp_name in [
|
||||||
if uuid and relationship:
|
('communicating_files', 'communicates-with'),
|
||||||
file_object.add_reference(uuid, relationship)
|
('downloaded_files', 'downloaded-from'),
|
||||||
|
('referrer_files', 'referring')
|
||||||
|
]:
|
||||||
|
files_iterator = self.client.iterator(f'/domains/{domain_report.id}/{relationship_name}', limit=self.limit)
|
||||||
|
for file in files_iterator:
|
||||||
|
file_object = self.create_misp_object(file)
|
||||||
|
file_object.add_reference(domain_object.uuid, misp_name)
|
||||||
self.misp_event.add_object(**file_object)
|
self.misp_event.add_object(**file_object)
|
||||||
return status_code
|
|
||||||
|
|
||||||
def parse_ip(self, ip, recurse=False):
|
# URLS
|
||||||
req = requests.get(self.base_url.format('ip-address'), params={'apikey': self.apikey, 'ip': ip}, proxies=self.proxies)
|
urls_iterator = self.client.iterator(f'/domains/{domain_report.id}/urls', limit=self.limit)
|
||||||
if req.status_code != 200:
|
for url in urls_iterator:
|
||||||
return req.status_code
|
url_object = self.create_misp_object(url)
|
||||||
req = req.json()
|
url_object.add_reference(domain_object.uuid, 'hosted-in')
|
||||||
if req.get('asn'):
|
self.misp_event.add_object(**url_object)
|
||||||
asn_mapping = {'network': ('ip-src', 'subnet-announced'),
|
|
||||||
'country': ('text', 'country')}
|
|
||||||
asn_object = MISPObject('asn')
|
|
||||||
asn_object.add_attribute('asn', type='AS', value=req['asn'])
|
|
||||||
for key, value in asn_mapping.items():
|
|
||||||
if req.get(key):
|
|
||||||
attribute_type, relation = value
|
|
||||||
asn_object.add_attribute(relation, type=attribute_type, value=req[key])
|
|
||||||
self.misp_event.add_object(**asn_object)
|
|
||||||
uuid = self.parse_resolutions(req['resolutions']) if req.get('resolutions') else None
|
|
||||||
return self.parse_related_urls(req, recurse, uuid)
|
|
||||||
|
|
||||||
def parse_url(self, url, recurse=False, uuid=None):
|
self.misp_event.add_object(**domain_object)
|
||||||
req = requests.get(self.base_url.format('url'), params={'apikey': self.apikey, 'resource': url}, proxies=self.proxies)
|
return domain_object.uuid
|
||||||
status_code = req.status_code
|
|
||||||
if req.status_code == 200:
|
|
||||||
req = req.json()
|
|
||||||
vt_uuid = self.parse_vt_object(req)
|
|
||||||
if not recurse:
|
|
||||||
feature = 'url'
|
|
||||||
url_object = MISPObject(feature)
|
|
||||||
url_object.add_attribute(feature, type=feature, value=url)
|
|
||||||
url_object.add_reference(vt_uuid, 'analyzed-with')
|
|
||||||
if uuid:
|
|
||||||
url_object.add_reference(uuid, 'hosted-in')
|
|
||||||
self.misp_event.add_object(**url_object)
|
|
||||||
return status_code
|
|
||||||
|
|
||||||
################################################################################
|
def parse_hash(self, file_hash: str) -> str:
|
||||||
#### Additional parsing functions #### # noqa
|
file_report = self.client.get_object(f'files/{file_hash}')
|
||||||
################################################################################
|
file_object = self.create_misp_object(file_report)
|
||||||
|
self.misp_event.add_object(**file_object)
|
||||||
|
return file_object.uuid
|
||||||
|
|
||||||
def parse_related_urls(self, query_result, recurse, uuid=None):
|
def parse_ip(self, ip: str) -> str:
|
||||||
if recurse:
|
ip_report = self.client.get_object(f'/ip_addresses/{ip}')
|
||||||
for feature in ('detected_urls', 'undetected_urls'):
|
|
||||||
if feature in query_result:
|
# IP
|
||||||
for url in query_result[feature]:
|
ip_object = self.create_misp_object(ip_report)
|
||||||
value = url['url'] if isinstance(url, dict) else url[0]
|
|
||||||
status_code = self.parse_url(value, False, uuid)
|
# ASN
|
||||||
if status_code != 200:
|
asn_object = MISPObject('asn')
|
||||||
return status_code
|
asn_object.add_attribute('asn', type='AS', value=ip_report.asn)
|
||||||
|
asn_object.add_attribute('subnet-announced', type='ip-src', value=ip_report.network)
|
||||||
|
asn_object.add_attribute('country', type='text', value=ip_report.country)
|
||||||
|
self.misp_event.add_object(**asn_object)
|
||||||
|
|
||||||
|
# RESOLUTIONS
|
||||||
|
resolutions_iterator = self.client.iterator(f'/ip_addresses/{ip_report.id}/resolutions', limit=self.limit)
|
||||||
|
for resolution in resolutions_iterator:
|
||||||
|
ip_object.add_attribute('domain', type='domain', value=resolution.host_name)
|
||||||
|
|
||||||
|
# URLS
|
||||||
|
urls_iterator = self.client.iterator(f'/ip_addresses/{ip_report.id}/urls', limit=self.limit)
|
||||||
|
for url in urls_iterator:
|
||||||
|
url_object = self.create_misp_object(url)
|
||||||
|
url_object.add_reference(ip_object.uuid, 'hosted-in')
|
||||||
|
self.misp_event.add_object(**url_object)
|
||||||
|
|
||||||
|
self.misp_event.add_object(**ip_object)
|
||||||
|
return ip_object.uuid
|
||||||
|
|
||||||
|
def parse_url(self, url: str) -> str:
|
||||||
|
url_id = vt.url_id(url)
|
||||||
|
url_report = self.client.get_object(f'/urls/{url_id}')
|
||||||
|
url_object = self.create_misp_object(url_report)
|
||||||
|
self.misp_event.add_object(**url_object)
|
||||||
|
return url_object.uuid
|
||||||
|
|
||||||
|
|
||||||
|
def get_proxy_settings(config: dict) -> dict:
|
||||||
|
"""Returns proxy settings in the requests format.
|
||||||
|
If no proxy settings are set, return None."""
|
||||||
|
proxies = None
|
||||||
|
host = config.get('proxy_host')
|
||||||
|
port = config.get('proxy_port')
|
||||||
|
username = config.get('proxy_username')
|
||||||
|
password = config.get('proxy_password')
|
||||||
|
|
||||||
|
if host:
|
||||||
|
if not port:
|
||||||
|
misperrors['error'] = 'The virustotal_proxy_host config is set, ' \
|
||||||
|
'please also set the virustotal_proxy_port.'
|
||||||
|
raise KeyError
|
||||||
|
parsed = urlparse(host)
|
||||||
|
if 'http' in parsed.scheme:
|
||||||
|
scheme = 'http'
|
||||||
else:
|
else:
|
||||||
for feature in ('detected_urls', 'undetected_urls'):
|
scheme = parsed.scheme
|
||||||
if feature in query_result:
|
netloc = parsed.netloc
|
||||||
for url in query_result[feature]:
|
host = f'{netloc}:{port}'
|
||||||
value = url['url'] if isinstance(url, dict) else url[0]
|
|
||||||
self.misp_event.add_attribute('url', value)
|
|
||||||
return 200
|
|
||||||
|
|
||||||
def parse_resolutions(self, resolutions, subdomains=None, uuids=None):
|
if username:
|
||||||
domain_ip_object = MISPObject('domain-ip')
|
if not password:
|
||||||
if self.attribute.type in ('domain', 'hostname'):
|
misperrors['error'] = 'The virustotal_proxy_username config is set, ' \
|
||||||
domain_ip_object.add_attribute('domain', type='domain', value=self.attribute.value)
|
'please also set the virustotal_proxy_password.'
|
||||||
attribute_type, relation, key = ('ip-dst', 'ip', 'ip_address')
|
|
||||||
else:
|
|
||||||
domain_ip_object.add_attribute('ip', type='ip-dst', value=self.attribute.value)
|
|
||||||
attribute_type, relation, key = ('domain', 'domain', 'hostname')
|
|
||||||
for resolution in resolutions:
|
|
||||||
domain_ip_object.add_attribute(relation, type=attribute_type, value=resolution[key])
|
|
||||||
if subdomains:
|
|
||||||
for subdomain in subdomains:
|
|
||||||
attribute = MISPAttribute()
|
|
||||||
attribute.from_dict(**dict(type='domain', value=subdomain))
|
|
||||||
self.misp_event.add_attribute(**attribute)
|
|
||||||
domain_ip_object.add_reference(attribute.uuid, 'subdomain')
|
|
||||||
if uuids:
|
|
||||||
for uuid in uuids:
|
|
||||||
domain_ip_object.add_reference(uuid, 'sibling-of')
|
|
||||||
self.misp_event.add_object(**domain_ip_object)
|
|
||||||
return domain_ip_object.uuid
|
|
||||||
|
|
||||||
def parse_siblings(self, domain):
|
|
||||||
attribute = MISPAttribute()
|
|
||||||
attribute.from_dict(**dict(type='domain', value=domain))
|
|
||||||
self.misp_event.add_attribute(**attribute)
|
|
||||||
return attribute.uuid
|
|
||||||
|
|
||||||
def parse_vt_object(self, query_result):
|
|
||||||
if query_result['response_code'] == 1:
|
|
||||||
vt_object = MISPObject('virustotal-report')
|
|
||||||
vt_object.add_attribute('permalink', type='link', value=query_result['permalink'])
|
|
||||||
detection_ratio = '{}/{}'.format(query_result['positives'], query_result['total'])
|
|
||||||
vt_object.add_attribute('detection-ratio', type='text', value=detection_ratio, disable_correlation=True)
|
|
||||||
self.misp_event.add_object(**vt_object)
|
|
||||||
return vt_object.uuid
|
|
||||||
|
|
||||||
def set_proxy_settings(self, config: dict) -> dict:
|
|
||||||
"""Returns proxy settings in the requests format.
|
|
||||||
If no proxy settings are set, return None."""
|
|
||||||
proxies = None
|
|
||||||
host = config.get('proxy_host')
|
|
||||||
port = config.get('proxy_port')
|
|
||||||
username = config.get('proxy_username')
|
|
||||||
password = config.get('proxy_password')
|
|
||||||
|
|
||||||
if host:
|
|
||||||
if not port:
|
|
||||||
misperrors['error'] = 'The virustotal_proxy_host config is set, ' \
|
|
||||||
'please also set the virustotal_proxy_port.'
|
|
||||||
raise KeyError
|
raise KeyError
|
||||||
parsed = urlparse(host)
|
auth = f'{username}:{password}'
|
||||||
if 'http' in parsed.scheme:
|
host = auth + '@' + host
|
||||||
scheme = 'http'
|
|
||||||
else:
|
|
||||||
scheme = parsed.scheme
|
|
||||||
netloc = parsed.netloc
|
|
||||||
host = f'{netloc}:{port}'
|
|
||||||
|
|
||||||
if username:
|
proxies = {
|
||||||
if not password:
|
'http': f'{scheme}://{host}',
|
||||||
misperrors['error'] = 'The virustotal_proxy_username config is set, ' \
|
'https': f'{scheme}://{host}'
|
||||||
'please also set the virustotal_proxy_password.'
|
}
|
||||||
raise KeyError
|
return proxies
|
||||||
auth = f'{username}:{password}'
|
|
||||||
host = auth + '@' + host
|
|
||||||
|
|
||||||
proxies = {
|
|
||||||
'http': f'{scheme}://{host}',
|
|
||||||
'https': f'{scheme}://{host}'
|
|
||||||
}
|
|
||||||
self.proxies = proxies
|
|
||||||
return True
|
|
||||||
|
|
||||||
|
|
||||||
def parse_error(status_code):
|
def parse_error(status_code: int) -> str:
|
||||||
status_mapping = {204: 'VirusTotal request rate limit exceeded.',
|
status_mapping = {204: 'VirusTotal request rate limit exceeded.',
|
||||||
400: 'Incorrect request, please check the arguments.',
|
400: 'Incorrect request, please check the arguments.',
|
||||||
403: 'You don\'t have enough privileges to make the request.'}
|
403: 'You don\'t have enough privileges to make the request.'}
|
||||||
|
@ -233,7 +227,7 @@ def handler(q=False):
|
||||||
return False
|
return False
|
||||||
request = json.loads(q)
|
request = json.loads(q)
|
||||||
if not request.get('config') or not request['config'].get('apikey'):
|
if not request.get('config') or not request['config'].get('apikey'):
|
||||||
misperrors['error'] = "A VirusTotal api key is required for this module."
|
misperrors['error'] = 'A VirusTotal api key is required for this module.'
|
||||||
return misperrors
|
return misperrors
|
||||||
if not request.get('attribute') or not check_input_attribute(request['attribute']):
|
if not request.get('attribute') or not check_input_attribute(request['attribute']):
|
||||||
return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'}
|
return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'}
|
||||||
|
@ -241,15 +235,21 @@ def handler(q=False):
|
||||||
return {'error': 'Unsupported attribute type.'}
|
return {'error': 'Unsupported attribute type.'}
|
||||||
|
|
||||||
event_limit = request['config'].get('event_limit')
|
event_limit = request['config'].get('event_limit')
|
||||||
if not isinstance(event_limit, int):
|
|
||||||
event_limit = 5
|
|
||||||
parser = VirusTotalParser(request['config']['apikey'], event_limit)
|
|
||||||
parser.set_proxy_settings(request.get('config'))
|
|
||||||
attribute = request['attribute']
|
attribute = request['attribute']
|
||||||
status = parser.query_api(attribute)
|
proxy_settings = get_proxy_settings(request.get('config'))
|
||||||
if status != 200:
|
|
||||||
misperrors['error'] = parse_error(status)
|
try:
|
||||||
|
client = vt.Client(request['config']['apikey'],
|
||||||
|
headers={
|
||||||
|
'x-tool': 'MISPModuleVirusTotalExpansion',
|
||||||
|
},
|
||||||
|
proxy=proxy_settings['http'] if proxy_settings else None)
|
||||||
|
parser = VirusTotalParser(client, int(event_limit) if event_limit else None)
|
||||||
|
parser.query_api(attribute)
|
||||||
|
except vt.APIError as ex:
|
||||||
|
misperrors['error'] = ex.message
|
||||||
return misperrors
|
return misperrors
|
||||||
|
|
||||||
return parser.get_result()
|
return parser.get_result()
|
||||||
|
|
||||||
|
|
||||||
|
@ -259,4 +259,4 @@ def introspection():
|
||||||
|
|
||||||
def version():
|
def version():
|
||||||
moduleinfo['config'] = moduleconfig
|
moduleinfo['config'] = moduleconfig
|
||||||
return moduleinfo
|
return moduleinfo
|
|
@ -1,6 +1,6 @@
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
import requests
|
import vt
|
||||||
from . import check_input_attribute, standard_error_message
|
from . import check_input_attribute, standard_error_message
|
||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
from pymisp import MISPAttribute, MISPEvent, MISPObject
|
from pymisp import MISPAttribute, MISPEvent, MISPObject
|
||||||
|
@ -8,8 +8,8 @@ from pymisp import MISPAttribute, MISPEvent, MISPObject
|
||||||
misperrors = {'error': 'Error'}
|
misperrors = {'error': 'Error'}
|
||||||
mispattributes = {'input': ['hostname', 'domain', "ip-src", "ip-dst", "md5", "sha1", "sha256", "url"],
|
mispattributes = {'input': ['hostname', 'domain', "ip-src", "ip-dst", "md5", "sha1", "sha256", "url"],
|
||||||
'format': 'misp_standard'}
|
'format': 'misp_standard'}
|
||||||
moduleinfo = {'version': '1', 'author': 'Christian Studer',
|
moduleinfo = {'version': '2', 'author': 'Christian Studer',
|
||||||
'description': 'Get information from VirusTotal public API v2.',
|
'description': 'Enrich observables with the VirusTotal v3 public API',
|
||||||
'module-type': ['expansion', 'hover']}
|
'module-type': ['expansion', 'hover']}
|
||||||
|
|
||||||
moduleconfig = ['apikey', 'proxy_host', 'proxy_port', 'proxy_username', 'proxy_password']
|
moduleconfig = ['apikey', 'proxy_host', 'proxy_port', 'proxy_username', 'proxy_password']
|
||||||
|
@ -18,191 +18,188 @@ LOGGER = logging.getLogger('virus_total_public')
|
||||||
LOGGER.setLevel(logging.INFO)
|
LOGGER.setLevel(logging.INFO)
|
||||||
|
|
||||||
|
|
||||||
class VirusTotalParser():
|
DEFAULT_RESULTS_LIMIT = 10
|
||||||
def __init__(self):
|
|
||||||
super(VirusTotalParser, self).__init__()
|
|
||||||
|
class VirusTotalParser:
|
||||||
|
def __init__(self, client: vt.Client, limit: int) -> None:
|
||||||
|
self.client = client
|
||||||
|
self.limit = limit or DEFAULT_RESULTS_LIMIT
|
||||||
self.misp_event = MISPEvent()
|
self.misp_event = MISPEvent()
|
||||||
|
self.attribute = MISPAttribute()
|
||||||
|
self.parsed_objects = {}
|
||||||
|
self.input_types_mapping = {'ip-src': self.parse_ip, 'ip-dst': self.parse_ip,
|
||||||
|
'domain': self.parse_domain, 'hostname': self.parse_domain,
|
||||||
|
'md5': self.parse_hash, 'sha1': self.parse_hash,
|
||||||
|
'sha256': self.parse_hash, 'url': self.parse_url}
|
||||||
self.proxies = None
|
self.proxies = None
|
||||||
|
|
||||||
def declare_variables(self, apikey, attribute):
|
@staticmethod
|
||||||
self.attribute = MISPAttribute()
|
def get_total_analysis(analysis: dict, known_distributors: dict = None) -> int:
|
||||||
self.attribute.from_dict(**attribute)
|
if not analysis:
|
||||||
self.apikey = apikey
|
return 0
|
||||||
|
count = sum([analysis['undetected'], analysis['suspicious'], analysis['harmless']])
|
||||||
|
return count if known_distributors else count + analysis['malicious']
|
||||||
|
|
||||||
def get_result(self):
|
def query_api(self, attribute: dict) -> None:
|
||||||
|
self.attribute.from_dict(**attribute)
|
||||||
|
self.input_types_mapping[self.attribute.type](self.attribute.value)
|
||||||
|
|
||||||
|
def get_result(self) -> dict:
|
||||||
event = json.loads(self.misp_event.to_json())
|
event = json.loads(self.misp_event.to_json())
|
||||||
results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])}
|
results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])}
|
||||||
return {'results': results}
|
return {'results': results}
|
||||||
|
|
||||||
def parse_urls(self, query_result):
|
def add_vt_report(self, report: vt.Object) -> str:
|
||||||
for feature in ('detected_urls', 'undetected_urls'):
|
analysis = report.get('last_analysis_stats')
|
||||||
if feature in query_result:
|
total = self.get_total_analysis(analysis, report.get('known_distributors'))
|
||||||
for url in query_result[feature]:
|
permalink = f'https://www.virustotal.com/gui/{report.type}/{report.id}'
|
||||||
value = url['url'] if isinstance(url, dict) else url[0]
|
|
||||||
self.misp_event.add_attribute('url', value)
|
|
||||||
|
|
||||||
def parse_resolutions(self, resolutions, subdomains=None, uuids=None):
|
vt_object = MISPObject('virustotal-report')
|
||||||
domain_ip_object = MISPObject('domain-ip')
|
vt_object.add_attribute('permalink', type='link', value=permalink)
|
||||||
if self.attribute.type in ('domain', 'hostname'):
|
detection_ratio = f"{analysis['malicious']}/{total}" if analysis else '-/-'
|
||||||
domain_ip_object.add_attribute('domain', type='domain', value=self.attribute.value)
|
vt_object.add_attribute('detection-ratio', type='text', value=detection_ratio, disable_correlation=True)
|
||||||
attribute_type, relation, key = ('ip-dst', 'ip', 'ip_address')
|
self.misp_event.add_object(**vt_object)
|
||||||
else:
|
return vt_object.uuid
|
||||||
domain_ip_object.add_attribute('ip', type='ip-dst', value=self.attribute.value)
|
|
||||||
attribute_type, relation, key = ('domain', 'domain', 'hostname')
|
|
||||||
for resolution in resolutions:
|
|
||||||
domain_ip_object.add_attribute(relation, type=attribute_type, value=resolution[key])
|
|
||||||
if subdomains:
|
|
||||||
for subdomain in subdomains:
|
|
||||||
attribute = MISPAttribute()
|
|
||||||
attribute.from_dict(**dict(type='domain', value=subdomain))
|
|
||||||
self.misp_event.add_attribute(**attribute)
|
|
||||||
domain_ip_object.add_reference(attribute.uuid, 'subdomain')
|
|
||||||
if uuids:
|
|
||||||
for uuid in uuids:
|
|
||||||
domain_ip_object.add_reference(uuid, 'sibling-of')
|
|
||||||
self.misp_event.add_object(**domain_ip_object)
|
|
||||||
|
|
||||||
def parse_vt_object(self, query_result):
|
def create_misp_object(self, report: vt.Object) -> MISPObject:
|
||||||
if query_result['response_code'] == 1:
|
misp_object = None
|
||||||
vt_object = MISPObject('virustotal-report')
|
vt_uuid = self.add_vt_report(report)
|
||||||
vt_object.add_attribute('permalink', type='link', value=query_result['permalink'])
|
if report.type == 'file':
|
||||||
detection_ratio = '{}/{}'.format(query_result['positives'], query_result['total'])
|
misp_object = MISPObject('file')
|
||||||
vt_object.add_attribute('detection-ratio', type='text', value=detection_ratio)
|
for hash_type in ('md5', 'sha1', 'sha256'):
|
||||||
self.misp_event.add_object(**vt_object)
|
misp_object.add_attribute(**{'type': hash_type,
|
||||||
|
'object_relation': hash_type,
|
||||||
|
'value': report.get(hash_type)})
|
||||||
|
elif report.type == 'domain':
|
||||||
|
misp_object = MISPObject('domain-ip')
|
||||||
|
misp_object.add_attribute('domain', type='domain', value=report.id)
|
||||||
|
elif report.type == 'ip_address':
|
||||||
|
misp_object = MISPObject('domain-ip')
|
||||||
|
misp_object.add_attribute('ip', type='ip-dst', value=report.id)
|
||||||
|
elif report.type == 'url':
|
||||||
|
misp_object = MISPObject('url')
|
||||||
|
misp_object.add_attribute('url', type='url', value=report.url)
|
||||||
|
misp_object.add_reference(vt_uuid, 'analyzed-with')
|
||||||
|
return misp_object
|
||||||
|
|
||||||
def get_query_result(self, query_type):
|
################################################################################
|
||||||
params = {query_type: self.attribute.value, 'apikey': self.apikey}
|
#### Main parsing functions #### # noqa
|
||||||
return requests.get(self.base_url, params=params, proxies=self.proxies)
|
################################################################################
|
||||||
|
|
||||||
def set_proxy_settings(self, config: dict) -> dict:
|
def parse_domain(self, domain: str) -> str:
|
||||||
"""Returns proxy settings in the requests format.
|
domain_report = self.client.get_object(f'/domains/{domain}')
|
||||||
If no proxy settings are set, return None."""
|
|
||||||
proxies = None
|
|
||||||
host = config.get('proxy_host')
|
|
||||||
port = config.get('proxy_port')
|
|
||||||
username = config.get('proxy_username')
|
|
||||||
password = config.get('proxy_password')
|
|
||||||
|
|
||||||
if host:
|
# DOMAIN
|
||||||
if not port:
|
domain_object = self.create_misp_object(domain_report)
|
||||||
misperrors['error'] = 'The virustotal_public_proxy_host config is set, ' \
|
|
||||||
'please also set the virustotal_public_proxy_port.'
|
|
||||||
raise KeyError
|
|
||||||
parsed = urlparse(host)
|
|
||||||
if 'http' in parsed.scheme:
|
|
||||||
scheme = 'http'
|
|
||||||
else:
|
|
||||||
scheme = parsed.scheme
|
|
||||||
netloc = parsed.netloc
|
|
||||||
host = f'{netloc}:{port}'
|
|
||||||
|
|
||||||
if username:
|
# WHOIS
|
||||||
if not password:
|
if domain_report.whois:
|
||||||
misperrors['error'] = 'The virustotal_public_proxy_username config is set, ' \
|
whois_object = MISPObject('whois')
|
||||||
'please also set the virustotal_public_proxy_password.'
|
whois_object.add_attribute('text', type='text', value=domain_report.whois)
|
||||||
raise KeyError
|
|
||||||
auth = f'{username}:{password}'
|
|
||||||
host = auth + '@' + host
|
|
||||||
|
|
||||||
proxies = {
|
|
||||||
'http': f'{scheme}://{host}',
|
|
||||||
'https': f'{scheme}://{host}'
|
|
||||||
}
|
|
||||||
self.proxies = proxies
|
|
||||||
return True
|
|
||||||
|
|
||||||
|
|
||||||
class DomainQuery(VirusTotalParser):
|
|
||||||
def __init__(self, apikey, attribute):
|
|
||||||
super(DomainQuery, self).__init__()
|
|
||||||
self.base_url = "https://www.virustotal.com/vtapi/v2/domain/report"
|
|
||||||
self.declare_variables(apikey, attribute)
|
|
||||||
|
|
||||||
def parse_report(self, query_result):
|
|
||||||
hash_type = 'sha256'
|
|
||||||
whois = 'whois'
|
|
||||||
for feature_type in ('referrer', 'downloaded', 'communicating'):
|
|
||||||
for feature in ('undetected_{}_samples', 'detected_{}_samples'):
|
|
||||||
for sample in query_result.get(feature.format(feature_type), []):
|
|
||||||
self.misp_event.add_attribute(hash_type, sample[hash_type])
|
|
||||||
if query_result.get(whois):
|
|
||||||
whois_object = MISPObject(whois)
|
|
||||||
whois_object.add_attribute('text', type='text', value=query_result[whois])
|
|
||||||
self.misp_event.add_object(**whois_object)
|
self.misp_event.add_object(**whois_object)
|
||||||
if 'domain_siblings' in query_result:
|
|
||||||
siblings = (self.parse_siblings(domain) for domain in query_result['domain_siblings'])
|
|
||||||
if 'subdomains' in query_result:
|
|
||||||
self.parse_resolutions(query_result['resolutions'], query_result['subdomains'], siblings)
|
|
||||||
self.parse_urls(query_result)
|
|
||||||
|
|
||||||
def parse_siblings(self, domain):
|
# SIBLINGS AND SUBDOMAINS
|
||||||
attribute = MISPAttribute()
|
for relationship_name, misp_name in [('siblings', 'sibling-of'), ('subdomains', 'subdomain')]:
|
||||||
attribute.from_dict(**dict(type='domain', value=domain))
|
rel_iterator = self.client.iterator(f'/domains/{domain_report.id}/{relationship_name}', limit=self.limit)
|
||||||
self.misp_event.add_attribute(**attribute)
|
for item in rel_iterator:
|
||||||
return attribute.uuid
|
attr = MISPAttribute()
|
||||||
|
attr.from_dict(**dict(type='domain', value=item.id))
|
||||||
|
self.misp_event.add_attribute(**attr)
|
||||||
|
domain_object.add_reference(attr.uuid, misp_name)
|
||||||
|
|
||||||
|
# RESOLUTIONS
|
||||||
|
resolutions_iterator = self.client.iterator(f'/domains/{domain_report.id}/resolutions', limit=self.limit)
|
||||||
|
for resolution in resolutions_iterator:
|
||||||
|
domain_object.add_attribute('ip', type='ip-dst', value=resolution.ip_address)
|
||||||
|
|
||||||
|
# COMMUNICATING AND REFERRER FILES
|
||||||
|
for relationship_name, misp_name in [
|
||||||
|
('communicating_files', 'communicates-with'),
|
||||||
|
('referrer_files', 'referring')
|
||||||
|
]:
|
||||||
|
files_iterator = self.client.iterator(f'/domains/{domain_report.id}/{relationship_name}', limit=self.limit)
|
||||||
|
for file in files_iterator:
|
||||||
|
file_object = self.create_misp_object(file)
|
||||||
|
file_object.add_reference(domain_object.uuid, misp_name)
|
||||||
|
self.misp_event.add_object(**file_object)
|
||||||
|
|
||||||
|
self.misp_event.add_object(**domain_object)
|
||||||
|
return domain_object.uuid
|
||||||
|
|
||||||
|
def parse_hash(self, file_hash: str) -> str:
|
||||||
|
file_report = self.client.get_object(f'files/{file_hash}')
|
||||||
|
file_object = self.create_misp_object(file_report)
|
||||||
|
self.misp_event.add_object(**file_object)
|
||||||
|
return file_object.uuid
|
||||||
|
|
||||||
|
def parse_ip(self, ip: str) -> str:
|
||||||
|
ip_report = self.client.get_object(f'/ip_addresses/{ip}')
|
||||||
|
|
||||||
|
# IP
|
||||||
|
ip_object = self.create_misp_object(ip_report)
|
||||||
|
|
||||||
|
# ASN
|
||||||
|
asn_object = MISPObject('asn')
|
||||||
|
asn_object.add_attribute('asn', type='AS', value=ip_report.asn)
|
||||||
|
asn_object.add_attribute('subnet-announced', type='ip-src', value=ip_report.network)
|
||||||
|
asn_object.add_attribute('country', type='text', value=ip_report.country)
|
||||||
|
self.misp_event.add_object(**asn_object)
|
||||||
|
|
||||||
|
# RESOLUTIONS
|
||||||
|
resolutions_iterator = self.client.iterator(f'/ip_addresses/{ip_report.id}/resolutions', limit=self.limit)
|
||||||
|
for resolution in resolutions_iterator:
|
||||||
|
ip_object.add_attribute('domain', type='domain', value=resolution.host_name)
|
||||||
|
|
||||||
|
self.misp_event.add_object(**ip_object)
|
||||||
|
return ip_object.uuid
|
||||||
|
|
||||||
|
def parse_url(self, url: str) -> str:
|
||||||
|
url_id = vt.url_id(url)
|
||||||
|
url_report = self.client.get_object(f'/urls/{url_id}')
|
||||||
|
url_object = self.create_misp_object(url_report)
|
||||||
|
self.misp_event.add_object(**url_object)
|
||||||
|
return url_object.uuid
|
||||||
|
|
||||||
|
|
||||||
class HashQuery(VirusTotalParser):
|
def get_proxy_settings(config: dict) -> dict:
|
||||||
def __init__(self, apikey, attribute):
|
"""Returns proxy settings in the requests format.
|
||||||
super(HashQuery, self).__init__()
|
If no proxy settings are set, return None."""
|
||||||
self.base_url = "https://www.virustotal.com/vtapi/v2/file/report"
|
proxies = None
|
||||||
self.declare_variables(apikey, attribute)
|
host = config.get('proxy_host')
|
||||||
|
port = config.get('proxy_port')
|
||||||
|
username = config.get('proxy_username')
|
||||||
|
password = config.get('proxy_password')
|
||||||
|
|
||||||
def parse_report(self, query_result):
|
if host:
|
||||||
file_attributes = []
|
if not port:
|
||||||
for hash_type in ('md5', 'sha1', 'sha256'):
|
misperrors['error'] = 'The virustotal_proxy_host config is set, ' \
|
||||||
if query_result.get(hash_type):
|
'please also set the virustotal_proxy_port.'
|
||||||
file_attributes.append({'type': hash_type, 'object_relation': hash_type,
|
raise KeyError
|
||||||
'value': query_result[hash_type]})
|
parsed = urlparse(host)
|
||||||
if file_attributes:
|
if 'http' in parsed.scheme:
|
||||||
file_object = MISPObject('file')
|
scheme = 'http'
|
||||||
for attribute in file_attributes:
|
else:
|
||||||
file_object.add_attribute(**attribute)
|
scheme = parsed.scheme
|
||||||
self.misp_event.add_object(**file_object)
|
netloc = parsed.netloc
|
||||||
self.parse_vt_object(query_result)
|
host = f'{netloc}:{port}'
|
||||||
|
|
||||||
|
if username:
|
||||||
|
if not password:
|
||||||
|
misperrors['error'] = 'The virustotal_proxy_username config is set, ' \
|
||||||
|
'please also set the virustotal_proxy_password.'
|
||||||
|
raise KeyError
|
||||||
|
auth = f'{username}:{password}'
|
||||||
|
host = auth + '@' + host
|
||||||
|
|
||||||
|
proxies = {
|
||||||
|
'http': f'{scheme}://{host}',
|
||||||
|
'https': f'{scheme}://{host}'
|
||||||
|
}
|
||||||
|
return proxies
|
||||||
|
|
||||||
|
|
||||||
class IpQuery(VirusTotalParser):
|
def parse_error(status_code: int) -> str:
|
||||||
def __init__(self, apikey, attribute):
|
|
||||||
super(IpQuery, self).__init__()
|
|
||||||
self.base_url = "https://www.virustotal.com/vtapi/v2/ip-address/report"
|
|
||||||
self.declare_variables(apikey, attribute)
|
|
||||||
|
|
||||||
def parse_report(self, query_result):
|
|
||||||
if query_result.get('asn'):
|
|
||||||
asn_mapping = {'network': ('ip-src', 'subnet-announced'),
|
|
||||||
'country': ('text', 'country')}
|
|
||||||
asn_object = MISPObject('asn')
|
|
||||||
asn_object.add_attribute('asn', type='AS', value=query_result['asn'])
|
|
||||||
for key, value in asn_mapping.items():
|
|
||||||
if query_result.get(key):
|
|
||||||
attribute_type, relation = value
|
|
||||||
asn_object.add_attribute(relation, type=attribute_type, value=query_result[key])
|
|
||||||
self.misp_event.add_object(**asn_object)
|
|
||||||
self.parse_urls(query_result)
|
|
||||||
if query_result.get('resolutions'):
|
|
||||||
self.parse_resolutions(query_result['resolutions'])
|
|
||||||
|
|
||||||
|
|
||||||
class UrlQuery(VirusTotalParser):
|
|
||||||
def __init__(self, apikey, attribute):
|
|
||||||
super(UrlQuery, self).__init__()
|
|
||||||
self.base_url = "https://www.virustotal.com/vtapi/v2/url/report"
|
|
||||||
self.declare_variables(apikey, attribute)
|
|
||||||
|
|
||||||
def parse_report(self, query_result):
|
|
||||||
self.parse_vt_object(query_result)
|
|
||||||
|
|
||||||
|
|
||||||
domain = ('domain', DomainQuery)
|
|
||||||
ip = ('ip', IpQuery)
|
|
||||||
file = ('resource', HashQuery)
|
|
||||||
misp_type_mapping = {'domain': domain, 'hostname': domain, 'ip-src': ip,
|
|
||||||
'ip-dst': ip, 'md5': file, 'sha1': file, 'sha256': file,
|
|
||||||
'url': ('resource', UrlQuery)}
|
|
||||||
|
|
||||||
|
|
||||||
def parse_error(status_code):
|
|
||||||
status_mapping = {204: 'VirusTotal request rate limit exceeded.',
|
status_mapping = {204: 'VirusTotal request rate limit exceeded.',
|
||||||
400: 'Incorrect request, please check the arguments.',
|
400: 'Incorrect request, please check the arguments.',
|
||||||
403: 'You don\'t have enough privileges to make the request.'}
|
403: 'You don\'t have enough privileges to make the request.'}
|
||||||
|
@ -216,23 +213,29 @@ def handler(q=False):
|
||||||
return False
|
return False
|
||||||
request = json.loads(q)
|
request = json.loads(q)
|
||||||
if not request.get('config') or not request['config'].get('apikey'):
|
if not request.get('config') or not request['config'].get('apikey'):
|
||||||
misperrors['error'] = "A VirusTotal api key is required for this module."
|
misperrors['error'] = 'A VirusTotal api key is required for this module.'
|
||||||
return misperrors
|
return misperrors
|
||||||
if not request.get('attribute') or not check_input_attribute(request['attribute']):
|
if not request.get('attribute') or not check_input_attribute(request['attribute']):
|
||||||
return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'}
|
return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'}
|
||||||
attribute = request['attribute']
|
if request['attribute']['type'] not in mispattributes['input']:
|
||||||
if attribute['type'] not in mispattributes['input']:
|
|
||||||
return {'error': 'Unsupported attribute type.'}
|
return {'error': 'Unsupported attribute type.'}
|
||||||
query_type, to_call = misp_type_mapping[attribute['type']]
|
|
||||||
parser = to_call(request['config']['apikey'], attribute)
|
event_limit = request['config'].get('event_limit')
|
||||||
parser.set_proxy_settings(request.get('config'))
|
attribute = request['attribute']
|
||||||
query_result = parser.get_query_result(query_type)
|
proxy_settings = get_proxy_settings(request.get('config'))
|
||||||
status_code = query_result.status_code
|
|
||||||
if status_code == 200:
|
try:
|
||||||
parser.parse_report(query_result.json())
|
client = vt.Client(request['config']['apikey'],
|
||||||
else:
|
headers={
|
||||||
misperrors['error'] = parse_error(status_code)
|
'x-tool': 'MISPModuleVirusTotalPublicExpansion',
|
||||||
|
},
|
||||||
|
proxy=proxy_settings['http'] if proxy_settings else None)
|
||||||
|
parser = VirusTotalParser(client, int(event_limit) if event_limit else None)
|
||||||
|
parser.query_api(attribute)
|
||||||
|
except vt.APIError as ex:
|
||||||
|
misperrors['error'] = ex.message
|
||||||
return misperrors
|
return misperrors
|
||||||
|
|
||||||
return parser.get_result()
|
return parser.get_result()
|
||||||
|
|
||||||
|
|
||||||
|
@ -242,4 +245,4 @@ def introspection():
|
||||||
|
|
||||||
def version():
|
def version():
|
||||||
moduleinfo['config'] = moduleconfig
|
moduleinfo['config'] = moduleconfig
|
||||||
return moduleinfo
|
return moduleinfo
|
Loading…
Reference in New Issue