mirror of https://github.com/MISP/misp-modules
Merge pull request #663 from VirusTotal/feat/google-threat-intel
feat: Google Threat Intelligence expansion modulepull/664/head
GitHub
commit
8e3deb85a9
|
|
@ -38,6 +38,7 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/
|
|||
* [EQL](misp_modules/modules/expansion/eql.py) - an expansion module to generate event query language (EQL) from an attribute. [Event Query Language](https://eql.readthedocs.io/en/latest/)
|
||||
* [Farsight DNSDB Passive DNS](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/farsight_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information.
|
||||
* [GeoIP](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/geoip_country.py) - a hover and expansion module to get GeoIP information from geolite/maxmind.
|
||||
* [Google Threat Intelligence] (https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/google_threat_intelligence.py) - An expansion module to have the observable's threat score assessed by Google Threat Intelligence.
|
||||
* [Greynoise](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/greynoise.py) - a hover to get information from greynoise.
|
||||
* [hashdd](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/hashdd.py) - a hover module to check file hashes against [hashdd.com](http://www.hashdd.com) including NSLR dataset.
|
||||
* [hibp](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/hibp.py) - a hover module to lookup against Have I Been Pwned?
|
||||
|
|
|
|||
Binary file not shown.
|
After Width: | Height: | Size: 4.6 KiB |
|
|
@ -640,6 +640,7 @@ Module to query a local copy of Maxmind's Geolite database.
|
|||
#### [google_search](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/google_search.py)
|
||||
|
||||
<img src=logos/google.png height=60>
|
||||
|
||||
- **descrption**:
|
||||
>A hover module to get information about an url using a Google search.
|
||||
- **features**:
|
||||
|
|
@ -655,6 +656,27 @@ Module to query a local copy of Maxmind's Geolite database.
|
|||
|
||||
-----
|
||||
|
||||
#### [google_threat_intelligence](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/google_threat_intelligence.py)
|
||||
|
||||
<img src=logos/google_threat_intelligence.png height=60>
|
||||
|
||||
- **description**:
|
||||
An expansion module to have the observable's threat score assessed by Google Threat Intelligence.
|
||||
- **features**:
|
||||
>The module gives the Google Threat Intelligence assessment including a verdict for the given obsevable. [Example screeshot](https://github.com/MISP/MISP/assets/4747608/e275db2f-bb1e-4413-8cc0-ec3cb05e0414)
|
||||
]
|
||||
- **input**:
|
||||
>'hostname', 'domain', 'ip-src', 'ip-dst', 'md5', 'sha1', 'sha256', 'url'.
|
||||
- **output**:
|
||||
>Text fields containing the threat score, the severity, the verdict and the threat label of the observable inspected.
|
||||
- **references**:
|
||||
>https://gtidocs.virustotal.com/reference
|
||||
- **requirements**:
|
||||
>- pymisp
|
||||
>- vt
|
||||
|
||||
-----
|
||||
|
||||
#### [greynoise](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/greynoise.py)
|
||||
|
||||
<img src=logos/greynoise.png height=60>
|
||||
|
|
|
|||
|
|
@ -20,7 +20,8 @@ __all__ = ['cuckoo_submit', 'vmray_submit', 'bgpranking', 'circl_passivedns', 'c
|
|||
'trustar_enrich', 'recordedfuture', 'html_to_markdown', 'socialscan', 'passive-ssh',
|
||||
'qintel_qsentry', 'mwdb', 'hashlookup', 'mmdb_lookup', 'ipqs_fraud_and_risk_scoring',
|
||||
'clamav', 'jinja_template_rendering','hyasinsight', 'variotdbs', 'crowdsec',
|
||||
'extract_url_components', 'ipinfo', 'whoisfreaks', 'ip2locationio', 'vysion', 'stairwell']
|
||||
'extract_url_components', 'ipinfo', 'whoisfreaks', 'ip2locationio', 'vysion', 'stairwell',
|
||||
'google_threat_intelligence']
|
||||
|
||||
|
||||
minimum_required_fields = ('type', 'uuid', 'value')
|
||||
|
|
|
|||
|
|
@ -0,0 +1,324 @@
|
|||
#!/usr/local/bin/python
|
||||
# Copyright © 2024 The Google Threat Intelligence authors. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Google Threat Intelligence MISP expansion module."""
|
||||
|
||||
from urllib import parse
|
||||
import vt
|
||||
import pymisp
|
||||
|
||||
|
||||
MISP_ATTRIBUTES = {
|
||||
'input': [
|
||||
'hostname',
|
||||
'domain',
|
||||
'ip-src',
|
||||
'ip-dst',
|
||||
'md5',
|
||||
'sha1',
|
||||
'sha256',
|
||||
'url',
|
||||
],
|
||||
'format': 'misp_standard',
|
||||
}
|
||||
|
||||
MODULE_INFO = {
|
||||
'version': '1',
|
||||
'author': 'Google Threat Intelligence team',
|
||||
'description': ('An expansion module to have the observable\'s threat'
|
||||
' score assessed by Google Threat Intelligence.'),
|
||||
'module-type': ['expansion'],
|
||||
'config': [
|
||||
'apikey',
|
||||
'event_limit',
|
||||
'proxy_host',
|
||||
'proxy_port',
|
||||
'proxy_username',
|
||||
'proxy_password'
|
||||
]
|
||||
}
|
||||
|
||||
DEFAULT_RESULTS_LIMIT = 10
|
||||
|
||||
|
||||
|
||||
|
||||
class GoogleThreatIntelligenceParser:
|
||||
"""Main parser class to create the MISP event."""
|
||||
def __init__(self, client: vt.Client, limit: int) -> None:
|
||||
self.client = client
|
||||
self.limit = limit or DEFAULT_RESULTS_LIMIT
|
||||
self.misp_event = pymisp.MISPEvent()
|
||||
self.attribute = pymisp.MISPAttribute()
|
||||
self.parsed_objects = {}
|
||||
self.input_types_mapping = {
|
||||
'ip-src': self.parse_ip,
|
||||
'ip-dst': self.parse_ip,
|
||||
'domain': self.parse_domain,
|
||||
'hostname': self.parse_domain,
|
||||
'md5': self.parse_hash,
|
||||
'sha1': self.parse_hash,
|
||||
'sha256': self.parse_hash,
|
||||
'url': self.parse_url
|
||||
}
|
||||
self.proxies = None
|
||||
|
||||
def query_api(self, attribute: dict) -> None:
|
||||
"""Get data from the API and parse it."""
|
||||
self.attribute.from_dict(**attribute)
|
||||
self.input_types_mapping[self.attribute.type](self.attribute.value)
|
||||
|
||||
def get_results(self) -> dict:
|
||||
"""Serialize the MISP event."""
|
||||
event = self.misp_event.to_dict()
|
||||
results = {
|
||||
key: event[key] for key in ('Attribute', 'Object') \
|
||||
if (key in event and event[key])
|
||||
}
|
||||
return {'results': results}
|
||||
|
||||
def create_gti_report_object(self, report):
|
||||
"""Create GTI report object."""
|
||||
report = report.to_dict()
|
||||
permalink = ('https://www.virustotal.com/gui/'
|
||||
f"{report['type']}/{report['id']}")
|
||||
report_object = pymisp.MISPObject('Google-Threat-Intel-report')
|
||||
report_object.add_attribute('permalink', type='link', value=permalink)
|
||||
report_object.add_attribute(
|
||||
'Threat Score', type='text',
|
||||
value=get_key(
|
||||
report, 'attributes.gti_assessment.threat_score.value'))
|
||||
report_object.add_attribute(
|
||||
'Verdict', type='text',
|
||||
value=get_key(
|
||||
report, 'attributes.gti_assessment.verdict.value').replace(
|
||||
'VERDICT_', ''))
|
||||
report_object.add_attribute(
|
||||
'Severity', type='text',
|
||||
value=get_key(
|
||||
report, 'attributes.gti_assessment.severity.value').replace(
|
||||
'SEVERITY_', ''))
|
||||
report_object.add_attribute(
|
||||
'Threat Label', type='text',
|
||||
value=get_key(
|
||||
report, ('attributes.popular_threat_classification'
|
||||
'.suggested_threat_label')))
|
||||
self.misp_event.add_object(**report_object)
|
||||
return report_object.uuid
|
||||
|
||||
def parse_domain(self, domain: str) -> str:
|
||||
"""Create domain MISP object."""
|
||||
domain_report = self.client.get_object(f'/domains/{domain}')
|
||||
|
||||
# DOMAIN
|
||||
domain_object = pymisp.MISPObject('domain-ip')
|
||||
domain_object.add_attribute(
|
||||
'domain', type='domain', value=domain_report.id)
|
||||
|
||||
report_uuid = self.create_gti_report_object(domain_report)
|
||||
domain_object.add_reference(report_uuid, 'analyzed-with')
|
||||
self.misp_event.add_object(**domain_object)
|
||||
return domain_object.uuid
|
||||
|
||||
def parse_hash(self, file_hash: str) -> str:
|
||||
"""Create hash MISP object."""
|
||||
file_report = self.client.get_object(f'/files/{file_hash}')
|
||||
file_object = pymisp.MISPObject('file')
|
||||
for hash_type in ('md5', 'sha1', 'sha256'):
|
||||
file_object.add_attribute(
|
||||
hash_type,
|
||||
**{'type': hash_type, 'value': file_report.get(hash_type)})
|
||||
|
||||
report_uuid = self.create_gti_report_object(file_report)
|
||||
file_object.add_reference(report_uuid, 'analyzed-with')
|
||||
self.misp_event.add_object(**file_object)
|
||||
return file_object.uuid
|
||||
|
||||
def parse_ip(self, ip: str) -> str:
|
||||
"""Create ip MISP object."""
|
||||
ip_report = self.client.get_object(f'/ip_addresses/{ip}')
|
||||
|
||||
# IP
|
||||
ip_object = pymisp.MISPObject('domain-ip')
|
||||
ip_object.add_attribute('ip', type='ip-dst', value=ip_report.id)
|
||||
|
||||
report_uuid = self.create_gti_report_object(ip_report)
|
||||
ip_object.add_reference(report_uuid, 'analyzed-with')
|
||||
self.misp_event.add_object(**ip_object)
|
||||
return ip_object.uuid
|
||||
|
||||
def parse_url(self, url: str) -> str:
|
||||
"""Create URL MISP object."""
|
||||
url_id = vt.url_id(url)
|
||||
url_report = self.client.get_object(f'/urls/{url_id}')
|
||||
|
||||
url_object = pymisp.MISPObject('url')
|
||||
url_object.add_attribute('url', type='url', value=url_report.url)
|
||||
|
||||
report_uuid = self.create_gti_report_object(url_report)
|
||||
url_object.add_reference(report_uuid, 'analyzed-with')
|
||||
self.misp_event.add_object(**url_object)
|
||||
return url_object.uuid
|
||||
|
||||
|
||||
def get_key(dictionary, key, default_value=''):
|
||||
"""Get value from nested dictionaries."""
|
||||
dictionary = dictionary or {}
|
||||
keys = key.split('.')
|
||||
field_name = keys.pop()
|
||||
for k in keys:
|
||||
if k not in dictionary:
|
||||
return default_value
|
||||
dictionary = dictionary[k]
|
||||
return dictionary.get(field_name, default_value)
|
||||
|
||||
|
||||
def get_proxy_settings(config: dict) -> dict:
|
||||
"""Returns proxy settings in the requests format or None if not set up."""
|
||||
proxies = None
|
||||
host = config.get('proxy_host')
|
||||
port = config.get('proxy_port')
|
||||
username = config.get('proxy_username')
|
||||
password = config.get('proxy_password')
|
||||
|
||||
if host:
|
||||
if not port:
|
||||
raise KeyError(
|
||||
('The google_threat_intelligence_proxy_host config is set, '
|
||||
'please also set the virustotal_proxy_port.'))
|
||||
parsed = parse.urlparse(host)
|
||||
if 'http' in parsed.scheme:
|
||||
scheme = 'http'
|
||||
else:
|
||||
scheme = parsed.scheme
|
||||
netloc = parsed.netloc
|
||||
host = f'{netloc}:{port}'
|
||||
|
||||
if username:
|
||||
if not password:
|
||||
raise KeyError(('The google_threat_intelligence_'
|
||||
' proxy_host config is set, please also'
|
||||
' set the virustotal_proxy_password.'))
|
||||
auth = f'{username}:{password}'
|
||||
host = auth + '@' + host
|
||||
|
||||
proxies = {
|
||||
'http': f'{scheme}://{host}',
|
||||
'https': f'{scheme}://{host}'
|
||||
}
|
||||
return proxies
|
||||
|
||||
|
||||
def dict_handler(request: dict):
|
||||
"""MISP entry point fo the module."""
|
||||
if not request.get('config') or not request['config'].get('apikey'):
|
||||
return {
|
||||
'error': ('A Google Threat Intelligence api '
|
||||
'key is required for this module.')
|
||||
}
|
||||
|
||||
if not request.get('attribute'):
|
||||
return {
|
||||
'error': ('This module requires an "attribute" field as input,'
|
||||
' which should contain at least a type, a value and an'
|
||||
' uuid.')
|
||||
}
|
||||
|
||||
if request['attribute']['type'] not in MISP_ATTRIBUTES['input']:
|
||||
return {'error': 'Unsupported attribute type.'}
|
||||
|
||||
event_limit = request['config'].get('event_limit')
|
||||
attribute = request['attribute']
|
||||
|
||||
try:
|
||||
proxy_settings = get_proxy_settings(request.get('config'))
|
||||
client = vt.Client(
|
||||
request['config']['apikey'],
|
||||
headers={
|
||||
'x-tool': 'MISPModuleGTIExpansion',
|
||||
},
|
||||
proxy=proxy_settings['http'] if proxy_settings else None)
|
||||
parser = GoogleThreatIntelligenceParser(
|
||||
client, int(event_limit) if event_limit else None)
|
||||
parser.query_api(attribute)
|
||||
except vt.APIError as ex:
|
||||
return {'error': ex.message}
|
||||
except KeyError as ex:
|
||||
return {'error': str(ex)}
|
||||
|
||||
return parser.get_results()
|
||||
|
||||
|
||||
def introspection():
|
||||
"""Returns the module input attributes required."""
|
||||
return MISP_ATTRIBUTES
|
||||
|
||||
|
||||
def version():
|
||||
"""Returns the module metadata."""
|
||||
return MODULE_INFO
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
# Testing/debug calls.
|
||||
import os
|
||||
api_key = os.getenv('GTI_API_KEY')
|
||||
# File
|
||||
request_data = {
|
||||
'config': {'apikey': api_key},
|
||||
'attribute': {
|
||||
'type': 'sha256',
|
||||
'value': ('ed01ebfbc9eb5bbea545af4d01bf5f10'
|
||||
'71661840480439c6e5babe8e080e41aa')
|
||||
}
|
||||
}
|
||||
response = dict_handler(request_data)
|
||||
report_obj = response['results']['Object'][0]
|
||||
print(report_obj.to_dict())
|
||||
|
||||
# URL
|
||||
request_data = {
|
||||
'config': {'apikey': api_key},
|
||||
'attribute': {
|
||||
'type': 'url',
|
||||
'value': 'http://47.21.48.182:60813/Mozi.a'
|
||||
}
|
||||
}
|
||||
response = dict_handler(request_data)
|
||||
report_obj = response['results']['Object'][0]
|
||||
print(report_obj.to_dict())
|
||||
|
||||
# Ip
|
||||
request_data = {
|
||||
'config': {'apikey': api_key},
|
||||
'attribute': {
|
||||
'type': 'ip-src',
|
||||
'value': '180.72.148.38'
|
||||
}
|
||||
}
|
||||
response = dict_handler(request_data)
|
||||
report_obj = response['results']['Object'][0]
|
||||
print(report_obj.to_dict())
|
||||
|
||||
# Domain
|
||||
request_data = {
|
||||
'config': {'apikey': api_key},
|
||||
'attribute': {
|
||||
'type': 'domain',
|
||||
'value': 'qexyhuv.com'
|
||||
}
|
||||
}
|
||||
response = dict_handler(request_data)
|
||||
report_obj = response['results']['Object'][0]
|
||||
print(report_obj.to_dict())
|
||||
Loading…
Reference in New Issue