MISP
/
misp-modules
mirror of https://github.com/MISP/misp-modules
2
0
Fork 0
Code Issues Releases Wiki Activity

fix: Making json_decode even happier with full json format 

- Using MISPEvent because it is cleaner & easier
- Also cleaner implementation globally
pull/304/head
chrisr3d 2019-03-14 18:48:13 +01:00
parent 62bc45e03a
commit 0b92fd5a53
No known key found for this signature in database
GPG Key ID: 6BBED1B63A6D639F
1 changed files with 93 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

188
misp_modules/modules/expansion/urlhaus.py
View File

@ -1,5 +1,5 @@
from collections import defaultdict from collections import defaultdict
from pymisp import MISPAttribute, MISPObject from pymisp import MISPAttribute, MISPEvent, MISPObject
import json import json
import requests import requests
@ -12,111 +12,107 @@ moduleinfo = {'version': '0.1', 'author': 'Christian Studer',
'module-type': ['expansion', 'hover']} 'module-type': ['expansion', 'hover']}
moduleconfig = [] moduleconfig = []
file_keys = ('filename', 'response_size', 'response_md5', 'response_sha256')
def _create_file_object(file_attributes): file_relations = ('filename', 'size-in-bytes', 'md5', 'sha256')
return _create_object(file_attributes, 'file') vt_keys = ('result', 'link')
vt_types = ('text', 'link')
vt_relations = ('detection-ratio', 'permalink')
def _create_object(attributes, name): class URLhaus():
misp_object = MISPObject(name) def __init__(self):
for relation, attribute in attributes.items(): super(URLhaus, self).__init__()
misp_object.add_attribute(relation, **attribute) self.misp_event = MISPEvent()
return [misp_object]
@staticmethod
def _create_vt_object(virustotal):
vt_object = MISPObject('virustotal-report')
for key, vt_type, relation in zip(vt_keys, vt_types, vt_relations):
vt_object.add_attribute(relation, **{'type': vt_type, 'value': virustotal[key]})
return vt_object
def get_result(self):
event = json.loads(self.misp_event.to_json())['Event']
results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])}
return {'results': results}
def _create_objects_with_relationship(file_attributes, vt_attributes): class HostQuery(URLhaus):
vt_object = _create_vt_object(vt_attributes)[0] def __init__(self, attribute):
vt_uuid = vt_object.uuid super(HostQuery, self).__init__()
file_object = _create_file_object(file_attributes)[0] self.attribute = MISPAttribute()
file_object.add_reference(vt_uuid, 'analysed-with') self.attribute.from_dict(**attribute)
return [file_object, vt_object] self.url = 'https://urlhaus-api.abuse.ch/v1/host/'
def query_api(self):
response = requests.post(self.url, data={'host': self.attribute.value}).json()
if 'urls' in response and response['urls']:
for url in response['urls']:
self.misp_event.add_attribute(type='url', value=url['url'])
def _create_url_attribute(value): class PayloadQuery(URLhaus):
attribute = MISPAttribute() def __init__(self, attribute):
attribute.from_dict(type='url', value=value) super(PayloadQuery, self).__init__()
return attribute self.attribute = MISPAttribute()
self.attribute.from_dict(**attribute)
self.url = 'https://urlhaus-api.abuse.ch/v1/payload/'
def query_api(self):
def _create_vt_object(vt_attributes): hash_type = self.attribute.type
return _create_object(vt_attributes, 'virustotal_report') file_object = MISPObject('file')
if self.attribute.event_id != '0':
file_object.id = self.attribute.event_id
def _handle_payload_urls(response): response = requests.post(self.url, data={'{}_hash'.format(hash_type): self.attribute.value}).json()
filenames = [] other_hash_type = 'md5' if hash_type == 'sha256' else 'sha256'
urls = [] for key, relation in zip(('{}_hash'.format(other_hash_type), 'file_size'), (other_hash_type, 'size-in-bytes')):
if response: if response[key]:
for url in response: file_object.add_attribute(relation, **{'type': relation, 'value': response[key]})
urls.append(url['url']) if response['virustotal']:
if url['filename']: vt_object = self._create_vt_object(response['virustotal'])
filenames.append(url['filename']) file_object.add_reference(vt_object.uuid, 'analyzed-with')
return filenames, urls self.misp_event.add_object(**vt_object)
_filename_ = 'filename'
def _query_host_api(attribute):
response = requests.post('https://urlhaus-api.abuse.ch/v1/host/', data={'host': attribute['value']}).json()
attributes = []
if 'urls' in response and response['urls']:
for url in response['urls']: for url in response['urls']:
attributes.append(_create_url_attribute(url['url']).to_dict()) attribute = MISPAttribute()
return {'results': {'Attribute': attributes}} attribute.from_dict(type='url', value=url['url'])
self.misp_event.add_attribute(**attribute)
file_object.add_reference(attribute.uuid, 'retrieved-from')
if url[_filename_]:
file_object.add_attribute(_filename_, **{'type': _filename_, 'value': url[_filename_]})
self.misp_event.add_object(**file_object)
def _query_payload_api(attribute): class UrlQuery(URLhaus):
hash_type = attribute['type'] def __init__(self, attribute):
response = requests.post('https://urlhaus-api.abuse.ch/v1/payload/', data={'{}_hash'.format(hash_type): attribute['value']}).json() super(UrlQuery, self).__init__()
results = defaultdict(list) self.attribute = MISPAttribute()
filenames, urls = _handle_payload_urls(response['urls']) self.attribute.from_dict(**attribute)
other_hash_type = 'md5' if hash_type == 'sha256' else 'sha256' self.url = 'https://urlhaus-api.abuse.ch/v1/url/'
file_object = MISPObject('file')
if attribute['object_id'] != '0': @staticmethod
file_object.id = attribute['object_id'] def _create_file_object(payload):
for key, relation in zip(('{}_hash'.format(other_hash_type), 'file_size'), (other_hash_type, 'size-in-bytes')): file_object = MISPObject('file')
if response[key]: for key, relation in zip(file_keys, file_relations):
file_object.add_attribute(relation, **{'type': relation, 'value': response[key]}) if payload[key]:
for filename in filenames: file_object.add_attribute(relation, **{'type': relation, 'value': payload[key]})
file_object.add_attribute('filename', **{'type': 'filename', 'value': filename}) return file_object
for url in urls:
attribute = _create_url_attribute(url) def query_api(self):
results['Attribute'].append(attribute.to_dict()) response = requests.post(self.url, data={'url': self.attribute.value}).json()
file_object.add_reference(attribute.uuid, 'retrieved-from') if 'payloads' in response and response['payloads']:
results['Object'].append(file_object.to_dict()) for payload in response['payloads']:
return {'results': results} file_object = self._create_file_object(payload)
if payload['virustotal']:
vt_object = self._create_vt_object(payload['virustotal'])
file_object.add_reference(vt_object.uuid, 'analyzed-with')
self.misp_event.add_object(**vt_object)
self.misp_event.add_object(**file_object)
def _query_url_api(attribute): _misp_type_mapping = {'url': UrlQuery, 'md5': PayloadQuery, 'sha256': PayloadQuery,
response = requests.post('https://urlhaus-api.abuse.ch/v1/url/', data={'url': attribute['value']}).json() 'domain': HostQuery, 'hostname': HostQuery,
results = defaultdict(list) 'ip-src': HostQuery, 'ip-dst': HostQuery}
if 'payloads' in response and response['payloads']:
objects_mapping = {1: _create_file_object, 2: _create_vt_object, 3: _create_objects_with_relationship}
file_keys = ('filename', 'response_size', 'response_md5', 'response_sha256')
file_relations = ('filename', 'size-in-bytes', 'md5', 'sha256')
vt_keys = ('result', 'link')
vt_types = ('text', 'link')
vt_relations = ('detection-ratio', 'permalink')
for payload in response['payloads']:
args = []
object_score = 0
file_attributes = {relation: {'type': relation, 'value': payload[key]} for key, relation in zip(file_keys, file_relations) if payload[key]}
if file_attributes:
object_score += 1
args.append(file_attributes)
if payload['virustotal']:
virustotal = payload['virustotal']
vt_attributes = {relation: {'type': vt_type, 'value': virustotal[key]} for key, vt_type, relation in zip(vt_keys, vt_types, vt_relations)}
if vt_attributes:
object_score += 2
args.append(vt_attributes)
try:
results['Object'].extend([misp_object.to_dict() for misp_object in objects_mapping[object_score](*args)])
except KeyError:
continue
return {'results': results}
_misp_type_mapping = {'url': _query_url_api, 'md5': _query_payload_api, 'sha256': _query_payload_api,
'domain': _query_host_api, 'hostname': _query_host_api,
'ip-src': _query_host_api, 'ip-dst': _query_host_api}
def handler(q=False): def handler(q=False):
@ -124,7 +120,9 @@ def handler(q=False):
return False return False
request = json.loads(q) request = json.loads(q)
attribute = request['attribute'] attribute = request['attribute']
return _misp_type_mapping[attribute['type']](attribute) urlhaus_parser = _misp_type_mapping[attribute['type']](attribute)
urlhaus_parser.query_api()
return urlhaus_parser.get_result()
def introspection(): def introspection():