MISP
/
misp-modules
mirror of https://github.com/MISP/misp-modules
2
0
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'main' of https://github.com/MISP/misp-modules into cof2misp

pull/507/head
aaronkaplan 2021-05-27 01:58:55 +02:00
parent 6824b4e991 68a210c7ca
commit 9813f7f7cb
No known key found for this signature in database
GPG Key ID: 1AED8B672DD4C9B1
7 changed files with 225 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.gitignore vendored
View File

@ -16,4 +16,7 @@ site*
.idea/*
#venv
venv*
venv*
#vscode
.vscode*

12
documentation/website/import_mod/cof2misp.json Normal file
View File

@ -0,0 +1,12 @@
{
"description": "Passive DNS Common Output Format (COF) MISP importer",
"requirements": [
"PyMISP"
],
"features": "Takes as input a valid COF file or the output of the dnsdbflex utility and creates MISP objects for the input.",
"references": [
"https://tools.ietf.org/id/draft-dulaunoy-dnsop-passive-dns-cof-08.html"
],
"input": "Passive DNS output in Common Output Format (COF)",
"output": "MISP objects"
}

0
misp_modules/lib/cof2misp/__init__.py Normal file
View File

11
misp_modules/modules/expansion/farsight_passivedns.py
View File

@ -2,7 +2,7 @@ import dnsdb2
import json
from . import check_input_attribute, standard_error_message
from datetime import datetime
from pymisp import MISPEvent, MISPObject
from pymisp import MISPEvent, MISPObject, Distribution
misperrors = {'error': 'Error'}
standard_query_input = [
@ -43,7 +43,7 @@ moduleconfig = ['apikey', 'server', 'limit', 'flex_queries']
DEFAULT_DNSDB_SERVER = 'https://api.dnsdb.info'
DEFAULT_LIMIT = 10
DEFAULT_DISTRIBUTION_SETTING = Distribution.your_organisation_only.value
TYPE_TO_FEATURE = {
"btc": "Bitcoin address",
"dkim": "domainkeys identified mail",
@ -85,7 +85,7 @@ class FarsightDnsdbParser():
self.misp_event = MISPEvent()
self.misp_event.add_attribute(**attribute)
self.passivedns_mapping = {
'bailiwick': {'type': 'text', 'object_relation': 'bailiwick'},
'bailiwick': {'type': 'domain', 'object_relation': 'bailiwick'},
'count': {'type': 'counter', 'object_relation': 'count'},
'raw_rdata': {'type': 'text', 'object_relation': 'raw_rdata'},
'rdata': {'type': 'text', 'object_relation': 'rdata'},
@ -103,6 +103,7 @@ class FarsightDnsdbParser():
comment = self.comment % (query_type, TYPE_TO_FEATURE[self.attribute['type']], self.attribute['value'])
for result in results:
passivedns_object = MISPObject('passive-dns')
passivedns_object.distribution = DEFAULT_DISTRIBUTION_SETTING
if result.get('rdata') and isinstance(result['rdata'], list):
for rdata in result.pop('rdata'):
passivedns_object.add_attribute(**self._parse_attribute(comment, 'rdata', rdata))
@ -121,7 +122,7 @@ class FarsightDnsdbParser():
return {'results': results}
def _parse_attribute(self, comment, feature, value):
attribute = {'value': value, 'comment': comment}
attribute = {'value': value, 'comment': comment, 'distribution': DEFAULT_DISTRIBUTION_SETTING}
attribute.update(self.passivedns_mapping[feature])
return attribute
@ -148,6 +149,8 @@ def handler(q=False):
response = to_query(client, *args)
except dnsdb2.DnsdbException as e:
return {'error': e.__str__()}
except dnsdb2.exceptions.QueryError:
return {'error': 'Communication error occurs while executing a query, or the server reports an error due to invalid arguments.'}
if not response:
return {'error': f"Empty results on Farsight DNSDB for the {TYPE_TO_FEATURE[attribute['type']]}: {attribute['value']}."}
parser = FarsightDnsdbParser(attribute)

263
misp_modules/modules/expansion/onyphe.py
View File

@ -1,6 +1,9 @@
# -*- coding: utf-8 -*-
import json
from pymisp import MISPEvent, MISPObject
try:
from onyphe import Onyphe
except ImportError:
@ -9,9 +12,10 @@ except ImportError:
misperrors = {'error': 'Error'}
mispattributes = {'input': ['ip-src', 'ip-dst', 'hostname', 'domain'],
'output': ['hostname', 'domain', 'ip-src', 'ip-dst', 'url']}
'output': ['hostname', 'domain', 'ip-src', 'ip-dst', 'url'],
'format': 'misp_standard'}
# possible module-types: 'expansion', 'hover' or both
moduleinfo = {'version': '1', 'author': 'Sebastien Larinier @sebdraven',
moduleinfo = {'version': '2', 'author': 'Sebastien Larinier @sebdraven',
'description': 'Query on Onyphe',
'module-type': ['expansion', 'hover']}
@ -19,84 +23,205 @@ moduleinfo = {'version': '1', 'author': 'Sebastien Larinier @sebdraven',
moduleconfig = ['apikey']
class OnypheClient:
def __init__(self, api_key, attribute):
self.onyphe_client = Onyphe(api_key=api_key)
self.attribute = attribute
self.misp_event = MISPEvent()
self.misp_event.add_attribute(**attribute)
def get_results(self):
event = json.loads(self.misp_event.to_json())
results = {key: event[key]
for key in ('Attribute', 'Object') if key in event}
return results
def get_query_onyphe(self):
if self.attribute['type'] == 'ip-src' or self.attribute['type'] == 'ip-dst':
self.__summary_ip()
if self.attribute['type'] == 'domain':
self.__summary_domain()
if self.attribute['type'] == 'hostname':
self.__summary_hostname()
def __summary_ip(self):
results = self.onyphe_client.summary_ip(self.attribute['value'])
if 'results' in results:
for r in results['results']:
if 'domain' in r:
domain = r['domain']
if type(domain) == list:
for d in domain:
self.__get_object_domain_ip(d, 'domain')
elif type(domain) == str:
self.__get_object_domain_ip(domain, 'domain')
if 'hostname' in r:
hostname = r['hostname']
if type(hostname) == list:
for d in hostname:
self.__get_object_domain_ip(d, 'domain')
elif type(hostname) == str:
self.__get_object_domain_ip(hostname, 'domain')
if 'issuer' in r:
self.__get_object_certificate(r)
def __summary_domain(self):
results = self.onyphe_client.summary_domain(self.attribute['value'])
if 'results' in results:
for r in results['results']:
for domain in r.get('domain'):
self.misp_event.add_attribute('domain', domain)
for hostname in r.get('hostname'):
self.misp_event.add_attribute('hostname', hostname)
if 'ip' in r:
if type(r['ip']) is str:
self.__get_object_domain_ip(r['ip'], 'ip')
if type(r['ip']) is list:
for ip in r['ip']:
self.__get_object_domain_ip(ip, 'ip')
if 'issuer' in r:
self.__get_object_certificate(r)
def __summary_hostname(self):
results = self.onyphe_client.summary_hostname(self.attribute['value'])
if 'results' in results:
for r in results['results']:
if 'domain' in r:
if type(r['domain']) is str:
self.misp_event.add_attribute(
'domain', r['domain'])
if type(r['domain']) is list:
for domain in r['domain']:
self.misp_event.add_attribute('domain', domain)
if 'hostname' in r:
if type(r['hostname']) is str:
self.misp_event.add_attribute(
'hostname', r['hostname'])
if type(r['hostname']) is list:
for hostname in r['hostname']:
self.misp_event.add_attribute(
'hostname', hostname)
if 'ip' in r:
if type(r['ip']) is str:
self.__get_object_domain_ip(r['ip'], 'ip')
if type(r['ip']) is list:
for ip in r['ip']:
self.__get_object_domain_ip(ip, 'ip')
if 'issuer' in r:
self.__get_object_certificate(r)
if 'cve' in r:
if type(r['cve']) is list:
for cve in r['cve']:
self.__get_object_cve(r, cve)
def __get_object_certificate(self, r):
object_certificate = MISPObject('x509')
object_certificate.add_attribute('ip', self.attribute['value'])
object_certificate.add_attribute('serial-number', r['serial'])
object_certificate.add_attribute(
'x509-fingerprint-sha256', r['fingerprint']['sha256'])
object_certificate.add_attribute(
'x509-fingerprint-sha1', r['fingerprint']['sha1'])
object_certificate.add_attribute(
'x509-fingerprint-md5', r['fingerprint']['md5'])
signature = r['signature']['algorithm']
value = ''
if 'sha256' in signature and 'RSA' in signature:
value = 'SHA256_WITH_RSA_ENCRYPTION'
elif 'sha1' in signature and 'RSA' in signature:
value = 'SHA1_WITH_RSA_ENCRYPTION'
if value:
object_certificate.add_attribute('signature_algorithm', value)
object_certificate.add_attribute(
'pubkey-info-algorithm', r['publickey']['algorithm'])
if 'exponent' in r['publickey']:
object_certificate.add_attribute(
'pubkey-info-exponent', r['publickey']['exponent'])
if 'length' in r['publickey']:
object_certificate.add_attribute(
'pubkey-info-size', r['publickey']['length'])
object_certificate.add_attribute('issuer', r['issuer']['commonname'])
object_certificate.add_attribute(
'validity-not-before', r['validity']['notbefore'])
object_certificate.add_attribute(
'validity-not-after', r['validity']['notbefore'])
object_certificate.add_reference(self.attribute['uuid'], 'related-to')
self.misp_event.add_object(object_certificate)
def __get_object_domain_ip(self, obs, relation):
objet_domain_ip = MISPObject('domain-ip')
objet_domain_ip.add_attribute(relation, obs)
relation_attr = self.__get_relation_attribute()
if relation_attr:
objet_domain_ip.add_attribute(
relation_attr, self.attribute['value'])
objet_domain_ip.add_reference(self.attribute['uuid'], 'related-to')
self.misp_event.add_object(objet_domain_ip)
def __get_relation_attribute(self):
if self.attribute['type'] == 'ip-src':
return 'ip'
elif self.attribute['type'] == 'ip-dst':
return 'ip'
elif self.attribute['type'] == 'domain':
return 'domain'
elif self.attribute['type'] == 'hostname':
return 'domain'
def __get_object_cve(self, item, cve):
attributes = []
object_cve = MISPObject('vulnerability')
object_cve.add_attribute('id', cve)
object_cve.add_attribute('state', 'Published')
if type(item['ip']) is list:
for ip in item['ip']:
attributes.extend(
list(filter(lambda x: x['value'] == ip, self.misp_event['Attribute'])))
for obj in self.misp_event['Object']:
attributes.extend(
list(filter(lambda x: x['value'] == ip, obj['Attribute'])))
if type(item['ip']) is str:
for obj in self.misp_event['Object']:
for att in obj['Attribute']:
if att['value'] == item['ip']:
object_cve.add_reference(obj['uuid'], 'cve')
self.misp_event.add_object(object_cve)
def handler(q=False):
if q:
request = json.loads(q)
attribute = request['attribute']
if not request.get('config') or not request['config'].get('apikey'):
misperrors['error'] = 'Onyphe authentication is missing'
return misperrors
api = Onyphe(request['config'].get('apikey'))
api_key = request['config'].get('apikey')
if not api:
misperrors['error'] = 'Onyphe Error instance api'
onyphe_client = OnypheClient(api_key, attribute)
onyphe_client.get_query_onyphe()
results = onyphe_client.get_results()
ip = ''
if request.get('ip-src'):
ip = request['ip-src']
elif request.get('ip-dst'):
ip = request['ip-dst']
else:
misperrors['error'] = "Unsupported attributes type"
return misperrors
return handle_expansion(api, ip, misperrors)
else:
return False
def handle_expansion(api, ip, misperrors):
result = api.ip(ip)
if result['status'] == 'nok':
misperrors['error'] = result['message']
return misperrors
# categories = list(set([item['@category'] for item in result['results']]))
result_filtered = {"results": []}
urls_pasties = []
asn_list = []
os_list = []
domains_resolver = []
domains_forward = []
for r in result['results']:
if r['@category'] == 'pastries':
if r['source'] == 'pastebin':
urls_pasties.append('https://pastebin.com/raw/%s' % r['key'])
elif r['@category'] == 'synscan':
asn_list.append(r['asn'])
os_target = r['os']
if os_target != 'Unknown':
os_list.append(r['os'])
elif r['@category'] == 'resolver' and r['type'] == 'reverse':
domains_resolver.append(r['reverse'])
elif r['@category'] == 'resolver' and r['type'] == 'forward':
domains_forward.append(r['forward'])
result_filtered['results'].append({'types': ['url'], 'values': urls_pasties,
'categories': ['External analysis']})
result_filtered['results'].append({'types': ['AS'], 'values': list(set(asn_list)),
'categories': ['Network activity']})
result_filtered['results'].append({'types': ['target-machine'],
'values': list(set(os_list)),
'categories': ['Targeting data']})
result_filtered['results'].append({'types': ['domain'],
'values': list(set(domains_resolver)),
'categories': ['Network activity'],
'comment': 'resolver to %s' % ip})
result_filtered['results'].append({'types': ['domain'],
'values': list(set(domains_forward)),
'categories': ['Network activity'],
'comment': 'forward to %s' % ip})
return result_filtered
return {'results': results}
def introspection():

7
misp_modules/modules/import_mod/cof2misp.py
View File

@ -81,7 +81,8 @@ def parse_and_insert_cof(data: str) -> dict:
o = MISPObject(name='passive-dns', standalone=False, comment='created by cof2misp')
# o.add_tag('tlp:amber') # FIXME: we'll want to add a tlp: tag to the object
o.add_attribute('bailiwick', value=entry['bailiwick'].rstrip('.'))
if 'bailiwick' in entry:
o.add_attribute('bailiwick', value=entry['bailiwick'].rstrip('.'))
#
# handle the combinations of rrtype (domain, ip) on both left and right side
@ -217,9 +218,9 @@ def handler(q=False):
# Validate it
# transform into MISP object
# push to MISP
event_id = request['event_id']
# event_id = request['event_id']
# event = misp.get_event(event_id)
print("event_id = %s" % event_id, file=sys.stderr)
# print("event_id = %s" % event_id, file=sys.stderr)
try:
data = base64.b64decode(request["data"]).decode('utf-8')
if not data:

4
tests/test_expansions.py
View File

@ -310,6 +310,8 @@ class TestExpansions(unittest.TestCase):
def test_onyphe(self):
module_name = "onyphe"
if LiveCI:
return True
query = {"module": module_name, "ip-src": "8.8.8.8"}
if module_name in self.configs:
query["config"] = self.configs[module_name]
@ -324,6 +326,8 @@ class TestExpansions(unittest.TestCase):
def test_onyphe_full(self):
module_name = "onyphe_full"
if LiveCI:
return True
query = {"module": module_name, "ip-src": "8.8.8.8"}
if module_name in self.configs:
query["config"] = self.configs[module_name]