From 48869335ee28179ebd21b31227bc91a4056c1695 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Wed, 31 Jan 2018 18:09:45 +0100 Subject: [PATCH 01/11] first tests for the GoAML export module --- misp_modules/modules/export_mod/goaml.py | 47 ++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 misp_modules/modules/export_mod/goaml.py diff --git a/misp_modules/modules/export_mod/goaml.py b/misp_modules/modules/export_mod/goaml.py new file mode 100644 index 0000000..f82590b --- /dev/null +++ b/misp_modules/modules/export_mod/goaml.py @@ -0,0 +1,47 @@ +import json +import pymisp +import base64 + +misperrors = {'error': 'Error'} +moduleinfo = {'version': '1', 'author': 'Christian Studer', + 'description': '', + 'module-type': ['export']} +moduleconfig = [] +mispattributes = {} + +objects_to_parse = ['bank-account', 'person'] + +class GoAmlGeneration(): + def __init__(self): + self.document = {} + + def from_event(self, event): + self.misp_event = pymisp.MISPEvent() + self.misp_event.load(event) + + def parse_objects(self): + for obj in self.misp_event.objects: + if obj.name in objects_to_parse: + obj_dict = {} + for attribute in obj.attributes: + obj_dict[attribute.object_relation] = attribute.value + self.document[obj.name] = obj_dict + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + if 'data' not in request: + return False + exp_doc = GoAmlGeneration() + exp_doc.from_event(request['data'][0]) + exp_doc.parse_objects() + return {'response': {}, 'data': exp_doc.document} + #return {'response': [], 'data': str(base64.b64encode(bytes(exp_doc.document, 'utf-8')), 'utf-8')} + +def introspection(): + return + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo From 8dce7935ae2679540f7eeb8a7d69472f85c7a15a Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Thu, 1 Feb 2018 14:55:48 +0100 Subject: [PATCH 02/11] Outputting xml format Also mapping MISP and GoAML types --- misp_modules/modules/export_mod/goaml.py | 65 ++++++++++++++++++++++-- 1 file changed, 61 insertions(+), 4 deletions(-) diff --git a/misp_modules/modules/export_mod/goaml.py b/misp_modules/modules/export_mod/goaml.py index f82590b..82375c2 100644 --- a/misp_modules/modules/export_mod/goaml.py +++ b/misp_modules/modules/export_mod/goaml.py @@ -1,5 +1,5 @@ import json -import pymisp +from pymisp import MISPEvent import base64 misperrors = {'error': 'Error'} @@ -8,15 +8,29 @@ moduleinfo = {'version': '1', 'author': 'Christian Studer', 'module-type': ['export']} moduleconfig = [] mispattributes = {} +outputFileExtension = "xml" +responseType = "application/xml" objects_to_parse = ['bank-account', 'person'] +goAMLmapping = {'bank-account': 't_account', 'text': 'institution_name', 'institution-code': 'institution_code', + 'swift': 'swift', 'branch': 'branch', 'non-banking-institution': 'non_bank_institution', + 'account': 'account', 'currency-code': 'currency_code', 'account-name': 'account_name', + 'iban': 'iban', 'client-number': 'client_number', 'personal-account-type': 'personal_account_type', + 'opened': 'opened', 'closed': 'closed', 'balance': 'balance', 'status-code': 'status_code', + 'beneficiary': 'beneficiary', 'beneficiary-comment': 'beneficiary_comment', 'comments': 'comments', + 'person': 't_person', 'text': 'comments', 'first-name': 'first_name', 'middle-name': 'middle_name', + 'last-name': 'last_name', 'mothers-name': 'mothers_name', 'title': 'title', 'alias': 'alias', + 'date-of-birth': 'birthdate', 'place-of-birth': 'birth_place', 'gender': 'gender', + 'passport-number': 'passport_number', 'passport-country': 'passport_country', + 'social-security-number': 'ssn', 'nationality': 'nationality1'} + class GoAmlGeneration(): def __init__(self): self.document = {} def from_event(self, event): - self.misp_event = pymisp.MISPEvent() + self.misp_event = MISPEvent() self.misp_event.load(event) def parse_objects(self): @@ -27,6 +41,27 @@ class GoAmlGeneration(): obj_dict[attribute.object_relation] = attribute.value self.document[obj.name] = obj_dict + def build_xml(self): + self.xml = "" + if 'bank-account' in self.document: + if 'report-code' in self.document['bank-account']: + self.xml += "{}".format(self.document['bank-account'].pop('report-code').split(' ')[0]) + for a in ('personal-account-type', 'status-code'): + if a in self.document['bank-account']: + self.document['bank-account'][a] = self.document['bank-account'][a].split(' - ')[0] + self.itterate() + self.xml += "" + + def itterate(self): + for t in self.document: + self.xml += "<{}>".format(goAMLmapping[t]) + for k in self.document[t]: + try: + self.xml += "<{0}>{1}".format(goAMLmapping[k], self.document[t][k]) + except KeyError: + pass + self.xml += "".format(goAMLmapping[t]) + def handler(q=False): if q is False: return False @@ -36,11 +71,33 @@ def handler(q=False): exp_doc = GoAmlGeneration() exp_doc.from_event(request['data'][0]) exp_doc.parse_objects() - return {'response': {}, 'data': exp_doc.document} + exp_doc.build_xml() + return {'response': {}, 'data': exp_doc.xml} #return {'response': [], 'data': str(base64.b64encode(bytes(exp_doc.document, 'utf-8')), 'utf-8')} def introspection(): - return + modulesetup = {} + try: + responseType + modulesetup['responseType'] = responseType + except NameError: + pass + try: + userConfig + modulesetup['userConfig'] = userConfig + except NameError: + pass + try: + outputFileExtension + modulesetup['outputFileExtension'] = outputFileExtension + except NameError: + pass + try: + inputSource + mmoduleSetup['inputSource'] = inputSource + except NameError: + pass + return modulesetup def version(): moduleinfo['config'] = moduleconfig From 8983ebc4b206106ad6b893f3c8a29e51605d0374 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Mon, 5 Feb 2018 15:51:03 +0100 Subject: [PATCH 03/11] wip: added location & signatory information --- misp_modules/modules/export_mod/goaml.py | 25 ++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/misp_modules/modules/export_mod/goaml.py b/misp_modules/modules/export_mod/goaml.py index 82375c2..ce29f8f 100644 --- a/misp_modules/modules/export_mod/goaml.py +++ b/misp_modules/modules/export_mod/goaml.py @@ -11,7 +11,7 @@ mispattributes = {} outputFileExtension = "xml" responseType = "application/xml" -objects_to_parse = ['bank-account', 'person'] +objects_to_parse = ['bank-account', 'person', 'geolocation'] goAMLmapping = {'bank-account': 't_account', 'text': 'institution_name', 'institution-code': 'institution_code', 'swift': 'swift', 'branch': 'branch', 'non-banking-institution': 'non_bank_institution', @@ -23,7 +23,9 @@ goAMLmapping = {'bank-account': 't_account', 'text': 'institution_name', 'instit 'last-name': 'last_name', 'mothers-name': 'mothers_name', 'title': 'title', 'alias': 'alias', 'date-of-birth': 'birthdate', 'place-of-birth': 'birth_place', 'gender': 'gender', 'passport-number': 'passport_number', 'passport-country': 'passport_country', - 'social-security-number': 'ssn', 'nationality': 'nationality1'} + 'social-security-number': 'ssn', 'nationality': 'nationality1', 'identity-card-number': 'id_number', + 'geolocation': 'location', 'city': 'city', 'region': 'state', 'country': 'country-code', + 'address': 'address', 'zipcode': 'zip'} class GoAmlGeneration(): def __init__(self): @@ -55,13 +57,20 @@ class GoAmlGeneration(): def itterate(self): for t in self.document: self.xml += "<{}>".format(goAMLmapping[t]) - for k in self.document[t]: - try: - self.xml += "<{0}>{1}".format(goAMLmapping[k], self.document[t][k]) - except KeyError: - pass + self.fill_xml(t) + if t == 'bank-account' and 'person' in self.document: + self.xml += "" + self.fill_xml('person') + self.xml += "" self.xml += "".format(goAMLmapping[t]) + def fill_xml(self, t): + for k in self.document[t]: + try: + self.xml += "<{0}>{1}".format(goAMLmapping[k], self.document[t][k]) + except KeyError: + pass + def handler(q=False): if q is False: return False @@ -73,7 +82,7 @@ def handler(q=False): exp_doc.parse_objects() exp_doc.build_xml() return {'response': {}, 'data': exp_doc.xml} - #return {'response': [], 'data': str(base64.b64encode(bytes(exp_doc.document, 'utf-8')), 'utf-8')} + #return {'response': [], 'data': str(base64.b64encode(bytes(exp_doc.xml, 'utf-8')), 'utf-8')} def introspection(): modulesetup = {} From 8569c3d70231f8ae79e436560194c50b6ba28664 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Mon, 12 Feb 2018 13:40:49 +0100 Subject: [PATCH 04/11] Suporting the recent objects added to misp-objects - Matching the aml documents structure - Some parts of the document still need to be added --- misp_modules/modules/export_mod/goaml.py | 115 ++++++++++++++++------- 1 file changed, 80 insertions(+), 35 deletions(-) diff --git a/misp_modules/modules/export_mod/goaml.py b/misp_modules/modules/export_mod/goaml.py index ce29f8f..05803d5 100644 --- a/misp_modules/modules/export_mod/goaml.py +++ b/misp_modules/modules/export_mod/goaml.py @@ -1,5 +1,6 @@ import json from pymisp import MISPEvent +from collections import defaultdict import base64 misperrors = {'error': 'Error'} @@ -11,12 +12,12 @@ mispattributes = {} outputFileExtension = "xml" responseType = "application/xml" -objects_to_parse = ['bank-account', 'person', 'geolocation'] +objects_to_parse = ['transaction', 'bank-account', 'person', 'entity', 'geolocation'] -goAMLmapping = {'bank-account': 't_account', 'text': 'institution_name', 'institution-code': 'institution_code', +goAMLmapping = {'bank-account': 't_account', 'institution-code': 'institution_code', 'iban': 'iban', 'swift': 'swift', 'branch': 'branch', 'non-banking-institution': 'non_bank_institution', 'account': 'account', 'currency-code': 'currency_code', 'account-name': 'account_name', - 'iban': 'iban', 'client-number': 'client_number', 'personal-account-type': 'personal_account_type', + 'client-number': 'client_number', 'personal-account-type': 'personal_account_type', 'opened': 'opened', 'closed': 'closed', 'balance': 'balance', 'status-code': 'status_code', 'beneficiary': 'beneficiary', 'beneficiary-comment': 'beneficiary_comment', 'comments': 'comments', 'person': 't_person', 'text': 'comments', 'first-name': 'first_name', 'middle-name': 'middle_name', @@ -25,64 +26,108 @@ goAMLmapping = {'bank-account': 't_account', 'text': 'institution_name', 'instit 'passport-number': 'passport_number', 'passport-country': 'passport_country', 'social-security-number': 'ssn', 'nationality': 'nationality1', 'identity-card-number': 'id_number', 'geolocation': 'location', 'city': 'city', 'region': 'state', 'country': 'country-code', - 'address': 'address', 'zipcode': 'zip'} + 'address': 'address', 'zipcode': 'zip', + 'transaction': 'transaction', 'transaction-number': 'transactionnumber', 'date': 'date_transaction', + 'location': 'transaction_location', 'transmode-code': 'transmode_code', 'amount': 'amount_local', + 'transmode-comment': 'transmode_comment', 'date-posting': 'date_posting', + 'entity': 'entity', 'name': 'name', 'commercial-name': 'commercial_name', 'business': 'business', + 'legal-form': 'incorporation_legal_form', 'registration-number': 'incorporation_number', + 'phone-number': 'phone'} -class GoAmlGeneration(): - def __init__(self): - self.document = {} +class GoAmlGeneration(object): + def __init__(self, config): + self.config = config def from_event(self, event): self.misp_event = MISPEvent() self.misp_event.load(event) def parse_objects(self): + uuids = defaultdict(list) + report_code = [] + currency_code = [] for obj in self.misp_event.objects: - if obj.name in objects_to_parse: - obj_dict = {} - for attribute in obj.attributes: - obj_dict[attribute.object_relation] = attribute.value - self.document[obj.name] = obj_dict + obj_type = obj.name + uuids[obj_type].append(obj.uuid) + if obj_type == 'bank-account': + try: + report_code.append(obj.get_attributes_by_relation('report-code')) + currency_code.append(obj.get_attributes_by_relation('currency-code')) + except: + print('non') + self.uuids, self.report_code, self.currency_code = uuids, report_code, currency_code def build_xml(self): - self.xml = "" - if 'bank-account' in self.document: - if 'report-code' in self.document['bank-account']: - self.xml += "{}".format(self.document['bank-account'].pop('report-code').split(' ')[0]) - for a in ('personal-account-type', 'status-code'): - if a in self.document['bank-account']: - self.document['bank-account'][a] = self.document['bank-account'][a].split(' - ')[0] - self.itterate() + self.xml = "{}".format(self.config) + for trans_uuid in self.uuids.get('transaction'): + self.itterate('transaction', 'transaction', trans_uuid) self.xml += "" - def itterate(self): - for t in self.document: - self.xml += "<{}>".format(goAMLmapping[t]) - self.fill_xml(t) - if t == 'bank-account' and 'person' in self.document: - self.xml += "" - self.fill_xml('person') - self.xml += "" - self.xml += "".format(goAMLmapping[t]) + def itterate(self, object_type, aml_type, uuid): + self.xml += "<{}>".format(aml_type) + obj = self.misp_event.get_object_by_uuid(uuid) + self.fill_xml(obj) + if obj.ObjectReference: + for ref in obj.ObjectReference: + uuid = ref.referenced_uuid + next_object_type = ref.Object.get('name') + relationship_type = ref.relationship_type + self.parse_references(object_type, next_object_type, uuid, relationship_type) + self.xml += "".format(aml_type) - def fill_xml(self, t): - for k in self.document[t]: + def fill_xml(self, obj): + for attribute in obj.attributes: + if obj.name == 'bank-account' and attribute.type in ('personal-account-type', 'status-code'): + attribute_value = attribute.value.split(' - ')[0] + else: + attribute_value = attribute.value try: - self.xml += "<{0}>{1}".format(goAMLmapping[k], self.document[t][k]) + self.xml += "<{0}>{1}".format(goAMLmapping[attribute.object_relation], attribute_value) except KeyError: pass + def parse_references(self, object_type, next_object_type, uuid, relationship_type): + if next_object_type == 'bank-account': + self.xml += "".format(relationship_type) + next_aml_type = "{}_account".format(relationship_type.split('_')[0]) + self.itterate(next_object_type, next_aml_type, uuid) + self.xml += "".format(relationship_type) + elif next_object_type == 'person': + if object_type == 'transaction': + self.xml += "".format(relationship_type) + next_aml_type = "{}_person".format(relationship_type.split('_')[0]) + self.itterate(next_object_type, next_aml_type, uuid) + self.xml += "".format(relationship_type) + elif object_type == 'bank-account': + self.xml += "" + next_aml_type = goAMLmapping[next_object_type] + self.itterate(next_object_type, next_aml_type, uuid) + self.xml += "" + elif next_object_type == 'legal-entity': + if object_type == 'transaction': + self.xml += "".format(relationship_type) + next_aml_type = "{}_entity".format(relationship_type.split('_')[0]) + self.itterate(next_object_type, next_aml_type, uuid) + self.xml += "".format(relationship_type) + elif object_type == 'bank-account': + next_aml_type = goAMLmapping[next_object_type] + self.itterate(next_object_type, next_aml_type, uuid) + def handler(q=False): if q is False: return False request = json.loads(q) if 'data' not in request: return False - exp_doc = GoAmlGeneration() + if not request.get('config') and not request['config'].get('rentity_id'): + misperrors['error'] = "Configuration error." + return misperrors + config = request['config'].get('rentity_id') + exp_doc = GoAmlGeneration(config) exp_doc.from_event(request['data'][0]) exp_doc.parse_objects() exp_doc.build_xml() - return {'response': {}, 'data': exp_doc.xml} - #return {'response': [], 'data': str(base64.b64encode(bytes(exp_doc.xml, 'utf-8')), 'utf-8')} + return {'response': [], 'data': str(base64.b64encode(bytes(exp_doc.xml, 'utf-8')), 'utf-8')} def introspection(): modulesetup = {} From a97eeb44fef91677591c46c1b404e19f9d5389f2 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Tue, 13 Feb 2018 11:51:34 +0100 Subject: [PATCH 05/11] Added some report information Also changed the ObjectReference parser to replace all the if conditions by a dictionary reading --- misp_modules/modules/export_mod/goaml.py | 99 +++++++++++++----------- 1 file changed, 52 insertions(+), 47 deletions(-) diff --git a/misp_modules/modules/export_mod/goaml.py b/misp_modules/modules/export_mod/goaml.py index 05803d5..798cc00 100644 --- a/misp_modules/modules/export_mod/goaml.py +++ b/misp_modules/modules/export_mod/goaml.py @@ -30,13 +30,19 @@ goAMLmapping = {'bank-account': 't_account', 'institution-code': 'institution_co 'transaction': 'transaction', 'transaction-number': 'transactionnumber', 'date': 'date_transaction', 'location': 'transaction_location', 'transmode-code': 'transmode_code', 'amount': 'amount_local', 'transmode-comment': 'transmode_comment', 'date-posting': 'date_posting', - 'entity': 'entity', 'name': 'name', 'commercial-name': 'commercial_name', 'business': 'business', + 'legal-entity': 'entity', 'name': 'name', 'commercial-name': 'commercial_name', 'business': 'business', 'legal-form': 'incorporation_legal_form', 'registration-number': 'incorporation_number', 'phone-number': 'phone'} +referencesMapping = {'bank-account': {'aml_type': '{}_account', 'bracket': 't_{}'}, + 'person': {'transaction': {'aml_type': '{}_person', 'bracket': 't_{}'}, 'bank-account': {'aml_type': 't_person', 'bracket': 'signatory'}}, + 'legal-entity': {'transaction': {'aml_type': '{}_entity', 'bracket': 't_{}'}, 'bank-account': {'aml_type': 'entity'}}, + 'geolocation': {'aml_type': 'address', 'bracket': 'addresses'}} + class GoAmlGeneration(object): def __init__(self, config): self.config = config + self.parsed_uuids = defaultdict(list) def from_event(self, event): self.misp_event = MISPEvent() @@ -51,67 +57,65 @@ class GoAmlGeneration(object): uuids[obj_type].append(obj.uuid) if obj_type == 'bank-account': try: - report_code.append(obj.get_attributes_by_relation('report-code')) - currency_code.append(obj.get_attributes_by_relation('currency-code')) + report_code.append(obj.get_attributes_by_relation('report-code')[0]) + currency_code.append(obj.get_attributes_by_relation('currency-code')[0]) except: - print('non') - self.uuids, self.report_code, self.currency_code = uuids, report_code, currency_code + print('report_code or currency_code error') + self.uuids, self.report_codes, self.currency_codes = uuids, report_code, currency_code def build_xml(self): - self.xml = "{}".format(self.config) + self.xml = {'header': "{}".format(self.config), + 'data': ""} for trans_uuid in self.uuids.get('transaction'): - self.itterate('transaction', 'transaction', trans_uuid) - self.xml += "" + self.itterate('transaction', 'transaction', trans_uuid, 'data') + person_to_parse = [person_uuid for person_uuid in self.uuids.get('person') if person_uuid not in self.parsed_uuids.get('person')] + if len(person_to_parse) == 1: + self.itterate('person', 'reporting_person', person_to_parse[0], 'header') + location_to_parse = [location_uuid for location_uuid in self.uuids.get('geolocation') if location_uuid not in self.parsed_uuids.get('geolocation')] + if len(location_to_parse) == 1: + self.itterate('geolocation', 'location', location_to_parse[0], 'header') + self.xml['data'] += "" - def itterate(self, object_type, aml_type, uuid): - self.xml += "<{}>".format(aml_type) + def itterate(self, object_type, aml_type, uuid, xml_part): + self.xml[xml_part] += "<{}>".format(aml_type) obj = self.misp_event.get_object_by_uuid(uuid) - self.fill_xml(obj) + self.fill_xml(obj, xml_part) + self.parsed_uuids[object_type].append(uuid) if obj.ObjectReference: for ref in obj.ObjectReference: uuid = ref.referenced_uuid next_object_type = ref.Object.get('name') relationship_type = ref.relationship_type - self.parse_references(object_type, next_object_type, uuid, relationship_type) - self.xml += "".format(aml_type) + self.parse_references(object_type, next_object_type, uuid, relationship_type, xml_part) + self.xml[xml_part] += "".format(aml_type) - def fill_xml(self, obj): + def fill_xml(self, obj, xml_part): for attribute in obj.attributes: - if obj.name == 'bank-account' and attribute.type in ('personal-account-type', 'status-code'): + if obj.name == 'bank-account' and attribute.object_relation in ('personal-account-type', 'status-code'): attribute_value = attribute.value.split(' - ')[0] else: attribute_value = attribute.value try: - self.xml += "<{0}>{1}".format(goAMLmapping[attribute.object_relation], attribute_value) + self.xml[xml_part] += "<{0}>{1}".format(goAMLmapping[attribute.object_relation], attribute_value) except KeyError: pass - def parse_references(self, object_type, next_object_type, uuid, relationship_type): - if next_object_type == 'bank-account': - self.xml += "".format(relationship_type) - next_aml_type = "{}_account".format(relationship_type.split('_')[0]) - self.itterate(next_object_type, next_aml_type, uuid) - self.xml += "".format(relationship_type) - elif next_object_type == 'person': - if object_type == 'transaction': - self.xml += "".format(relationship_type) - next_aml_type = "{}_person".format(relationship_type.split('_')[0]) - self.itterate(next_object_type, next_aml_type, uuid) - self.xml += "".format(relationship_type) - elif object_type == 'bank-account': - self.xml += "" - next_aml_type = goAMLmapping[next_object_type] - self.itterate(next_object_type, next_aml_type, uuid) - self.xml += "" - elif next_object_type == 'legal-entity': - if object_type == 'transaction': - self.xml += "".format(relationship_type) - next_aml_type = "{}_entity".format(relationship_type.split('_')[0]) - self.itterate(next_object_type, next_aml_type, uuid) - self.xml += "".format(relationship_type) - elif object_type == 'bank-account': - next_aml_type = goAMLmapping[next_object_type] - self.itterate(next_object_type, next_aml_type, uuid) + def parse_references(self, object_type, next_object_type, uuid, relationship_type, xml_part): + try: + next_aml_type = referencesMapping[next_object_type][object_type].get('aml_type').format(relationship_type.split('_')[0]) + try: + bracket = referencesMapping[next_object_type][object_type].get('bracket').format(relationship_type) + self.xml[xml_part] += "<{}>".format(bracket) + self.itterate(next_object_type, next_aml_type, uuid, xml_part) + self.xml[xml_part] += "".format(bracket) + except KeyError: + self.itterate(next_object_type, next_aml_type, uuid, xml_part) + except KeyError: + next_aml_type = referencesMapping[next_object_type].get('aml_type').format(relationship_type.split('_')[0]) + bracket = referencesMapping[next_object_type].get('bracket').format(relationship_type) + self.xml[xml_part] += "<{}>".format(bracket) + self.itterate(next_object_type, next_aml_type, uuid, xml_part) + self.xml[xml_part] += "".format(bracket) def handler(q=False): if q is False: @@ -123,11 +127,12 @@ def handler(q=False): misperrors['error'] = "Configuration error." return misperrors config = request['config'].get('rentity_id') - exp_doc = GoAmlGeneration(config) - exp_doc.from_event(request['data'][0]) - exp_doc.parse_objects() - exp_doc.build_xml() - return {'response': [], 'data': str(base64.b64encode(bytes(exp_doc.xml, 'utf-8')), 'utf-8')} + export_doc = GoAmlGeneration(config) + export_doc.from_event(request['data'][0]) + export_doc.parse_objects() + export_doc.build_xml() + exp_doc = "{}{}".format(export_doc.xml.get('header'), export_doc.xml.get('data')) + return {'response': [], 'data': str(base64.b64encode(bytes(exp_doc, 'utf-8')), 'utf-8')} def introspection(): modulesetup = {} From d4538382d03bf59a6bdad76a24a9efb15b81cad2 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Tue, 13 Feb 2018 13:41:22 +0100 Subject: [PATCH 06/11] Simplified ObjectReference dictionary reading --- misp_modules/modules/export_mod/goaml.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/misp_modules/modules/export_mod/goaml.py b/misp_modules/modules/export_mod/goaml.py index 798cc00..fd5cde7 100644 --- a/misp_modules/modules/export_mod/goaml.py +++ b/misp_modules/modules/export_mod/goaml.py @@ -101,18 +101,19 @@ class GoAmlGeneration(object): pass def parse_references(self, object_type, next_object_type, uuid, relationship_type, xml_part): + reference = referencesMapping[next_object_type] try: - next_aml_type = referencesMapping[next_object_type][object_type].get('aml_type').format(relationship_type.split('_')[0]) + next_aml_type = reference[object_type].get('aml_type').format(relationship_type.split('_')[0]) try: - bracket = referencesMapping[next_object_type][object_type].get('bracket').format(relationship_type) + bracket = reference[object_type].get('bracket').format(relationship_type) self.xml[xml_part] += "<{}>".format(bracket) self.itterate(next_object_type, next_aml_type, uuid, xml_part) self.xml[xml_part] += "".format(bracket) except KeyError: self.itterate(next_object_type, next_aml_type, uuid, xml_part) except KeyError: - next_aml_type = referencesMapping[next_object_type].get('aml_type').format(relationship_type.split('_')[0]) - bracket = referencesMapping[next_object_type].get('bracket').format(relationship_type) + next_aml_type = reference.get('aml_type').format(relationship_type.split('_')[0]) + bracket = reference.get('bracket').format(relationship_type) self.xml[xml_part] += "<{}>".format(bracket) self.itterate(next_object_type, next_aml_type, uuid, xml_part) self.xml[xml_part] += "".format(bracket) From 43e9010858d4a8652c98d47e9cf11bb1efdb94e4 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Tue, 13 Feb 2018 16:39:19 +0100 Subject: [PATCH 07/11] Added report expected information fields --- misp_modules/modules/export_mod/goaml.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/misp_modules/modules/export_mod/goaml.py b/misp_modules/modules/export_mod/goaml.py index fd5cde7..23f9ba0 100644 --- a/misp_modules/modules/export_mod/goaml.py +++ b/misp_modules/modules/export_mod/goaml.py @@ -1,7 +1,6 @@ -import json +import json, datetime, base64 from pymisp import MISPEvent -from collections import defaultdict -import base64 +from collections import defaultdict, Counter misperrors = {'error': 'Error'} moduleinfo = {'version': '1', 'author': 'Christian Studer', @@ -57,15 +56,23 @@ class GoAmlGeneration(object): uuids[obj_type].append(obj.uuid) if obj_type == 'bank-account': try: - report_code.append(obj.get_attributes_by_relation('report-code')[0]) - currency_code.append(obj.get_attributes_by_relation('currency-code')[0]) + report_code.append(obj.get_attributes_by_relation('report-code')[0].value.split(' ')[0]) + currency_code.append(obj.get_attributes_by_relation('currency-code')[0].value) except: print('report_code or currency_code error') self.uuids, self.report_codes, self.currency_codes = uuids, report_code, currency_code def build_xml(self): - self.xml = {'header': "{}".format(self.config), + self.xml = {'header': "{}E".format(self.config), 'data': ""} + if "STR" in self.report_codes: + report_code = "STR" + else: + report_code = Counter(self.report_codes).most_common(1)[0][0] + self.xml['header'] += "{}".format(report_code) + submission_date = str(self.misp_event.timestamp).replace(' ', 'T') + self.xml['header'] += "{}".format(submission_date) + self.xml['header'] += "{}".format(Counter(self.currency_codes).most_common(1)[0][0]) for trans_uuid in self.uuids.get('transaction'): self.itterate('transaction', 'transaction', trans_uuid, 'data') person_to_parse = [person_uuid for person_uuid in self.uuids.get('person') if person_uuid not in self.parsed_uuids.get('person')] From be1b541966396e8e67fbcbdd4c508f8ac0010610 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Wed, 14 Feb 2018 12:18:12 +0100 Subject: [PATCH 08/11] Added a required field & the latest attributes in transaction --- misp_modules/modules/export_mod/goaml.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/misp_modules/modules/export_mod/goaml.py b/misp_modules/modules/export_mod/goaml.py index 23f9ba0..acddd43 100644 --- a/misp_modules/modules/export_mod/goaml.py +++ b/misp_modules/modules/export_mod/goaml.py @@ -28,7 +28,8 @@ goAMLmapping = {'bank-account': 't_account', 'institution-code': 'institution_co 'address': 'address', 'zipcode': 'zip', 'transaction': 'transaction', 'transaction-number': 'transactionnumber', 'date': 'date_transaction', 'location': 'transaction_location', 'transmode-code': 'transmode_code', 'amount': 'amount_local', - 'transmode-comment': 'transmode_comment', 'date-posting': 'date_posting', + 'transmode-comment': 'transmode_comment', 'date-posting': 'date_posting', 'teller': 'teller', + 'authorized': 'authorized', 'legal-entity': 'entity', 'name': 'name', 'commercial-name': 'commercial_name', 'business': 'business', 'legal-form': 'incorporation_legal_form', 'registration-number': 'incorporation_number', 'phone-number': 'phone'} @@ -102,6 +103,8 @@ class GoAmlGeneration(object): attribute_value = attribute.value.split(' - ')[0] else: attribute_value = attribute.value + if obj.name == 'transaction' and attribute.object_relation == 'date-posting': + self.xml[xml_part] += "True" try: self.xml[xml_part] += "<{0}>{1}".format(goAMLmapping[attribute.object_relation], attribute_value) except KeyError: From 92ab1d5c23e4c6b05da2d8caaa98a80c8eebf56e Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Wed, 14 Feb 2018 21:30:48 +0100 Subject: [PATCH 09/11] Added "t_to" and "t_from" required fields: funds code & country --- misp_modules/modules/export_mod/goaml.py | 80 +++++++++++++++++++----- 1 file changed, 63 insertions(+), 17 deletions(-) diff --git a/misp_modules/modules/export_mod/goaml.py b/misp_modules/modules/export_mod/goaml.py index acddd43..61311e6 100644 --- a/misp_modules/modules/export_mod/goaml.py +++ b/misp_modules/modules/export_mod/goaml.py @@ -4,10 +4,10 @@ from collections import defaultdict, Counter misperrors = {'error': 'Error'} moduleinfo = {'version': '1', 'author': 'Christian Studer', - 'description': '', + 'description': 'Export to GoAML', 'module-type': ['export']} -moduleconfig = [] -mispattributes = {} +moduleconfig = ['rentity_id'] +mispattributes = {'input': ['MISPEvent'], 'output': ['xml file']} outputFileExtension = "xml" responseType = "application/xml" @@ -85,30 +85,76 @@ class GoAmlGeneration(object): self.xml['data'] += "" def itterate(self, object_type, aml_type, uuid, xml_part): - self.xml[xml_part] += "<{}>".format(aml_type) obj = self.misp_event.get_object_by_uuid(uuid) + if object_type == 'transaction': + self.xml[xml_part] += "<{}>".format(aml_type) + self.fill_xml_transaction(obj.attributes, xml_part) + self.parsed_uuids[object_type].append(uuid) + if obj.ObjectReference: + self.parseObjectReferences(object_type, xml_part, obj.ObjectReference) + self.xml[xml_part] += "".format(aml_type) + else: + if 'to_' in aml_type or 'from_' in aml_type: + relation_type = aml_type.split('_')[0] + self.xml[xml_part] += "<{0}_funds_code>{1}".format(relation_type, self.from_and_to_fields[relation_type]['funds'].split(' ')[0]) + self.itterate_normal_case(object_type, obj, aml_type, uuid, xml_part) + self.xml[xml_part] += "<{0}_country>{1}".format(relation_type, self.from_and_to_fields[relation_type]['country']) + else: + self.itterate_normal_case(object_type, obj, aml_type, uuid, xml_part) + + def itterate_normal_case(self, object_type, obj, aml_type, uuid, xml_part): + self.xml[xml_part] += "<{}>".format(aml_type) self.fill_xml(obj, xml_part) self.parsed_uuids[object_type].append(uuid) if obj.ObjectReference: - for ref in obj.ObjectReference: - uuid = ref.referenced_uuid - next_object_type = ref.Object.get('name') - relationship_type = ref.relationship_type - self.parse_references(object_type, next_object_type, uuid, relationship_type, xml_part) + self.parseObjectReferences(object_type, xml_part, obj.ObjectReference) self.xml[xml_part] += "".format(aml_type) - def fill_xml(self, obj, xml_part): - for attribute in obj.attributes: - if obj.name == 'bank-account' and attribute.object_relation in ('personal-account-type', 'status-code'): - attribute_value = attribute.value.split(' - ')[0] - else: - attribute_value = attribute.value - if obj.name == 'transaction' and attribute.object_relation == 'date-posting': + def parseObjectReferences(self, object_type, xml_part, references): + for ref in references: + next_uuid = ref.referenced_uuid + next_object_type = ref.Object.get('name') + relationship_type = ref.relationship_type + self.parse_references(object_type, next_object_type, next_uuid, relationship_type, xml_part) + + def fill_xml_transaction(self, attributes, xml_part): + from_and_to_fields = {'from': {}, 'to': {}} + for attribute in attributes: + object_relation = attribute.object_relation + attribute_value = attribute.value + if object_relation == 'date-posting': self.xml[xml_part] += "True" + elif object_relation in ('from-funds-code', 'to-funds-code'): + relation_type, field, _ = object_relation.split('-') + from_and_to_fields[relation_type][field] = attribute_value + continue + elif object_relation in ('from-country', 'to-country'): + relation_type, field = object_relation.split('-') + from_and_to_fields[relation_type][field] = attribute_value + continue try: - self.xml[xml_part] += "<{0}>{1}".format(goAMLmapping[attribute.object_relation], attribute_value) + self.xml[xml_part] += "<{0}>{1}".format(goAMLmapping[object_relation], attribute_value) except KeyError: pass + self.from_and_to_fields = from_and_to_fields + + def fill_xml(self, obj, xml_part): + if obj.name == 'bank-account': + for attribute in obj.attributes: + if attribute.object_relation in ('personal-account-type', 'status-code'): + attribute_value = attribute.value.split(' - ')[0] + else: + attribute_value = attribute.value + try: + self.xml[xml_part] += "<{0}>{1}".format(goAMLmapping[attribute.object_relation], attribute_value) + except KeyError: + pass + else: + for attribute in obj.attributes: + try: + self.xml[xml_part] += "<{0}>{1}".format(goAMLmapping[attribute.object_relation], attribute.value) + except KeyError: + pass def parse_references(self, object_type, next_object_type, uuid, relationship_type, xml_part): reference = referencesMapping[next_object_type] From eb9e06f1cc4aa07cd35c3766cdcb83b7f580eab5 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Tue, 20 Feb 2018 15:15:54 +0100 Subject: [PATCH 10/11] explicit name Avoiding confusion with the coming import module for goaml --- misp_modules/modules/export_mod/{goaml.py => goamlexport.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename misp_modules/modules/export_mod/{goaml.py => goamlexport.py} (100%) diff --git a/misp_modules/modules/export_mod/goaml.py b/misp_modules/modules/export_mod/goamlexport.py similarity index 100% rename from misp_modules/modules/export_mod/goaml.py rename to misp_modules/modules/export_mod/goamlexport.py From 02b8938b2aefbe10d4eb5099206be492753fb136 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Tue, 20 Feb 2018 16:57:27 +0100 Subject: [PATCH 11/11] typo --- misp_modules/modules/export_mod/goamlexport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/export_mod/goamlexport.py b/misp_modules/modules/export_mod/goamlexport.py index 61311e6..2859b3c 100644 --- a/misp_modules/modules/export_mod/goamlexport.py +++ b/misp_modules/modules/export_mod/goamlexport.py @@ -210,7 +210,7 @@ def introspection(): pass try: inputSource - mmoduleSetup['inputSource'] = inputSource + moduleSetup['inputSource'] = inputSource except NameError: pass return modulesetup