From f361fb4ee3f9de69bbe6d0c27c42af7e4f373769 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Tue, 20 Feb 2018 17:00:13 +0100 Subject: [PATCH 01/12] Reading the entire document, to create a big dictionary containing the data, as a beginning --- .../modules/import_mod/goamlimport.py | 99 +++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 misp_modules/modules/import_mod/goamlimport.py diff --git a/misp_modules/modules/import_mod/goamlimport.py b/misp_modules/modules/import_mod/goamlimport.py new file mode 100644 index 0000000..dbef826 --- /dev/null +++ b/misp_modules/modules/import_mod/goamlimport.py @@ -0,0 +1,99 @@ +import json, datetime +import xml.etree.ElementTree as ET +from collections import defaultdict + +misperrors = {'error': 'Error'} +moduleinfo = {'version': 1, 'author': 'Christian Studer', + 'description': 'Import from GoAML', + 'module-type': ['import']} +moduleconfig = [] +mispattributes = {'input': ['xml file'], 'output': ['MISPEvent']} + +t_from = {'nodes': ['from_person', 'from_account', 'from_entity'], + 'leaves': ['from_funds_code', 'from_country']} +t_to = {'nodes': ['to_person', 'to_account', 'to_entity'], + 'leaves': ['to_funds_code', 'to_country']} +t_person = {'nodes': ['addresses'], + 'leaves': ['first_name', 'middle_name', 'last_name', 'gender', 'title', 'mothers_name', 'birthdate', + 'passport_number', 'passport_country', 'id_number', 'birth_place', 'alias', 'nationality1']} +t_account = {'nodes': ['signatory'], + 'leaves': ['institution_name', 'institution_code', 'swift', 'branch', 'non_banking_insitution', + 'account', 'currency_code', 'account_name', 'iban', 'client_number', 'opened', 'closed', + 'personal_account_type', 'balance', 'date_balance', 'status_code', 'beneficiary', + 'beneficiary_comment', 'comments']} +entity = {'nodes': ['addresses'], + 'leaves': ['name', 'commercial_name', 'incorporation_legal_form', 'incorporation_number', 'business', 'phone']} + +goAMLobjects = {'report': {'nodes': ['reporting_person', 'location', 'transaction'], + 'leaves': ['rentity_id', 'submission_code', 'report_code', 'submission_date', + 'currency_code_local']}, + 'reporting_person': {'nodes': ['addresses'], + 'leaves': ['first_name', 'middle_name', 'last_name', 'title']}, + 'location': {'nodes': [], + 'leaves': ['address_type', 'address', 'city', 'zip', 'country_code', 'state']}, + 'transaction': {'nodes': ['t_from', 't_from_my_client', 't_to', 't_to_my_client'], + 'leaves': ['transactionnumber', 'transaction_location', 'date_transaction', + 'transmode_code', 'amount_local']}, + 't_from': t_from, + 't_from_my_client': t_from, + 't_to': t_to, + 't_to_my_client': t_to, + 'addresses': {'nodes': ['address'], 'leaves': []}, + 'address': {'nodes': [], + 'leaves': ['address_type', 'address', 'city', 'zip', 'country_code', 'state']}, + 'from_person': t_person, + 'to_person': t_person, + 't_person': t_person, + 'from_account': t_account, + 'to_account': t_account, + 'signatory': {'nodes': ['t_person'], 'leaves': []}, + 'from_entity': entity, + 'to_entity': entity, + } + +class GoAmlParser(): + def __init__(self): + self.dict = defaultdict(list) + + def readFile(self, filename): + self.tree = ET.parse(filename).getroot() + + def parse_xml(self): + self.itterate(self.tree, 'report') + + def itterate(self, tree, aml_type): + elementDict = {} + for element in tree: + tag = element.tag + print(tag) + mapping = goAMLobjects.get(aml_type) + if tag in mapping.get('nodes'): + self.itterate(element, tag) + elif tag in mapping.get('leaves'): + elementDict[tag] = element.text + self.dict[aml_type].append(elementDict) + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + if request.get('file'): + filename = request['file'] + else: + misperrors['error'] = "Unsupported attributes type" + return misperrors + aml_parser = GoAmlParser() + try: + aml_parser.readFile(filename) + except: + misperrors['error'] = "Impossible to read the file" + return misperrors + aml_parser.parse_xml() + return aml_parser.dict + +def introspection(): + return mispattributes + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo From b2b0fccd47c42f54dff290d5b5a000cb0e134b93 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Thu, 22 Feb 2018 16:37:27 +0100 Subject: [PATCH 02/12] fix: Added an object checking - Checking if there are objects in the event, and then if there is at least 1 transaction object - This prevents the module from crashing, but does not guaranty having a valid GoAML file (depending on objects and their relations) --- misp_modules/modules/export_mod/goamlexport.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/misp_modules/modules/export_mod/goamlexport.py b/misp_modules/modules/export_mod/goamlexport.py index e732584..76bbdc8 100644 --- a/misp_modules/modules/export_mod/goamlexport.py +++ b/misp_modules/modules/export_mod/goamlexport.py @@ -194,6 +194,15 @@ def handler(q=False): config = request['config'].get('rentity_id') export_doc = GoAmlGeneration(config) export_doc.from_event(request['data'][0]) + if not export_doc.misp_event.Object: + misperrors['error'] = "There is no object in this event." + return misperrors + types = [] + for obj in export_doc.misp_event.Object: + types.append(obj.name) + if 'transaction' not in types: + misperrors['error'] = "There is no transaction object in this event." + return misperrors export_doc.parse_objects() export_doc.build_xml() exp_doc = "{}{}".format(export_doc.xml.get('header'), export_doc.xml.get('data')) From 359ac9100ebb9ed41c853d228cc84e2d4cb2441c Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Fri, 23 Feb 2018 15:58:04 +0100 Subject: [PATCH 03/12] fix: typo in references mapping dictionary --- misp_modules/modules/export_mod/goamlexport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/export_mod/goamlexport.py b/misp_modules/modules/export_mod/goamlexport.py index 76bbdc8..f6d3ff5 100644 --- a/misp_modules/modules/export_mod/goamlexport.py +++ b/misp_modules/modules/export_mod/goamlexport.py @@ -44,7 +44,7 @@ goAMLmapping = {'bank-account': {'bank-account': 't_account', 'institution-name' referencesMapping = {'bank-account': {'aml_type': '{}_account', 'bracket': 't_{}'}, 'person': {'transaction': {'aml_type': '{}_person', 'bracket': 't_{}'}, 'bank-account': {'aml_type': 't_person', 'bracket': 'signatory'}}, - 'legal-entity': {'transaction': {'aml_type': '{}_entity', 'bracket': 't_{}'}, 'bank-account': {'aml_type': 'entity'}}, + 'legal-entity': {'transaction': {'aml_type': '{}_entity', 'bracket': 't_{}'}, 'bank-account': {'aml_type': 't_entity'}}, 'geolocation': {'aml_type': 'address', 'bracket': 'addresses'}} class GoAmlGeneration(object): From 81a6be17d3f567ced1517b216d8c847cbd1e0267 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Mon, 26 Feb 2018 11:47:35 +0100 Subject: [PATCH 04/12] chg: Structurded data --- misp_modules/modules/import_mod/goamlimport.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/misp_modules/modules/import_mod/goamlimport.py b/misp_modules/modules/import_mod/goamlimport.py index dbef826..a2cda32 100644 --- a/misp_modules/modules/import_mod/goamlimport.py +++ b/misp_modules/modules/import_mod/goamlimport.py @@ -24,7 +24,7 @@ t_account = {'nodes': ['signatory'], entity = {'nodes': ['addresses'], 'leaves': ['name', 'commercial_name', 'incorporation_legal_form', 'incorporation_number', 'business', 'phone']} -goAMLobjects = {'report': {'nodes': ['reporting_person', 'location', 'transaction'], +goAMLobjects = {'report': {'nodes': ['reporting_person', 'location'], 'leaves': ['rentity_id', 'submission_code', 'report_code', 'submission_date', 'currency_code_local']}, 'reporting_person': {'nodes': ['addresses'], @@ -53,25 +53,27 @@ goAMLobjects = {'report': {'nodes': ['reporting_person', 'location', 'transactio class GoAmlParser(): def __init__(self): - self.dict = defaultdict(list) + self.dict = {} def readFile(self, filename): self.tree = ET.parse(filename).getroot() def parse_xml(self): - self.itterate(self.tree, 'report') + self.dict = self.itterate(self.tree, 'report') + self.dict['transaction'] = [] + for t in self.tree.findall('transaction'): + self.dict['transaction'].append(self.itterate(t, 'transaction')) def itterate(self, tree, aml_type): elementDict = {} for element in tree: tag = element.tag - print(tag) mapping = goAMLobjects.get(aml_type) if tag in mapping.get('nodes'): - self.itterate(element, tag) + elementDict[tag] = self.itterate(element, tag) elif tag in mapping.get('leaves'): elementDict[tag] = element.text - self.dict[aml_type].append(elementDict) + return elementDict def handler(q=False): if q is False: From 5df2d309a0e514b848746e9cb7ed5f1c9f11a9c8 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Mon, 26 Feb 2018 15:58:53 +0100 Subject: [PATCH 05/12] typo --- misp_modules/modules/export_mod/goamlexport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/export_mod/goamlexport.py b/misp_modules/modules/export_mod/goamlexport.py index f6d3ff5..f32aef2 100644 --- a/misp_modules/modules/export_mod/goamlexport.py +++ b/misp_modules/modules/export_mod/goamlexport.py @@ -37,7 +37,7 @@ goAMLmapping = {'bank-account': {'bank-account': 't_account', 'institution-name' 'transmode-comment': 'transmode_comment', 'date-posting': 'date_posting', 'teller': 'teller', 'authorized': 'authorized', 'text': 'transaction_description'}, - 'legal-enitty': {'legal-entity': 'entity', 'name': 'name', 'business': 'business', + 'legal-entity': {'legal-entity': 'entity', 'name': 'name', 'business': 'business', 'commercial-name': 'commercial_name', 'phone-number': 'phone', 'legal-form': 'incorporation_legal_form', 'registration-number': 'incorporation_number'}} From 478cd53912238c820928ebb615efb62dd987736e Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Mon, 26 Feb 2018 18:13:43 +0100 Subject: [PATCH 06/12] add: Added dictionary to map aml types into MISP types --- .../modules/import_mod/goamlimport.py | 71 ++++++++++++------- 1 file changed, 47 insertions(+), 24 deletions(-) diff --git a/misp_modules/modules/import_mod/goamlimport.py b/misp_modules/modules/import_mod/goamlimport.py index a2cda32..ea6a3cb 100644 --- a/misp_modules/modules/import_mod/goamlimport.py +++ b/misp_modules/modules/import_mod/goamlimport.py @@ -1,6 +1,7 @@ import json, datetime import xml.etree.ElementTree as ET from collections import defaultdict +from pymisp import MISPEvent misperrors = {'error': 'Error'} moduleinfo = {'version': 1, 'author': 'Christian Studer', @@ -9,51 +10,72 @@ moduleinfo = {'version': 1, 'author': 'Christian Studer', moduleconfig = [] mispattributes = {'input': ['xml file'], 'output': ['MISPEvent']} -t_from = {'nodes': ['from_person', 'from_account', 'from_entity'], +t_from_objects = {'nodes': ['from_person', 'from_account', 'from_entity'], 'leaves': ['from_funds_code', 'from_country']} -t_to = {'nodes': ['to_person', 'to_account', 'to_entity'], +t_to_objects = {'nodes': ['to_person', 'to_account', 'to_entity'], 'leaves': ['to_funds_code', 'to_country']} -t_person = {'nodes': ['addresses'], +t_person_objects = {'nodes': ['addresses'], 'leaves': ['first_name', 'middle_name', 'last_name', 'gender', 'title', 'mothers_name', 'birthdate', 'passport_number', 'passport_country', 'id_number', 'birth_place', 'alias', 'nationality1']} -t_account = {'nodes': ['signatory'], +t_account_objects = {'nodes': ['signatory'], 'leaves': ['institution_name', 'institution_code', 'swift', 'branch', 'non_banking_insitution', 'account', 'currency_code', 'account_name', 'iban', 'client_number', 'opened', 'closed', 'personal_account_type', 'balance', 'date_balance', 'status_code', 'beneficiary', 'beneficiary_comment', 'comments']} -entity = {'nodes': ['addresses'], +entity_objects = {'nodes': ['addresses'], 'leaves': ['name', 'commercial_name', 'incorporation_legal_form', 'incorporation_number', 'business', 'phone']} goAMLobjects = {'report': {'nodes': ['reporting_person', 'location'], - 'leaves': ['rentity_id', 'submission_code', 'report_code', 'submission_date', - 'currency_code_local']}, - 'reporting_person': {'nodes': ['addresses'], - 'leaves': ['first_name', 'middle_name', 'last_name', 'title']}, - 'location': {'nodes': [], - 'leaves': ['address_type', 'address', 'city', 'zip', 'country_code', 'state']}, + 'leaves': ['rentity_id', 'submission_code', 'report_code', 'submission_date', 'currency_code_local']}, + 'reporting_person': {'nodes': ['addresses'], 'leaves': ['first_name', 'middle_name', 'last_name', 'title']}, + 'location': {'nodes': [], 'leaves': ['address_type', 'address', 'city', 'zip', 'country_code', 'state']}, 'transaction': {'nodes': ['t_from', 't_from_my_client', 't_to', 't_to_my_client'], 'leaves': ['transactionnumber', 'transaction_location', 'date_transaction', 'transmode_code', 'amount_local']}, - 't_from': t_from, - 't_from_my_client': t_from, - 't_to': t_to, - 't_to_my_client': t_to, + 't_from': t_from_objects, 't_from_my_client': t_from_objects, + 't_to': t_to_objects, 't_to_my_client': t_to_objects, 'addresses': {'nodes': ['address'], 'leaves': []}, - 'address': {'nodes': [], - 'leaves': ['address_type', 'address', 'city', 'zip', 'country_code', 'state']}, - 'from_person': t_person, - 'to_person': t_person, - 't_person': t_person, - 'from_account': t_account, - 'to_account': t_account, + 'address': {'nodes': [], 'leaves': ['address_type', 'address', 'city', 'zip', 'country_code', 'state']}, + 'from_person': t_person_objects, 'to_person': t_person_objects, 't_person': t_person_objects, + 'from_account': t_account_objects, 'to_account': t_account_objects, 'signatory': {'nodes': ['t_person'], 'leaves': []}, - 'from_entity': entity, - 'to_entity': entity, + 'from_entity': entity_objects, 'to_entity': entity_objects, } +t_account_mapping = {'t_account': 'bank-account', 'institution_name': 'institution-name', 'institution_code': 'institution-code', + 'iban': 'iban', 'swift': 'swift', 'branch': 'branch', 'non_banking_institution': 'non-bank-institution', + 'account': 'account', 'currency_code': 'currency-code', 'account_name': 'account-name', + 'client_number': 'client-number', 'personal_account_type': 'personal-account-type', 'opened': 'opened', + 'closed': 'closed', 'balance': 'balance', 'status_code': 'status-code', 'beneficiary': 'beneficiary', + 'beneficiary_comment': 'beneficiary-comment', 'comments': 'comments'} + +t_person_mapping = {'t_person': 'person', 'comments': 'text', 'first_name': 'first-name', 'middle_name': 'middle-name', + 'last_name': 'last-name', 'title': 'title', 'mothers_name': 'mothers-name', 'alias': 'alias', + 'birthdate': 'date-of-birth', 'birth_place': 'place-of-birth', 'gender': 'gender','nationality1': 'nationality', + 'passport_number': 'passport-number', 'passport_country': 'passport-country', 'ssn': 'social-security-number', + 'id_number': 'identity-card-number'} + +location_mapping = {'location': 'geolocation', 'city': 'city', 'state': 'region', 'country-code': 'country', 'address': 'address', + 'zip': 'zipcode'} + +t_entity_mapping = {'entity': 'legal-entity', 'name': 'name', 'business': 'business', 'commercial_name': 'commercial-name', + 'phone': 'phone-number', 'incorporation_legal_form': 'legal-form', 'incorporation_number': 'registration-number'} + +goAMLmapping = {'from_account': t_account_mapping, 'to_account': t_account_mapping, + 'from_person': t_person_mapping, 'to_person': t_person_mapping, 'reporting_person': t_person_mapping, + 'from_entity': t_entity_mapping, 'to_entity': t_entity_mapping, + 'location': location_mapping, 'address': location_mapping, + 'transaction': {'transaction': 'transaction', 'transactionnumber': 'transaction-number', 'date_transaction': 'date', + 'transaction_location': 'location', 'transmode_code': 'transmode-code', 'amount_local': 'amount', + 'transmode_comment': 'transmode-comment', 'date_posting': 'date-posting', 'teller': 'teller', + 'authorized': 'authorized', 'transaction_description': 'text'}} + +nodes_to_ignore = ['addresses', 'signatory'] + class GoAmlParser(): def __init__(self): self.dict = {} + self.misp_event = MISPEvent() def readFile(self, filename): self.tree = ET.parse(filename).getroot() @@ -63,6 +85,7 @@ class GoAmlParser(): self.dict['transaction'] = [] for t in self.tree.findall('transaction'): self.dict['transaction'].append(self.itterate(t, 'transaction')) + self.misp_event.timestamp = self.dict.get('submission_date') def itterate(self, tree, aml_type): elementDict = {} From a02dbd6a8dc595f734fb289e651dea8ee8465e40 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Mon, 26 Feb 2018 18:44:44 +0100 Subject: [PATCH 07/12] fix: Fixed typo of the aml type for country codes --- misp_modules/modules/export_mod/goamlexport.py | 2 +- tests/goamlexport.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/export_mod/goamlexport.py b/misp_modules/modules/export_mod/goamlexport.py index f32aef2..c277640 100644 --- a/misp_modules/modules/export_mod/goamlexport.py +++ b/misp_modules/modules/export_mod/goamlexport.py @@ -30,7 +30,7 @@ goAMLmapping = {'bank-account': {'bank-account': 't_account', 'institution-name' 'passport-number': 'passport_number', 'passport-country': 'passport_country', 'social-security-number': 'ssn', 'identity-card-number': 'id_number'}, 'geolocation': {'geolocation': 'location', 'city': 'city', 'region': 'state', - 'country': 'country-code', 'address': 'address', 'zipcode': 'zip'}, + 'country': 'country_code', 'address': 'address', 'zipcode': 'zip'}, 'transaction': {'transaction': 'transaction', 'transaction-number': 'transactionnumber', 'date': 'date_transaction', 'location': 'transaction_location', 'transmode-code': 'transmode_code', 'amount': 'amount_local', diff --git a/tests/goamlexport.xml b/tests/goamlexport.xml index ae3ea80..4a001b9 100644 --- a/tests/goamlexport.xml +++ b/tests/goamlexport.xml @@ -1 +1 @@ -2510ESTR2018-02-22T08:34:16+00:00EURTW000009011 Manners Street WellingtonBG2015-12-01T10:03:0012345when it transactsEAAEUR31032027088ATTBVIThe bankNickPittSir1993-09-25Mulhouse, FranceMale
ParisFrance
FRAKMichelJeanHimselfPrefer not to say
LuxembourgLuxembourg
LUX +2510ESTR2018-02-22T08:34:16+00:00EURTW000009011 Manners Street WellingtonBG2015-12-01T10:03:0012345when it transactsEAAEUR31032027088ATTBVIThe bankNickPittSir1993-09-25Mulhouse, FranceMale
ParisFrance
FRAKMichelJeanHimselfPrefer not to say
LuxembourgLuxembourg
LUX From cad62464c5f19aefba17605315cad9d196cbc874 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Tue, 27 Feb 2018 11:08:37 +0100 Subject: [PATCH 08/12] Now parsing all the transaction attributes --- .../modules/import_mod/goamlimport.py | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/misp_modules/modules/import_mod/goamlimport.py b/misp_modules/modules/import_mod/goamlimport.py index ea6a3cb..4cda375 100644 --- a/misp_modules/modules/import_mod/goamlimport.py +++ b/misp_modules/modules/import_mod/goamlimport.py @@ -88,15 +88,29 @@ class GoAmlParser(): self.misp_event.timestamp = self.dict.get('submission_date') def itterate(self, tree, aml_type): - elementDict = {} + element_dict = {} for element in tree: tag = element.tag mapping = goAMLobjects.get(aml_type) if tag in mapping.get('nodes'): - elementDict[tag] = self.itterate(element, tag) + if aml_type == 'transaction': + self.fill_transaction(element, element_dict, tag) + element_dict[tag] = self.itterate(element, tag) elif tag in mapping.get('leaves'): - elementDict[tag] = element.text - return elementDict + try: + element_dict[goAMLmapping[aml_type][tag]] = element.text + except KeyError: + pass + return element_dict + + @staticmethod + def fill_transaction(element, element_dict, tag): + if 't_from' in tag: + element_dict['from-funds-code'] = element.find('from_funds_code').text + element_dict['from-country'] = element.find('from_country').text + if 't_to' in tag: + element_dict['to-funds-code'] = element.find('to_funds_code').text + element_dict['to-country'] = element.find('to_country').text def handler(q=False): if q is False: From 8f5c08e2c6c52e5f06f6cbc1ff037fe86d354856 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Wed, 28 Feb 2018 15:07:55 +0100 Subject: [PATCH 09/12] Converting GoAML into MISPEvent --- .../modules/import_mod/goamlimport.py | 95 +++++++++++-------- 1 file changed, 58 insertions(+), 37 deletions(-) diff --git a/misp_modules/modules/import_mod/goamlimport.py b/misp_modules/modules/import_mod/goamlimport.py index 4cda375..a9174db 100644 --- a/misp_modules/modules/import_mod/goamlimport.py +++ b/misp_modules/modules/import_mod/goamlimport.py @@ -1,7 +1,7 @@ import json, datetime import xml.etree.ElementTree as ET from collections import defaultdict -from pymisp import MISPEvent +from pymisp import MISPEvent, MISPObject misperrors = {'error': 'Error'} moduleinfo = {'version': 1, 'author': 'Christian Studer', @@ -18,12 +18,12 @@ t_person_objects = {'nodes': ['addresses'], 'leaves': ['first_name', 'middle_name', 'last_name', 'gender', 'title', 'mothers_name', 'birthdate', 'passport_number', 'passport_country', 'id_number', 'birth_place', 'alias', 'nationality1']} t_account_objects = {'nodes': ['signatory'], - 'leaves': ['institution_name', 'institution_code', 'swift', 'branch', 'non_banking_insitution', - 'account', 'currency_code', 'account_name', 'iban', 'client_number', 'opened', 'closed', - 'personal_account_type', 'balance', 'date_balance', 'status_code', 'beneficiary', - 'beneficiary_comment', 'comments']} + 'leaves': ['institution_name', 'institution_code', 'swift', 'branch', 'non_banking_insitution', + 'account', 'currency_code', 'account_name', 'iban', 'client_number', 'opened', 'closed', + 'personal_account_type', 'balance', 'date_balance', 'status_code', 'beneficiary', + 'beneficiary_comment', 'comments']} entity_objects = {'nodes': ['addresses'], - 'leaves': ['name', 'commercial_name', 'incorporation_legal_form', 'incorporation_number', 'business', 'phone']} + 'leaves': ['name', 'commercial_name', 'incorporation_legal_form', 'incorporation_number', 'business', 'phone']} goAMLobjects = {'report': {'nodes': ['reporting_person', 'location'], 'leaves': ['rentity_id', 'submission_code', 'report_code', 'submission_date', 'currency_code_local']}, @@ -42,30 +42,30 @@ goAMLobjects = {'report': {'nodes': ['reporting_person', 'location'], 'from_entity': entity_objects, 'to_entity': entity_objects, } -t_account_mapping = {'t_account': 'bank-account', 'institution_name': 'institution-name', 'institution_code': 'institution-code', +t_account_mapping = {'misp_name': 'bank-account', 'institution_name': 'institution-name', 'institution_code': 'institution-code', 'iban': 'iban', 'swift': 'swift', 'branch': 'branch', 'non_banking_institution': 'non-bank-institution', 'account': 'account', 'currency_code': 'currency-code', 'account_name': 'account-name', 'client_number': 'client-number', 'personal_account_type': 'personal-account-type', 'opened': 'opened', 'closed': 'closed', 'balance': 'balance', 'status_code': 'status-code', 'beneficiary': 'beneficiary', 'beneficiary_comment': 'beneficiary-comment', 'comments': 'comments'} -t_person_mapping = {'t_person': 'person', 'comments': 'text', 'first_name': 'first-name', 'middle_name': 'middle-name', +t_person_mapping = {'misp_name': 'person', 'comments': 'text', 'first_name': 'first-name', 'middle_name': 'middle-name', 'last_name': 'last-name', 'title': 'title', 'mothers_name': 'mothers-name', 'alias': 'alias', 'birthdate': 'date-of-birth', 'birth_place': 'place-of-birth', 'gender': 'gender','nationality1': 'nationality', 'passport_number': 'passport-number', 'passport_country': 'passport-country', 'ssn': 'social-security-number', 'id_number': 'identity-card-number'} -location_mapping = {'location': 'geolocation', 'city': 'city', 'state': 'region', 'country-code': 'country', 'address': 'address', - 'zip': 'zipcode'} +location_mapping = {'misp_name': 'geolocation', 'city': 'city', 'state': 'region', 'country_code': 'country', 'address': 'address', + 'zip': 'zipcode'} -t_entity_mapping = {'entity': 'legal-entity', 'name': 'name', 'business': 'business', 'commercial_name': 'commercial-name', +t_entity_mapping = {'misp_name': 'legal-entity', 'name': 'name', 'business': 'business', 'commercial_name': 'commercial-name', 'phone': 'phone-number', 'incorporation_legal_form': 'legal-form', 'incorporation_number': 'registration-number'} -goAMLmapping = {'from_account': t_account_mapping, 'to_account': t_account_mapping, +goAMLmapping = {'from_account': t_account_mapping, 'to_account': t_account_mapping, 't_person': t_person_mapping, 'from_person': t_person_mapping, 'to_person': t_person_mapping, 'reporting_person': t_person_mapping, 'from_entity': t_entity_mapping, 'to_entity': t_entity_mapping, 'location': location_mapping, 'address': location_mapping, - 'transaction': {'transaction': 'transaction', 'transactionnumber': 'transaction-number', 'date_transaction': 'date', + 'transaction': {'misp_name': 'transaction', 'transactionnumber': 'transaction-number', 'date_transaction': 'date', 'transaction_location': 'location', 'transmode_code': 'transmode-code', 'amount_local': 'amount', 'transmode_comment': 'transmode-comment', 'date_posting': 'date-posting', 'teller': 'teller', 'authorized': 'authorized', 'transaction_description': 'text'}} @@ -74,43 +74,64 @@ nodes_to_ignore = ['addresses', 'signatory'] class GoAmlParser(): def __init__(self): - self.dict = {} self.misp_event = MISPEvent() def readFile(self, filename): self.tree = ET.parse(filename).getroot() def parse_xml(self): - self.dict = self.itterate(self.tree, 'report') - self.dict['transaction'] = [] + self.first_itteration() for t in self.tree.findall('transaction'): - self.dict['transaction'].append(self.itterate(t, 'transaction')) - self.misp_event.timestamp = self.dict.get('submission_date') + self.itterate(t, 'transaction') + + def first_itteration(self): + self.misp_event.timestamp = self.tree.find('submission_date').text + for node in goAMLobjects['report']['nodes']: + element = self.tree.find(node) + if element is not None: + self.itterate(element, element.tag) def itterate(self, tree, aml_type): - element_dict = {} - for element in tree: - tag = element.tag - mapping = goAMLobjects.get(aml_type) - if tag in mapping.get('nodes'): + objects = goAMLobjects[aml_type] + if aml_type not in nodes_to_ignore: + try: + mapping = goAMLmapping[aml_type] + misp_object = MISPObject(name=mapping['misp_name']) + for leaf in objects['leaves']: + element = tree.find(leaf) + if element is not None: + object_relation = mapping[element.tag] + attribute = {'object_relation': object_relation, 'value': element.text} + misp_object.add_attribute(**attribute) if aml_type == 'transaction': - self.fill_transaction(element, element_dict, tag) - element_dict[tag] = self.itterate(element, tag) - elif tag in mapping.get('leaves'): - try: - element_dict[goAMLmapping[aml_type][tag]] = element.text - except KeyError: - pass - return element_dict + for node in objects['nodes']: + element = tree.find(node) + if element is not None: + self.fill_transaction(element, element.tag, misp_object) + self.misp_event.add_object(misp_object) + except KeyError: + pass + for node in objects['nodes']: + element = tree.find(node) + if element is not None: + self.itterate(element, element.tag) @staticmethod - def fill_transaction(element, element_dict, tag): + def fill_transaction(element, tag, misp_object): if 't_from' in tag: - element_dict['from-funds-code'] = element.find('from_funds_code').text - element_dict['from-country'] = element.find('from_country').text + from_funds = element.find('from_funds_code').text + from_funds_attribute = {'object_relation': 'from-funds-code', 'value': from_funds} + misp_object.add_attribute(**from_funds_attribute) + from_country = element.find('from_country').text + from_country_attribute = {'object_relation': 'from-country', 'value': from_country} + misp_object.add_attribute(**from_country_attribute) if 't_to' in tag: - element_dict['to-funds-code'] = element.find('to_funds_code').text - element_dict['to-country'] = element.find('to_country').text + to_funds = element.find('to_funds_code').text + to_funds_attribute = {'object_relation': 'to-funds-code', 'value': to_funds} + misp_object.add_attribute(**to_funds_attribute) + to_country = element.find('to_country').text + to_country_attribute = {'object_relation': 'to-country', 'value': to_country} + misp_object.add_attribute(**to_country_attribute) def handler(q=False): if q is False: @@ -128,7 +149,7 @@ def handler(q=False): misperrors['error'] = "Impossible to read the file" return misperrors aml_parser.parse_xml() - return aml_parser.dict + return aml_parser.misp_event.to_json() def introspection(): return mispattributes From 323f71cdd3f1a253a520af07b6f7a56a262be693 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Wed, 28 Feb 2018 17:41:45 +0100 Subject: [PATCH 10/12] Fixed some details about the module output --- misp_modules/modules/import_mod/goamlimport.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/misp_modules/modules/import_mod/goamlimport.py b/misp_modules/modules/import_mod/goamlimport.py index a9174db..10bdaab 100644 --- a/misp_modules/modules/import_mod/goamlimport.py +++ b/misp_modules/modules/import_mod/goamlimport.py @@ -1,4 +1,4 @@ -import json, datetime +import json, datetime, time import xml.etree.ElementTree as ET from collections import defaultdict from pymisp import MISPEvent, MISPObject @@ -85,7 +85,8 @@ class GoAmlParser(): self.itterate(t, 'transaction') def first_itteration(self): - self.misp_event.timestamp = self.tree.find('submission_date').text + submission_date = self.tree.find('submission_date').text.split('+')[0] + self.misp_event.timestamp = int(time.mktime(time.strptime(submission_date, "%Y-%m-%dT%H:%M:%S"))) for node in goAMLobjects['report']['nodes']: element = self.tree.find(node) if element is not None: @@ -149,7 +150,8 @@ def handler(q=False): misperrors['error'] = "Impossible to read the file" return misperrors aml_parser.parse_xml() - return aml_parser.misp_event.to_json() + r = {'results': [{'types': mispattributes['output'], 'values': aml_parser.misp_event.to_json()}]} + return r def introspection(): return mispattributes From 03d20856d9c71b2778e4b5d13b29099c0b0a0f56 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Wed, 28 Feb 2018 22:46:39 +0100 Subject: [PATCH 11/12] add: added goamlimport --- misp_modules/modules/import_mod/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/import_mod/__init__.py b/misp_modules/modules/import_mod/__init__.py index 8acccbd..886eaf7 100644 --- a/misp_modules/modules/import_mod/__init__.py +++ b/misp_modules/modules/import_mod/__init__.py @@ -1,4 +1,4 @@ from . import _vmray -__all__ = ['vmray_import', 'testimport', 'ocr', 'stiximport', 'cuckooimport', +__all__ = ['vmray_import', 'testimport', 'ocr', 'stiximport', 'cuckooimport', 'goamlimport', 'email_import', 'mispjson', 'openiocimport', 'threatanalyzer_import', 'csvimport'] From 82fe8ba78ca5d46643f3ef2c29ed51d82084cfd1 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Fri, 2 Mar 2018 11:03:21 +0100 Subject: [PATCH 12/12] fix: Fixed input & output of the module --- misp_modules/modules/import_mod/goamlimport.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/misp_modules/modules/import_mod/goamlimport.py b/misp_modules/modules/import_mod/goamlimport.py index 10bdaab..9b2a34c 100644 --- a/misp_modules/modules/import_mod/goamlimport.py +++ b/misp_modules/modules/import_mod/goamlimport.py @@ -1,4 +1,4 @@ -import json, datetime, time +import json, datetime, time, base64 import xml.etree.ElementTree as ET from collections import defaultdict from pymisp import MISPEvent, MISPObject @@ -8,7 +8,7 @@ moduleinfo = {'version': 1, 'author': 'Christian Studer', 'description': 'Import from GoAML', 'module-type': ['import']} moduleconfig = [] -mispattributes = {'input': ['xml file'], 'output': ['MISPEvent']} +mispattributes = {'inputSource': ['file'], 'output': ['MISP objects']} t_from_objects = {'nodes': ['from_person', 'from_account', 'from_entity'], 'leaves': ['from_funds_code', 'from_country']} @@ -76,8 +76,8 @@ class GoAmlParser(): def __init__(self): self.misp_event = MISPEvent() - def readFile(self, filename): - self.tree = ET.parse(filename).getroot() + def read_xml(self, data): + self.tree = ET.fromstring(data) def parse_xml(self): self.first_itteration() @@ -138,19 +138,19 @@ def handler(q=False): if q is False: return False request = json.loads(q) - if request.get('file'): - filename = request['file'] + if request.get('data'): + data = base64.b64decode(request['data']).decode('utf-8') else: misperrors['error'] = "Unsupported attributes type" return misperrors aml_parser = GoAmlParser() try: - aml_parser.readFile(filename) + aml_parser.read_xml(data) except: - misperrors['error'] = "Impossible to read the file" + misperrors['error'] = "Impossible to read XML data" return misperrors aml_parser.parse_xml() - r = {'results': [{'types': mispattributes['output'], 'values': aml_parser.misp_event.to_json()}]} + r = {'results': [obj.to_json() for obj in aml_parser.misp_event.objects]} return r def introspection():