From 37d9b3831c1066c6e443e9213869cf178dc8ec9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 26 Oct 2017 16:54:20 -0400 Subject: [PATCH 01/66] Add quick and dirty pdf export --- misp_modules/modules/export_mod/__init__.py | 2 +- misp_modules/modules/export_mod/pdfexport.py | 186 +++++++++++++++++++ 2 files changed, 187 insertions(+), 1 deletion(-) create mode 100755 misp_modules/modules/export_mod/pdfexport.py diff --git a/misp_modules/modules/export_mod/__init__.py b/misp_modules/modules/export_mod/__init__.py index 1b6d161..1932b6f 100644 --- a/misp_modules/modules/export_mod/__init__.py +++ b/misp_modules/modules/export_mod/__init__.py @@ -1 +1 @@ -__all__ = ['testexport','cef_export','liteexport','threat_connect_export'] +__all__ = ['testexport','cef_export','liteexport','threat_connect_export', 'pdfexport'] diff --git a/misp_modules/modules/export_mod/pdfexport.py b/misp_modules/modules/export_mod/pdfexport.py new file mode 100755 index 0000000..4ee7bd7 --- /dev/null +++ b/misp_modules/modules/export_mod/pdfexport.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +from datetime import date +import json +import shlex +import subprocess +import base64 + +from pymisp import MISPEvent + + +misperrors = {'error': 'Error'} + +moduleinfo = {'version': '1', + 'author': 'Raphaël Vinot', + 'description': 'Simple export to PDF', + 'module-type': ['export']} + +moduleconfig = [] + +mispattributes = {} +outputFileExtension = "pdf" +responseType = "application/pdf" + +types_to_attach = ['ip-dst', 'url', 'domain'] +objects_to_attach = ['domain-ip'] + +headers = """ +:toc: right +:toclevels: 1 +:toc-title: Daily Report +:icons: font +:sectanchors: +:sectlinks: += Daily report by {org_name} +{date} + +:icons: font + +""" + +event_level_tags = """ +IMPORTANT: This event is classified TLP:{value}. + +{expanded} + +""" + +attributes = """ +=== Indicator(s) of compromise + +{list_attributes} + +""" + +title = """ +== ({internal_id}) {title} + +{summary} + +""" + + +class ReportGenerator(): + def __init__(self): + self.report = '' + + def from_remote(self, event_id): + from pymisp import PyMISP + from keys import misp_url, misp_key, misp_verifycert + misp = PyMISP(misp_url, misp_key, misp_verifycert) + result = misp.get(event_id) + self.misp_event = MISPEvent() + self.misp_event.load(result) + + def from_event(self, event): + self.misp_event = MISPEvent() + self.misp_event.load(event) + + def attributes(self): + if not self.misp_event.attributes: + return '' + list_attributes = [] + for attribute in self.misp_event.attributes: + if attribute.type in types_to_attach: + list_attributes.append("* {}".format(attribute.value)) + for obj in self.misp_event.Object: + if obj.name in objects_to_attach: + for attribute in obj.Attribute: + if attribute.type in types_to_attach: + list_attributes.append("* {}".format(attribute.value)) + return attributes.format(list_attributes="\n".join(list_attributes)) + + def _get_tag_info(self, machinetag): + return self.taxonomies.revert_machinetag(machinetag) + + def report_headers(self): + content = {'org_name': 'name', + 'date': date.today().isoformat()} + self.report += headers.format(**content) + + def event_level_tags(self): + if not self.misp_event.Tag: + return '' + for tag in self.misp_event.Tag: + # Only look for TLP for now + if tag['name'].startswith('tlp'): + tax, predicate = self._get_tag_info(tag['name']) + return self.event_level_tags.format(value=predicate.predicate.upper(), expanded=predicate.expanded) + + def title(self): + internal_id = '' + summary = '' + # Get internal refs for report + if not hasattr(self.misp_event, 'Object'): + return '' + for obj in self.misp_event.Object: + if obj.name != 'report': + continue + for a in obj.Attribute: + if a.object_relation == 'case-number': + internal_id = a.value + if a.object_relation == 'summary': + summary = a.value + + return title.format(internal_id=internal_id, title=self.misp_event.info, + summary=summary) + + def asciidoc(self, lang='en'): + self.report += self.title() + self.report += self.event_level_tags() + self.report += self.attributes() + + +def handler(q=False): + if q is False: + return False + + request = json.loads(q) + + if 'data' not in request: + return False + + for evt in request['data']: + report = ReportGenerator() + report.report_headers() + report.from_event(evt) + report.asciidoc() + + command_line = 'asciidoctor-pdf -' + args = shlex.split(command_line) + with subprocess.Popen(args, stdout=subprocess.PIPE, stdin=subprocess.PIPE) as process: + cmd_out, cmd_err = process.communicate(input=report.report.encode('utf-8')) + return {'response': [], 'data': str(base64.b64encode(cmd_out), 'utf-8')} + + +def introspection(): + modulesetup = {} + try: + responseType + modulesetup['responseType'] = responseType + except NameError: + pass + + try: + userConfig + modulesetup['userConfig'] = userConfig + except NameError: + pass + try: + outputFileExtension + modulesetup['outputFileExtension'] = outputFileExtension + except NameError: + pass + try: + inputSource + modulesetup['inputSource'] = inputSource + except NameError: + pass + return modulesetup + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo From 32958324ca70ec1b92d610b03249b41a68e662c7 Mon Sep 17 00:00:00 2001 From: Jericho Date: Thu, 16 Nov 2017 23:04:41 -0700 Subject: [PATCH 02/66] minor touch-ups on error messages for user friendliness --- misp_modules/modules/expansion/vulndb.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/misp_modules/modules/expansion/vulndb.py b/misp_modules/modules/expansion/vulndb.py index 1b97b56..6476199 100644 --- a/misp_modules/modules/expansion/vulndb.py +++ b/misp_modules/modules/expansion/vulndb.py @@ -43,12 +43,12 @@ def handler(q=False): # Only continue if we have a vulnerability attribute if not request.get('vulnerability'): - misperrors['error'] = 'Vulnerability id missing for VulnDB' + misperrors['error'] = 'Vulnerability ID missing for VulnDB.' return misperrors vulnerability = request.get('vulnerability') if request["config"].get("apikey") is None or request["config"].get("apisecret") is None: - misperrors["error"] = "Missing API key or secret value for VulnDB" + misperrors["error"] = "Missing API key or secret value for VulnDB." return misperrors apikey = request["config"].get("apikey") apisecret = request["config"].get("apisecret") @@ -90,7 +90,7 @@ def handler(q=False): if content_json: if 'error' in content_json: - misperrors["error"] = "No CVE information found" + misperrors["error"] = "No CVE information found." return misperrors else: output = {'results': list()} @@ -266,7 +266,7 @@ def handler(q=False): output['results'] += [{'types': 'cpe', 'values': values_cpe }] return output else: - misperrors["error"] = "No information retrieved from VulnDB" + misperrors["error"] = "No information retrieved from VulnDB." return misperrors except: misperrors["error"] = "Error while fetching information from VulnDB, wrong API keys?" From 6d7f041dfe1b18b9003eecf0ab29f5830fa6ff2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 20 Nov 2017 14:55:47 +0100 Subject: [PATCH 03/66] chg: dnspython3 has been superseded by the regular dnspython kit. --- REQUIREMENTS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/REQUIREMENTS b/REQUIREMENTS index 0665aac..fc998d9 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -1,7 +1,7 @@ stix cybox tornado -dnspython3 +dnspython requests urlarchiver passivetotal From 5a546dac345409ccd198796b0c5cc4c27662a131 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 20 Nov 2017 14:56:08 +0100 Subject: [PATCH 04/66] chg: Add new dependency (oauth2) --- REQUIREMENTS | 1 + 1 file changed, 1 insertion(+) diff --git a/REQUIREMENTS b/REQUIREMENTS index fc998d9..3b31b16 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -20,3 +20,4 @@ SPARQLWrapper domaintools_api pygeoip bs4 +oauth2 From f30cf99ff9f9b76001e5621f004c61166226a5cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 20 Nov 2017 15:06:24 +0100 Subject: [PATCH 05/66] fix: Make travis happy. --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index c70bed3..7a39758 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,13 +19,13 @@ install: - pip install . script: - - coverage run -m --parallel-mode --source=misp_modules misp_modules.__init__ & + - coverage run -m --parallel-mode --source=misp_modules misp_modules.__init__ -l 127.0.0.1 & - pid=$! - sleep 5 - nosetests --with-coverage --cover-package=misp_modules - kill -s INT $pid - pushd ~/ - - coverage run -m --parallel-mode --source=misp_modules misp_modules.__init__ -s & + - coverage run -m --parallel-mode --source=misp_modules misp_modules.__init__ -s -l 127.0.0.1 & - pid=$! - popd - sleep 5 From 0ec8339d7ae40e2e97749aa391bd2cb97aa3d707 Mon Sep 17 00:00:00 2001 From: Christophe Vandeplas Date: Tue, 5 Dec 2017 16:41:41 +0100 Subject: [PATCH 06/66] New Farsight DNSDB Passive DNS expansion module --- README.md | 1 + misp_modules/modules/expansion/__init__.py | 2 +- .../modules/expansion/_dnsdb_query/COPYRIGHT | 27 ++ .../modules/expansion/_dnsdb_query/LICENSE | 202 +++++++++++ .../modules/expansion/_dnsdb_query/README.md | 162 +++++++++ .../expansion/_dnsdb_query/dnsdb_query.py | 323 ++++++++++++++++++ .../modules/expansion/farsight_passivedns.py | 72 ++++ 7 files changed, 788 insertions(+), 1 deletion(-) create mode 100755 misp_modules/modules/expansion/_dnsdb_query/COPYRIGHT create mode 100755 misp_modules/modules/expansion/_dnsdb_query/LICENSE create mode 100755 misp_modules/modules/expansion/_dnsdb_query/README.md create mode 100755 misp_modules/modules/expansion/_dnsdb_query/dnsdb_query.py create mode 100755 misp_modules/modules/expansion/farsight_passivedns.py diff --git a/README.md b/README.md index 1eafb9b..f630b7b 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/ * [DNS](misp_modules/modules/expansion/dns.py) - a simple module to resolve MISP attributes like hostname and domain to expand IP addresses attributes. * [DomainTools](misp_modules/modules/expansion/domaintools.py) - a hover and expansion module to get information from [DomainTools](http://www.domaintools.com/) whois. * [EUPI](misp_modules/modules/expansion/eupi.py) - a hover and expansion module to get information about an URL from the [Phishing Initiative project](https://phishing-initiative.eu/?lang=en). +* [Farsight DNSDB Passive DNS](misp_modules/modules/expansion/farsight_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information. * [GeoIP](misp_modules/modules/expansion/geoip_country.py) - a hover and expansion module to get GeoIP information from geolite/maxmind. * [IPASN](misp_modules/modules/expansion/ipasn.py) - a hover and expansion to get the BGP ASN of an IP address. * [iprep](misp-modules/modules/expansion/iprep.py) - an expansion module to get IP reputation from packetmail.net. diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py index 48117e1..231d30e 100644 --- a/misp_modules/modules/expansion/__init__.py +++ b/misp_modules/modules/expansion/__init__.py @@ -1,6 +1,6 @@ from . import _vmray __all__ = ['vmray_submit', 'asn_history', 'circl_passivedns', 'circl_passivessl', - 'countrycode', 'cve', 'dns', 'domaintools', 'eupi', 'ipasn', 'passivetotal', 'sourcecache', + 'countrycode', 'cve', 'dns', 'domaintools', 'eupi', 'farsight_passivedns', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal', 'whois', 'shodan', 'reversedns', 'geoip_country', 'wiki', 'iprep', 'threatminer' ,'otx', 'threatcrowd','vulndb'] diff --git a/misp_modules/modules/expansion/_dnsdb_query/COPYRIGHT b/misp_modules/modules/expansion/_dnsdb_query/COPYRIGHT new file mode 100755 index 0000000..9606e4a --- /dev/null +++ b/misp_modules/modules/expansion/_dnsdb_query/COPYRIGHT @@ -0,0 +1,27 @@ +Copyright (c) 2013 by Farsight Security, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Copyright (c) 2010-2012 by Internet Systems Consortium, Inc. ("ISC") + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT +OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/misp_modules/modules/expansion/_dnsdb_query/LICENSE b/misp_modules/modules/expansion/_dnsdb_query/LICENSE new file mode 100755 index 0000000..d645695 --- /dev/null +++ b/misp_modules/modules/expansion/_dnsdb_query/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/misp_modules/modules/expansion/_dnsdb_query/README.md b/misp_modules/modules/expansion/_dnsdb_query/README.md new file mode 100755 index 0000000..af97937 --- /dev/null +++ b/misp_modules/modules/expansion/_dnsdb_query/README.md @@ -0,0 +1,162 @@ +dnsdb-query +=========== + +These clients are reference implementations of the [DNSDB HTTP API](https://api.dnsdb.info/). Output is +compliant with the [Passive DNS Common Output Format](http://tools.ietf.org/html/draft-dulaunoy-kaplan-passive-dns-cof-01). + +Please see https://www.dnsdb.info/ for more information. + +Requirements +------------ + * Linux, BSD, OS X + * Curl + * Python 2.7.x + * Farsight DNSDB API key + +Installation +------------ +1. Create a directory + + ``` + mkdir ~/dnsdb + ``` +1. Download the software + + ``` + curl https://codeload.github.com/dnsdb/dnsdb-query/tar.gz/debian/0.2-1 -o ~/dnsdb/0.2-1.tar.gz + ``` +1. Extract the software + + ``` + tar xzvf ~/dnsdb/0.2-1.tar.gz -C ~/dnsdb/ --strip-components=1 + ``` +1. Create a API key file + + ``` + nano ~/.dnsdb-query.conf + ``` +1. Cut and paste the following and replace '\' with your API Key + + ``` + APIKEY="" + ``` +1. Test the Python client + + ``` + $ python dnsdb/dnsdb_query.py -i 104.244.13.104 + ``` + ``` + ... + www.farsightsecurity.com. IN A 104.244.13.104 + ``` + +dnsdb_query.py +-------------- + +dnsdb_query.py is a Python client for the DNSDB HTTP API. It is similar +to the dnsdb-query shell script but supports some additional features +like sorting and setting the result limit parameter. It is also embeddable +as a Python module. + +``` +Usage: dnsdb_query.py [options] + +Options: + -h, --help show this help message and exit + -c CONFIG, --config=CONFIG + config file + -r RRSET, --rrset=RRSET + rrset [/[/BAILIWICK]] + -n RDATA_NAME, --rdataname=RDATA_NAME + rdata name [/] + -i RDATA_IP, --rdataip=RDATA_IP + rdata ip + -s SORT, --sort=SORT sort key + -R, --reverse reverse sort + -j, --json output in JSON format + -l LIMIT, --limit=LIMIT + limit number of results + --before=BEFORE only output results seen before this time + --after=AFTER only output results seen after this time + +Time formats are: "%Y-%m-%d", "%Y-%m-%d %H:%M:%S", "%d" (UNIX timestamp), +"-%d" (Relative time in seconds), BIND format relative timestamp (e.g. 1w1h, +(w)eek, (d)ay, (h)our, (m)inute, (s)econd) +``` + +Or, from Python: + +``` +from dnsdb_query import DnsdbClient + +server='https://api.dnsdb.info' +apikey='d41d8cd98f00b204e9800998ecf8427e' + +client = DnsdbClient(server,apikey) +for rrset in client.query_rrset('www.dnsdb.info'): + # rrset is a decoded JSON blob + print repr(rrset) +``` + +Other configuration options that may be set: + +`DNSDB_SERVER` +The base URL of the DNSDB HTTP API, minus the /lookup component. Defaults to +`https://api.dnsdb.info.` + +`HTTP_PROXY` +The URL of the HTTP proxy that you wish to use. + +`HTTPS_PROXY` +The URL of the HTTPS proxy that you wish to use. + +dnsdb-query +----------- + +dnsdb-query is a simple curl-based wrapper for the DNSDB HTTP API. + +The script sources the config file `/etc/dnsdb-query.conf` as a shell fragment. +If the config file is not present in `/etc`, the file `$HOME/.dnsdb-query.conf` +is sourced instead. + +The config file MUST set the value of the APIKEY shell variable to the API +key provided to you by Farsight Security. + +For example, if your API key is d41d8cd98f00b204e9800998ecf8427e, place the +following line in `/etc/dnsdb-query.conf` or `$HOME/.dnsdb-query.conf`: + +``` +APIKEY="d41d8cd98f00b204e9800998ecf8427e" +``` + +Other shell variables that may be set via the config file or command line +are: + +`DNSDB_SERVER` +The base URL of the DNSDB HTTP API, minus the /lookup component. Defaults to +`https://api.dnsdb.info.` + +`DNSDB_FORMAT` +The result format to use, either text or json. Defaults to text. + +`HTTP_PROXY` +The URL of the HTTP proxy that you wish to use. + +`HTTPS_PROXY` +The URL of the HTTPS proxy that you wish to use. + +dnsdb-query supports the following usages: + +``` +Usage: dnsdb-query rrset [/[/]] +Usage: dnsdb-query rdata ip +Usage: dnsdb-query rdata name [/] +Usage: dnsdb-query rdata raw [/] +``` + +If your rrname, bailiwick or rdata contains the `/` character you +will need to escape it to `%2F` on the command line. eg: + +`./dnsdb_query -r 1.0%2F1.0.168.192.in-addr.arpa` + +retrieves the rrsets for `1.0/1.0.168.192.in-addr.arpa`. diff --git a/misp_modules/modules/expansion/_dnsdb_query/dnsdb_query.py b/misp_modules/modules/expansion/_dnsdb_query/dnsdb_query.py new file mode 100755 index 0000000..0ab58e8 --- /dev/null +++ b/misp_modules/modules/expansion/_dnsdb_query/dnsdb_query.py @@ -0,0 +1,323 @@ +#!/usr/bin/env python +# +# Note: This file is NOT the official one from dnsdb, as it has a python3 cherry-picked pull-request applied for python3 compatibility +# See https://github.com/dnsdb/dnsdb-query/pull/30 +# +# Copyright (c) 2013 by Farsight Security, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import print_function + +import calendar +import errno +import locale +import optparse +import os +import re +import sys +import time +import json +from io import StringIO + +try: + from urllib2 import build_opener, Request, ProxyHandler, HTTPError, URLError + from urllib import quote as urllib_quote, urlencode +except ImportError: + from urllib.request import build_opener, Request, ProxyHandler, HTTPError, URLError + from urllib.parse import quote as urllib_quote, urlencode + + +DEFAULT_CONFIG_FILES = filter(os.path.isfile, ('/etc/dnsdb-query.conf', os.path.expanduser('~/.dnsdb-query.conf'))) +DEFAULT_DNSDB_SERVER = 'https://api.dnsdb.info' +DEFAULT_HTTP_PROXY = '' +DEFAULT_HTTPS_PROXY = '' + +cfg = None +options = None + +locale.setlocale(locale.LC_ALL, '') + +class QueryError(Exception): + pass + +class DnsdbClient(object): + def __init__(self, server, apikey, limit=None, http_proxy=None, https_proxy=None): + self.server = server + self.apikey = apikey + self.limit = limit + self.http_proxy = http_proxy + self.https_proxy = https_proxy + + def query_rrset(self, oname, rrtype=None, bailiwick=None, before=None, after=None): + if bailiwick: + if not rrtype: + rrtype = 'ANY' + path = 'rrset/name/%s/%s/%s' % (quote(oname), rrtype, quote(bailiwick)) + elif rrtype: + path = 'rrset/name/%s/%s' % (quote(oname), rrtype) + else: + path = 'rrset/name/%s' % quote(oname) + return self._query(path, before, after) + + def query_rdata_name(self, rdata_name, rrtype=None, before=None, after=None): + if rrtype: + path = 'rdata/name/%s/%s' % (quote(rdata_name), rrtype) + else: + path = 'rdata/name/%s' % quote(rdata_name) + return self._query(path, before, after) + + def query_rdata_ip(self, rdata_ip, before=None, after=None): + path = 'rdata/ip/%s' % rdata_ip.replace('/', ',') + return self._query(path, before, after) + + def _query(self, path, before=None, after=None): + res = [] + url = '%s/lookup/%s' % (self.server, path) + + params = {} + if self.limit: + params['limit'] = self.limit + if before and after: + params['time_first_after'] = after + params['time_last_before'] = before + else: + if before: + params['time_first_before'] = before + if after: + params['time_last_after'] = after + if params: + url += '?{0}'.format(urlencode(params)) + + req = Request(url) + req.add_header('Accept', 'application/json') + req.add_header('X-Api-Key', self.apikey) + + proxy_args = {} + if self.http_proxy: + proxy_args['http'] = self.http_proxy + if self.https_proxy: + proxy_args['https'] = self.https_proxy + proxy_handler = ProxyHandler(proxy_args) + opener = build_opener(proxy_handler) + + try: + http = opener.open(req) + while True: + line = http.readline() + if not line: + break + yield json.loads(line.decode('ascii')) + except (HTTPError, URLError) as e: + raise QueryError(str(e), sys.exc_traceback) + +def quote(path): + return urllib_quote(path, safe='') + +def sec_to_text(ts): + return time.strftime('%Y-%m-%d %H:%M:%S -0000', time.gmtime(ts)) + +def rrset_to_text(m): + s = StringIO() + + try: + if 'bailiwick' in m: + s.write(';; bailiwick: %s\n' % m['bailiwick']) + + if 'count' in m: + s.write(';; count: %s\n' % locale.format('%d', m['count'], True)) + + if 'time_first' in m: + s.write(';; first seen: %s\n' % sec_to_text(m['time_first'])) + if 'time_last' in m: + s.write(';; last seen: %s\n' % sec_to_text(m['time_last'])) + + if 'zone_time_first' in m: + s.write(';; first seen in zone file: %s\n' % sec_to_text(m['zone_time_first'])) + if 'zone_time_last' in m: + s.write(';; last seen in zone file: %s\n' % sec_to_text(m['zone_time_last'])) + + if 'rdata' in m: + for rdata in m['rdata']: + s.write('%s IN %s %s\n' % (m['rrname'], m['rrtype'], rdata)) + + s.seek(0) + return s.read() + finally: + s.close() + +def rdata_to_text(m): + return '%s IN %s %s' % (m['rrname'], m['rrtype'], m['rdata']) + +def parse_config(cfg_files): + config = {} + + if not cfg_files: + raise IOError(errno.ENOENT, 'dnsdb_query: No config files found') + + for fname in cfg_files: + for line in open(fname): + key, eq, val = line.strip().partition('=') + val = val.strip('"') + config[key] = val + + return config + +def time_parse(s): + try: + epoch = int(s) + return epoch + except ValueError: + pass + + try: + epoch = int(calendar.timegm(time.strptime(s, '%Y-%m-%d'))) + return epoch + except ValueError: + pass + + try: + epoch = int(calendar.timegm(time.strptime(s, '%Y-%m-%d %H:%M:%S'))) + return epoch + except ValueError: + pass + + m = re.match(r'^(?=\d)(?:(\d+)w)?(?:(\d+)d)?(?:(\d+)h)?(?:(\d+)m)?(?:(\d+)s?)?$', s, re.I) + if m: + return -1*(int(m.group(1) or 0)*604800 + + int(m.group(2) or 0)*86400+ + int(m.group(3) or 0)*3600+ + int(m.group(4) or 0)*60+ + int(m.group(5) or 0)) + + raise ValueError('Invalid time: "%s"' % s) + +def epipe_wrapper(func): + def f(*args, **kwargs): + try: + return func(*args, **kwargs) + except IOError as e: + if e.errno == errno.EPIPE: + sys.exit(e.errno) + raise + return f + +@epipe_wrapper +def main(): + global cfg + global options + + parser = optparse.OptionParser(epilog='Time formats are: "%Y-%m-%d", "%Y-%m-%d %H:%M:%S", "%d" (UNIX timestamp), "-%d" (Relative time in seconds), BIND format (e.g. 1w1h, (w)eek, (d)ay, (h)our, (m)inute, (s)econd)') + parser.add_option('-c', '--config', dest='config', + help='config file', action='append') + parser.add_option('-r', '--rrset', dest='rrset', type='string', + help='rrset [/[/BAILIWICK]]') + parser.add_option('-n', '--rdataname', dest='rdata_name', type='string', + help='rdata name [/]') + parser.add_option('-i', '--rdataip', dest='rdata_ip', type='string', + help='rdata ip ') + parser.add_option('-t', '--rrtype', dest='rrtype', type='string', + help='rrset or rdata rrtype') + parser.add_option('-b', '--bailiwick', dest='bailiwick', type='string', + help='rrset bailiwick') + parser.add_option('-s', '--sort', dest='sort', type='string', help='sort key') + parser.add_option('-R', '--reverse', dest='reverse', action='store_true', default=False, + help='reverse sort') + parser.add_option('-j', '--json', dest='json', action='store_true', default=False, + help='output in JSON format') + parser.add_option('-l', '--limit', dest='limit', type='int', default=0, + help='limit number of results') + + parser.add_option('', '--before', dest='before', type='string', help='only output results seen before this time') + parser.add_option('', '--after', dest='after', type='string', help='only output results seen after this time') + + options, args = parser.parse_args() + if args: + parser.print_help() + sys.exit(1) + + try: + if options.before: + options.before = time_parse(options.before) + except ValueError: + print('Could not parse before: {}'.format(options.before)) + + try: + if options.after: + options.after = time_parse(options.after) + except ValueError: + print('Could not parse after: {}'.format(options.after)) + + try: + cfg = parse_config(options.config or DEFAULT_CONFIG_FILES) + except IOError as e: + print(str(e), file=sys.stderr) + sys.exit(1) + + if not 'DNSDB_SERVER' in cfg: + cfg['DNSDB_SERVER'] = DEFAULT_DNSDB_SERVER + if not 'HTTP_PROXY' in cfg: + cfg['HTTP_PROXY'] = DEFAULT_HTTP_PROXY + if not 'HTTPS_PROXY' in cfg: + cfg['HTTPS_PROXY'] = DEFAULT_HTTPS_PROXY + if not 'APIKEY' in cfg: + sys.stderr.write('dnsdb_query: APIKEY not defined in config file\n') + sys.exit(1) + + client = DnsdbClient(cfg['DNSDB_SERVER'], cfg['APIKEY'], + limit=options.limit, + http_proxy=cfg['HTTP_PROXY'], + https_proxy=cfg['HTTPS_PROXY']) + if options.rrset: + if options.rrtype or options.bailiwick: + qargs = (options.rrset, options.rrtype, options.bailiwick) + else: + qargs = (options.rrset.split('/', 2)) + + results = client.query_rrset(*qargs, before=options.before, after=options.after) + fmt_func = rrset_to_text + elif options.rdata_name: + if options.rrtype: + qargs = (options.rdata_name, options.rrtype) + else: + qargs = (options.rdata_name.split('/', 1)) + + results = client.query_rdata_name(*qargs, before=options.before, after=options.after) + fmt_func = rdata_to_text + elif options.rdata_ip: + results = client.query_rdata_ip(options.rdata_ip, before=options.before, after=options.after) + fmt_func = rdata_to_text + else: + parser.print_help() + sys.exit(1) + + if options.json: + fmt_func = json.dumps + + try: + if options.sort: + results = list(results) + if len(results) > 0: + if not options.sort in results[0]: + sort_keys = results[0].keys() + sort_keys.sort() + sys.stderr.write('dnsdb_query: invalid sort key "%s". valid sort keys are %s\n' % (options.sort, ', '.join(sort_keys))) + sys.exit(1) + results.sort(key=lambda r: r[options.sort], reverse=options.reverse) + for res in results: + sys.stdout.write('%s\n' % fmt_func(res)) + except QueryError as e: + print(e.message, file=sys.stderr) + sys.exit(1) + +if __name__ == '__main__': + main() diff --git a/misp_modules/modules/expansion/farsight_passivedns.py b/misp_modules/modules/expansion/farsight_passivedns.py new file mode 100755 index 0000000..7c0ad23 --- /dev/null +++ b/misp_modules/modules/expansion/farsight_passivedns.py @@ -0,0 +1,72 @@ +import json +from ._dnsdb_query.dnsdb_query import DnsdbClient + + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['hostname', 'domain', 'ip-src', 'ip-dst'], 'output': ['freetext']} +moduleinfo = {'version': '0.1', 'author': 'Christophe Vandeplas', 'description': 'Module to access Farsight DNSDB Passive DNS', 'module-type': ['expansion', 'hover']} +moduleconfig = ['apikey'] + +server = 'https://api.dnsdb.info' + +# TODO return a MISP object with the different attributes + + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + if (request.get('config')): + if (request['config'].get('apikey') is None): + misperrors['error'] = 'Farsight DNSDB apikey is missing' + return misperrors + client = DnsdbClient(server, request['config']['apikey']) + if request.get('hostname'): + res = lookup_name(client, request['hostname']) + elif request.get('domain'): + res = lookup_name(client, request['domain']) + elif request.get('ip-src'): + res = lookup_ip(client, request['ip-src']) + elif request.get('ip-dst'): + res = lookup_ip(client, request['ip-dst']) + else: + misperrors['error'] = "Unsupported attributes type" + return misperrors + + out = '' + for v in set(res): # uniquify entries + out = out + "{} ".format(v) + r = {'results': [{'types': mispattributes['output'], 'values': out}]} + return r + + +def lookup_name(client, name): + res = client.query_rrset(name) # RRSET = entries in the left-hand side of the domain name related labels + for item in res: + if item.get('rrtype') in ['A', 'AAAA', 'CNAME']: + for i in item.get('rdata'): + yield(i.rstrip('.')) + if item.get('rrtype') in ['SOA']: + for i in item.get('rdata'): + # grab email field and replace first dot by @ to convert to an email address + yield(i.split(' ')[1].rstrip('.').replace('.', '@', 1)) + res = client.query_rdata_name(name) # RDATA = entries on the right-hand side of the domain name related labels + for item in res: + if item.get('rrtype') in ['A', 'AAAA', 'CNAME']: + yield(item.get('rrname').rstrip('.')) + + +def lookup_ip(client, ip): + res = client.query_rdata_ip(ip) + for item in res: + print(item) + yield(item['rrname'].rstrip('.')) + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo From 4cdb143733c22c80802e8cdd8aad3f7bc8754ab8 Mon Sep 17 00:00:00 2001 From: Christophe Vandeplas Date: Wed, 6 Dec 2017 09:23:44 +0100 Subject: [PATCH 07/66] fixes missing init file in dnsdb library folder --- misp_modules/modules/expansion/_dnsdb_query/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 misp_modules/modules/expansion/_dnsdb_query/__init__.py diff --git a/misp_modules/modules/expansion/_dnsdb_query/__init__.py b/misp_modules/modules/expansion/_dnsdb_query/__init__.py new file mode 100644 index 0000000..e69de29 From 49d5520fa3c246e0e1e66bdc057493586289c855 Mon Sep 17 00:00:00 2001 From: Robert Nixon Date: Mon, 8 Jan 2018 11:01:16 -0500 Subject: [PATCH 08/66] Added threatStream_misp_export.py --- .../export_mod/threatStream_misp_export.py | 110 ++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 misp_modules/modules/export_mod/threatStream_misp_export.py diff --git a/misp_modules/modules/export_mod/threatStream_misp_export.py b/misp_modules/modules/export_mod/threatStream_misp_export.py new file mode 100644 index 0000000..2729de1 --- /dev/null +++ b/misp_modules/modules/export_mod/threatStream_misp_export.py @@ -0,0 +1,110 @@ +""" +Export module for coverting MISP events into ThreatStream Structured Import files. Based of work by the CenturyLink CIRT. +Source: https://github.com/MISP/misp-modules/blob/master/misp_modules/modules/export_mod/threat_connect_export.py +""" + +import base64 +import csv +import io +import json +import logging + + +misperrors = {"error": "Error"} + +moduleinfo = { + "version": "1.0", + "author": "Robert Nixon, based off of the ThreatConnect MISP Module written by the CenturyLink CIRT", + "description": "Export a structured CSV file for uploading to ThreatStream", + "module-type": ["export"] +} + + +moduleconfig = ["Default_Source"] + + +# Map of MISP fields => ThreatStream itypes +fieldmap = { + "domain": "mal_domain", + "hostname": "mal_domain", + "ip-src": "mal_ip", + "ip-dst": "mal_ip", + "email-src": "phish_email", + "url": "mal_url", + "md5": "mal_md5", +} + +# combine all the MISP fields from fieldmap into one big list +mispattributes = { + "input": list(fieldmap.keys()) +} + + +def handler(q=False): + """ + Convert a MISP query into a CSV file matching the ThreatStream Structured Import file format. + Input + q: Query dictionary + """ + if q is False or not q: + return False + + # Check if we were given a configuration + request = json.loads(q) + config = request.get("config", {"Default_Source": ""}) + logging.info("Setting config to: %s", config) + + response = io.StringIO() + writer = csv.DictWriter(response, fieldnames=["value", "itype", "tags"]) + writer.writeheader() + + # start parsing MISP data + for event in request["data"]: + for attribute in event["Attribute"]: + if attribute["type"] in mispattributes["input"]: + logging.debug("Adding %s to structured CSV export of ThreatStream Export", attribute["value"]) + if "|" in attribute["type"]: + # if the attribute type has multiple values, line it up with the corresponding ThreatStream values in fieldmap + indicators = tuple(attribute["value"].split("|")) + ts_types = tuple(fieldmap[attribute["type"]].split("|")) + for i, indicator in enumerate(indicators): + writer.writerow({ + "value": indicator, + "itype": ts_types[i], + "tags": attribute["comment"] + }) + else: + writer.writerow({ + "itype": fieldmap[attribute["type"]], + "value": attribute["value"], + "tags": attribute["comment"] + }) + + return {"response": [], "data": str(base64.b64encode(bytes(response.getvalue(), 'utf-8')), 'utf-8')} + + +def introspection(): + """ + Relay the supported attributes to MISP. + No Input + Output + Dictionary of supported MISP attributes + """ + modulesetup = { + "responseType": "application/txt", + "outputFileExtension": "csv", + "userConfig": {}, + "inputSource": [] + } + return modulesetup + + +def version(): + """ + Relay module version and associated metadata to MISP. + No Input + Output + moduleinfo: metadata output containing all potential configuration values + """ + moduleinfo["config"] = moduleconfig + return moduleinfo From 1d2f3d9c3c746b9af22959762ed09b981f7963e8 Mon Sep 17 00:00:00 2001 From: Robert Nixon Date: Mon, 8 Jan 2018 11:03:42 -0500 Subject: [PATCH 09/66] Updated __init__.py Added reference to new ThreatStream export module --- misp_modules/modules/export_mod/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/export_mod/__init__.py b/misp_modules/modules/export_mod/__init__.py index 1932b6f..3d5b5c2 100644 --- a/misp_modules/modules/export_mod/__init__.py +++ b/misp_modules/modules/export_mod/__init__.py @@ -1 +1 @@ -__all__ = ['testexport','cef_export','liteexport','threat_connect_export', 'pdfexport'] +__all__ = ['testexport','cef_export','liteexport','threat_connect_export','pdfexport','threatStream_misp_export.py'] From 85f1a9bd9157c527e23585b34a463e6cc2aa44a6 Mon Sep 17 00:00:00 2001 From: Robert Nixon Date: Mon, 8 Jan 2018 12:09:23 -0500 Subject: [PATCH 10/66] Update threatStream_misp_export.py --- .../modules/export_mod/threatStream_misp_export.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/misp_modules/modules/export_mod/threatStream_misp_export.py b/misp_modules/modules/export_mod/threatStream_misp_export.py index 2729de1..3fd88c8 100644 --- a/misp_modules/modules/export_mod/threatStream_misp_export.py +++ b/misp_modules/modules/export_mod/threatStream_misp_export.py @@ -20,10 +20,10 @@ moduleinfo = { } -moduleconfig = ["Default_Source"] +moduleconfig = [] -# Map of MISP fields => ThreatStream itypes +# Map of MISP fields => ThreatStream itypes, you can modify this to your liking fieldmap = { "domain": "mal_domain", "hostname": "mal_domain", @@ -49,10 +49,9 @@ def handler(q=False): if q is False or not q: return False - # Check if we were given a configuration + request = json.loads(q) - config = request.get("config", {"Default_Source": ""}) - logging.info("Setting config to: %s", config) + response = io.StringIO() writer = csv.DictWriter(response, fieldnames=["value", "itype", "tags"]) From 5c4df3075e559d5357fd4613621b319d9525fc7a Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Mon, 8 Jan 2018 20:31:26 +0100 Subject: [PATCH 11/66] Fix the __init__ import --- misp_modules/modules/export_mod/__init__.py | 2 +- misp_modules/modules/export_mod/threatStream_misp_export.py | 0 2 files changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 100755 misp_modules/modules/export_mod/threatStream_misp_export.py diff --git a/misp_modules/modules/export_mod/__init__.py b/misp_modules/modules/export_mod/__init__.py index 3d5b5c2..fff02d7 100644 --- a/misp_modules/modules/export_mod/__init__.py +++ b/misp_modules/modules/export_mod/__init__.py @@ -1 +1 @@ -__all__ = ['testexport','cef_export','liteexport','threat_connect_export','pdfexport','threatStream_misp_export.py'] +__all__ = ['testexport','cef_export','liteexport','threat_connect_export','pdfexport','threatStream_misp_export'] diff --git a/misp_modules/modules/export_mod/threatStream_misp_export.py b/misp_modules/modules/export_mod/threatStream_misp_export.py old mode 100644 new mode 100755 From 9734f59b020259e42ba9baa3ca7bdb071d19970e Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Mon, 8 Jan 2018 20:45:30 +0100 Subject: [PATCH 12/66] Added ThreatStream and PDF export --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index f630b7b..6dd06ce 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,9 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/ * [CEF](misp_modules/modules/export_mod/cef_export.py) module to export Common Event Format (CEF). * [Lite Export](misp_modules/modules/export_mod/liteexport.py) module to export a lite event. +* [Simple PDF export](misp_modules/modules/export_mod/pdfexport.py) module to export in PDF (required: asciidoctor-pdf). * [ThreatConnect](misp_modules/modules/export_mod/threat_connect_export.py) module to export in ThreatConnect CSV format. +* [ThreatStream](misp_modules/modules/export_mod/threatStream_misp_export.py) module to export in ThreatStream format. ### Import modules From 46975f4f16e6ebfee4127809401131549d76d257 Mon Sep 17 00:00:00 2001 From: Christophe Vandeplas Date: Tue, 16 Jan 2018 11:05:26 +0100 Subject: [PATCH 13/66] Added ThreatAnalyzer sandbox import Experimental module - some parts should be migrated to --- README.md | 9 +- misp_modules/modules/import_mod/__init__.py | 2 +- .../import_mod/threatanalyzer_import.py | 507 ++++++++++++++++++ 3 files changed, 513 insertions(+), 5 deletions(-) create mode 100755 misp_modules/modules/import_mod/threatanalyzer_import.py diff --git a/README.md b/README.md index 6dd06ce..339c56f 100644 --- a/README.md +++ b/README.md @@ -18,8 +18,9 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/ ### Expansion modules * [ASN History](misp_modules/modules/expansion/asn_history.py) - a hover and expansion module to expand an AS number with the ASN description and its history. -* [CIRCL Passive SSL](misp_modules/modules/expansion/circl_passivessl.py) - a hover and expansion module to expand IP addresses with the X.509 certificate seen. * [CIRCL Passive DNS](misp_modules/modules/expansion/circl_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information. +* [CIRCL Passive SSL](misp_modules/modules/expansion/circl_passivessl.py) - a hover and expansion module to expand IP addresses with the X.509 certificate seen. +* [countrycode](misp_modules/modules/expansion/countrycode.py) - a hover module to tell you what country a URL belongs to. * [CVE](misp_modules/modules/expansion/cve.py) - a hover module to give more information about a vulnerability (CVE). * [DNS](misp_modules/modules/expansion/dns.py) - a simple module to resolve MISP attributes like hostname and domain to expand IP addresses attributes. * [DomainTools](misp_modules/modules/expansion/domaintools.py) - a hover and expansion module to get information from [DomainTools](http://www.domaintools.com/) whois. @@ -28,13 +29,12 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/ * [GeoIP](misp_modules/modules/expansion/geoip_country.py) - a hover and expansion module to get GeoIP information from geolite/maxmind. * [IPASN](misp_modules/modules/expansion/ipasn.py) - a hover and expansion to get the BGP ASN of an IP address. * [iprep](misp-modules/modules/expansion/iprep.py) - an expansion module to get IP reputation from packetmail.net. +* [OTX](misp_modules/modules/expansion/otx.py) - an expansion module for [OTX](https://otx.alienvault.com/). * [passivetotal](misp_modules/modules/expansion/passivetotal.py) - a [passivetotal](https://www.passivetotal.org/) module that queries a number of different PassiveTotal datasets. * [shodan](misp_modules/modules/expansion/shodan.py) - a minimal [shodan](https://www.shodan.io/) expansion module. * [sourcecache](misp_modules/modules/expansion/sourcecache.py) - a module to cache a specific link from a MISP instance. * [ThreatCrowd](misp_modules/modules/expansion/threatcrowd.py) - an expansion module for [ThreatCrowd](https://www.threatcrowd.org/). -* [OTX](misp_modules/modules/expansion/otx.py) - an expansion module for [OTX](https://otx.alienvault.com/). * [threatminer](misp_modules/modules/expansion/threatminer.py) - an expansion module to expand from [ThreatMiner](https://www.threatminer.org/). -* [countrycode](misp_modules/modules/expansion/countrycode.py) - a hover module to tell you what country a URL belongs to. * [virustotal](misp_modules/modules/expansion/virustotal.py) - an expansion module to pull known resolutions and malware samples related with an IP/Domain from virusTotal (this modules require a VirusTotal private API key) * [wikidata](misp_modules/modules/expansion/wiki.py) - a [wikidata](https://www.wikidata.org) expansion module. * [xforce](misp_modules/modules/expansion/xforceexchange.py) - an IBM X-Force Exchange expansion module. @@ -50,10 +50,11 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/ ### Import modules * [Cuckoo JSON](misp_modules/modules/import_mod/cuckooimport.py) Cuckoo JSON import. +* [Email Import](misp_modules/modules/import_mod/email_import.py) Email import module for MISP to import basic metadata. * [OCR](misp_modules/modules/import_mod/ocr.py) Optical Character Recognition (OCR) module for MISP to import attributes from images, scan or faxes. * [OpenIOC](misp_modules/modules/import_mod/openiocimport.py) OpenIOC import based on PyMISP library. * [stiximport](misp_modules/modules/import_mod/stiximport.py) - An import module to process STIX xml/json. -* [Email Import](misp_modules/modules/import_mod/email_import.py) Email import module for MISP to import basic metadata. +* [ThreatAnalyzer](misp_modules/modules/import_mod/threatanalyzer_import.py) - An import module to process ThreatAnalyzer archive.zip/analysis.json sandbox exports. * [VMRay](misp_modules/modules/import_mod/vmray_import.py) - An import module to process VMRay export. ## How to install and start MISP modules? diff --git a/misp_modules/modules/import_mod/__init__.py b/misp_modules/modules/import_mod/__init__.py index 6beeaa2..e2941d7 100644 --- a/misp_modules/modules/import_mod/__init__.py +++ b/misp_modules/modules/import_mod/__init__.py @@ -1,4 +1,4 @@ from . import _vmray __all__ = ['vmray_import', 'testimport', 'ocr', 'stiximport', 'cuckooimport', - 'email_import', 'mispjson', 'openiocimport'] + 'email_import', 'mispjson', 'openiocimport', 'threatanalyzer_import'] diff --git a/misp_modules/modules/import_mod/threatanalyzer_import.py b/misp_modules/modules/import_mod/threatanalyzer_import.py new file mode 100755 index 0000000..fded508 --- /dev/null +++ b/misp_modules/modules/import_mod/threatanalyzer_import.py @@ -0,0 +1,507 @@ +''' +import +define mandatory + +''' +import json +import base64 +import re +import zipfile +import ipaddress +import io +import logging + +misperrors = {'error': 'Error'} +userConfig = {} +inputSource = ['file'] + +moduleinfo = {'version': '0.6', 'author': 'Christophe Vandeplas', + 'description': 'Import for ThreatAnalyzer archive.zip/analysis.json files', + 'module-type': ['import']} + +moduleconfig = [] +log = logging.getLogger('misp-modules') + +# FIXME - many hardcoded filters should be migrated to import regexes. See also https://github.com/MISP/MISP/issues/2712 +# DISCLAIMER - This module is to be considered as experimental and needs much fine-tuning. +# more can be done with what's in the ThreatAnalyzer archive.zip + + +def handler(q=False): + if q is False: + return False + results = [] + zip_starts = 'PK' + request = json.loads(q) + data = base64.b64decode(request['data']) + + if data[:len(zip_starts)].decode() == zip_starts: + with zipfile.ZipFile(io.BytesIO(data), 'r') as zf: + # unzipped_files = [] + modified_files_mapping = {} + # pre-process some of the files in the zip + for zip_file_name in zf.namelist(): # Get all files in the zip file + # find the filenames of the modified_files + if re.match(r"Analysis/proc_\d+/modified_files/mapping\.log", zip_file_name): + with zf.open(zip_file_name, mode='r', pwd=None) as fp: + file_data = fp.read() + for line in file_data.decode().split('\n'): + if line: + l_fname, l_size, l_md5, l_created = line.split('|') + l_fname = cleanup_filepath(l_fname) + if l_fname: + if l_size == 0: + pass # FIXME create an attribute for the filename/path + else: + # file is a non empty sample, upload the sample later + modified_files_mapping[l_md5] = l_fname + + # now really process the data + for zip_file_name in zf.namelist(): # Get all files in the zip file + # print('Processing file: {}'.format(zip_file_name)) + if re.match(r"Analysis/proc_\d+/modified_files/.+\.", zip_file_name) and "mapping.log" not in zip_file_name: + sample_md5 = zip_file_name.split('/')[-1].split('.')[0] + if sample_md5 in modified_files_mapping: + sample_filename = modified_files_mapping[sample_md5] + # print("{} maps to {}".format(sample_md5, sample_filename)) + with zf.open(zip_file_name, mode='r', pwd=None) as fp: + file_data = fp.read() + results.append({ + 'values': sample_filename, + 'data': base64.b64encode(file_data).decode(), + 'type': 'malware-sample', 'categories': ['Artifacts dropped', 'Payload delivery'], 'to_ids': True, 'comment': ''}) + + if 'Analysis/analysis.json' in zip_file_name: + with zf.open(zip_file_name, mode='r', pwd=None) as fp: + file_data = fp.read() + analysis_json = json.loads(file_data.decode('utf-8')) + results += process_analysis_json(analysis_json) + # if 'sample' in zip_file_name: + # sample['data'] = base64.b64encode(file_data).decode() + + else: + try: + results = process_analysis_json(json.loads(data.decode('utf-8'))) + except ValueError: + log.warning('MISP modules {0} failed: uploaded file is not a zip or json file.'.format(request['module'])) + return {'error': 'Uploaded file is not a zip or json file.'.format(request['module'])} + pass + # keep only unique entries based on the value field + results = list({v['values']: v for v in results}.values()) + r = {'results': results} + return r + + +def process_analysis_json(analysis_json): + if 'analysis' in analysis_json and 'processes' in analysis_json['analysis'] and 'process' in analysis_json['analysis']['processes']: + # if 'analysis' in analysis_json and '@filename' in analysis_json['analysis']: + # sample['values'] = analysis_json['analysis']['@filename'] + for process in analysis_json['analysis']['processes']['process']: + # print_json(process) + if 'connection_section' in process and 'connection' in process['connection_section']: + for connection_section_connection in process['connection_section']['connection']: + + connection_section_connection['@remote_ip'] = cleanup_ip(connection_section_connection['@remote_ip']) + connection_section_connection['@remote_hostname'] = cleanup_hostname(connection_section_connection['@remote_hostname']) + if connection_section_connection['@remote_ip'] and connection_section_connection['@remote_hostname']: + val = '{}|{}'.format(connection_section_connection['@remote_hostname'], + connection_section_connection['@remote_ip']) + # print("connection_section_connection hostname|ip: {}|{} IDS:yes".format( + # connection_section_connection['@remote_hostname'], + # connection_section_connection['@remote_ip']) + # ) + yield({'values': val, 'type': 'domain|ip', 'categories': 'Network activity', 'to_ids': True, 'comment': ''}) + elif connection_section_connection['@remote_ip']: + # print("connection_section_connection ip-dst: {} IDS:yes".format( + # connection_section_connection['@remote_ip']) + # ) + yield({'values': connection_section_connection['@remote_ip'], 'type': 'ip-dst', 'to_ids': True, 'comment': ''}) + elif connection_section_connection['@remote_hostname']: + # print("connection_section_connection hostname: {} IDS:yes".format( + # connection_section_connection['@remote_hostname']) + # ) + yield({'values': connection_section_connection['@remote_hostname'], 'type': 'hostname', 'to_ids': True, 'comment': ''}) + if 'http_command' in connection_section_connection: + for http_command in connection_section_connection['http_command']: + # print('connection_section_connection HTTP COMMAND: {}\t{}'.format( + # http_command['@method'], # comment + # http_command['@url']) # url + # ) + val = cleanup_url(http_command['@url']) + if val: + yield({'values': val, 'type': 'url', 'categories': 'Network activity', 'to_ids': True, 'comment': http_command['@method']}) + + if 'http_header' in connection_section_connection: + for http_header in connection_section_connection['http_header']: + if 'User-Agent:' in http_header['@header']: + val = http_header['@header'][len('User-Agent: '):] + yield({'values': val, 'type': 'user-agent', 'categories': 'Network activity', 'to_ids': False, 'comment': ''}) + elif 'Host:' in http_header['@header']: + val = http_header['@header'][len('Host: '):] + if ':' in val: + try: + val_port = int(val.split(':')[1]) + except ValueError as e: + val_port = False + val_hostname = cleanup_hostname(val.split(':')[0]) + val_ip = cleanup_ip(val.split(':')[0]) + if val_hostname and val_port: + val_combined = '{}|{}'.format(val_hostname, val_port) + # print({'values': val_combined, 'type': 'hostname|port', 'to_ids': True, 'comment': ''}) + yield({'values': val_combined, 'type': 'hostname|port', 'to_ids': True, 'comment': ''}) + elif val_ip and val_port: + val_combined = '{}|{}'.format(val_ip, val_port) + # print({'values': val_combined, 'type': 'ip-dst|port', 'to_ids': True, 'comment': ''}) + yield({'values': val_combined, 'type': 'ip-dst|port', 'to_ids': True, 'comment': ''}) + else: + continue + val_hostname = cleanup_hostname(val) + if val_hostname: + # print({'values': val_hostname, 'type': 'hostname', 'to_ids': True, 'comment': ''}) + yield({'values': val_hostname, 'type': 'hostname', 'to_ids': True, 'comment': ''}) + else: + # LATER header not processed + pass + if 'filesystem_section' in process and 'create_file' in process['filesystem_section']: + for filesystem_section_create_file in process['filesystem_section']['create_file']: + # first skip some items + if filesystem_section_create_file['@create_disposition'] in {'FILE_OPEN_IF'}: + continue + # FIXME - this section is probably not needed considering the 'stored_files stored_created_file' section we process later. + # print('CREATE FILE: {}\t{}'.format( + # filesystem_section_create_file['@srcfile'], # filename + # filesystem_section_create_file['@create_disposition']) # comment - use this to filter out cases + # ) + + if 'networkoperation_section' in process and 'dns_request_by_addr' in process['networkoperation_section']: + for networkoperation_section_dns_request_by_addr in process['networkoperation_section']['dns_request_by_addr']: + # FIXME - it's unclear what this section is for. + # TODO filter this + # print('DNS REQUEST: {}\t{}'.format( + # networkoperation_section_dns_request_by_addr['@request_address'], # ip-dst + # networkoperation_section_dns_request_by_addr['@result_name']) # hostname + # ) # => NOT hostname|ip + pass + if 'networkoperation_section' in process and 'dns_request_by_name' in process['networkoperation_section']: + for networkoperation_section_dns_request_by_name in process['networkoperation_section']['dns_request_by_name']: + networkoperation_section_dns_request_by_name['@request_name'] = cleanup_hostname(networkoperation_section_dns_request_by_name['@request_name'].rstrip('.')) + networkoperation_section_dns_request_by_name['@result_addresses'] = cleanup_ip(networkoperation_section_dns_request_by_name['@result_addresses']) + if networkoperation_section_dns_request_by_name['@request_name'] and networkoperation_section_dns_request_by_name['@result_addresses']: + val = '{}|{}'.format(networkoperation_section_dns_request_by_name['@request_name'], + networkoperation_section_dns_request_by_name['@result_addresses']) + # print("networkoperation_section_dns_request_by_name hostname|ip: {}|{} IDS:yes".format( + # networkoperation_section_dns_request_by_name['@request_name'], + # networkoperation_section_dns_request_by_name['@result_addresses']) + # ) + yield({'values': val, 'type': 'domain|ip', 'categories': 'Network activity', 'to_ids': True, 'comment': ''}) + elif networkoperation_section_dns_request_by_name['@request_name']: + # print("networkoperation_section_dns_request_by_name hostname: {} IDS:yes".format( + # networkoperation_section_dns_request_by_name['@request_name']) + # ) + yield({'values': networkoperation_section_dns_request_by_name['@request_name'], 'type': 'hostname', 'to_ids': True, 'comment': ''}) + elif networkoperation_section_dns_request_by_name['@result_addresses']: + # this happens when the IP is both in the request_name and result_address. + # print("networkoperation_section_dns_request_by_name hostname: {} IDS:yes".format( + # networkoperation_section_dns_request_by_name['@result_addresses']) + # ) + yield({'values': networkoperation_section_dns_request_by_name['@result_addresses'], 'type': 'ip-dst', 'to_ids': True, 'comment': ''}) + + if 'networkpacket_section' in process and 'connect_to_computer' in process['networkpacket_section']: + for networkpacket_section_connect_to_computer in process['networkpacket_section']['connect_to_computer']: + networkpacket_section_connect_to_computer['@remote_hostname'] = cleanup_hostname(networkpacket_section_connect_to_computer['@remote_hostname']) + networkpacket_section_connect_to_computer['@remote_ip'] = cleanup_ip(networkpacket_section_connect_to_computer['@remote_ip']) + if networkpacket_section_connect_to_computer['@remote_hostname'] and networkpacket_section_connect_to_computer['@remote_ip']: + # print("networkpacket_section_connect_to_computer hostname|ip: {}|{} IDS:yes COMMENT:port {}".format( + # networkpacket_section_connect_to_computer['@remote_hostname'], + # networkpacket_section_connect_to_computer['@remote_ip'], + # networkpacket_section_connect_to_computer['@remote_port']) + # ) + val_combined = "{}|{}".format(networkpacket_section_connect_to_computer['@remote_hostname'], networkpacket_section_connect_to_computer['@remote_ip']) + yield({'values': val_combined, 'type': 'hostname|ip', 'to_ids': True, 'comment': ''}) + elif networkpacket_section_connect_to_computer['@remote_hostname']: + # print("networkpacket_section_connect_to_computer hostname: {} IDS:yes COMMENT:port {}".format( + # networkpacket_section_connect_to_computer['@remote_hostname'], + # networkpacket_section_connect_to_computer['@remote_port']) + # ) + val_combined = "{}|{}".format(networkpacket_section_connect_to_computer['@remote_hostname'], networkpacket_section_connect_to_computer['@remote_port']) + yield({'values': val_combined, 'type': 'hostname|port', 'to_ids': True, 'comment': ''}) + elif networkpacket_section_connect_to_computer['@remote_ip']: + # print("networkpacket_section_connect_to_computer ip-dst: {} IDS:yes COMMENT:port {}".format( + # networkpacket_section_connect_to_computer['@remote_ip'], + # networkpacket_section_connect_to_computer['@remote_port']) + # ) + val_combined = "{}|{}".format(networkpacket_section_connect_to_computer['@remote_ip'], networkpacket_section_connect_to_computer['@remote_port']) + yield({'values': val_combined, 'type': 'ip-dst|port', 'to_ids': True, 'comment': ''}) + + if 'registry_section' in process and 'create_key' in process['registry_section']: + # FIXME this is a complicated section, together with the 'set_value'. + # it looks like this section is not ONLY about creating registry keys, + # more about accessing a handle to keys (with specific permissions) + # maybe we don't want to keep this, in favor of 'set_value' + for create_key in process['registry_section']['create_key']: + # print('REG CREATE: {}\t{}'.format( + # create_key['@desired_access'], + # create_key['@key_name'])) + pass + if 'registry_section' in process and 'delete_key' in process['registry_section']: + # LATER we probably don't want to keep this. Much pollution. + # Maybe for later once we have filtered out this. + for delete_key in process['registry_section']['delete_key']: + # print('REG DELETE: {}'.format( + # delete_key['@key_name']) + # ) + pass + if 'registry_section' in process and 'set_value' in process['registry_section']: + # FIXME this is a complicated section, together with the 'create_key'. + for set_value in process['registry_section']['set_value']: + # '@data_type' == 'REG_BINARY', + # '@data_type' == 'REG_DWORD', + # '@data_type' == 'REG_EXPAND_SZ', + # '@data_type' == 'REG_MULTI_SZ', + # '@data_type' == 'REG_NONE', + # '@data_type' == 'REG_QWORD', + # '@data_type' == 'REG_SZ', + regkey = cleanup_regkey("{}\\{}".format(set_value['@key_name'], set_value['@value_name'])) + regdata = cleanup_regdata(set_value.get('@data')) + if not regkey: + continue + if set_value['@data_size'] == '0' or not regdata: + # print('registry_section set_value REG SET: {}\t{}\t{}'.format( + # set_value['@data_type'], + # set_value['@key_name'], + # set_value['@value_name']) + # ) + yield({'values': regkey, 'type': 'regkey', 'to_ids': True, + 'categories': ['External analysis', 'Persistence mechanism', 'Artifacts dropped'], 'comment': set_value['@data_type']}) + else: + try: + # unicode fun... + # print('registry_section set_value REG SET: {}\t{}\t{}\t{}'.format( + # set_value['@data_type'], + # set_value['@key_name'], + # set_value['@value_name'], + # set_value['@data']) + # ) + val = "{}|{}".format(regkey, regdata) + yield({'values': val, 'type': 'regkey|value', 'to_ids': True, + 'categories': ['External analysis', 'Persistence mechanism', 'Artifacts dropped'], 'comment': set_value['@data_type']}) + except Exception as e: + print("EXCEPTION registry_section {}".format(e)) + # TODO - maybe we want to handle these later, or not... + pass + pass + + if 'stored_files' in process and 'stored_created_file' in process['stored_files']: + for stored_created_file in process['stored_files']['stored_created_file']: + stored_created_file['@filename'] = cleanup_filepath(stored_created_file['@filename']) + if stored_created_file['@filename']: + if stored_created_file['@filesize'] is not '0': + val = '{}|{}'.format(stored_created_file['@filename'], stored_created_file['@md5']) + # print("stored_created_file filename|md5: {}|{} IDS:yes".format( + # stored_created_file['@filename'], # filename + # stored_created_file['@md5']) # md5 + # ) # => filename|md5 + yield({'values': val, 'type': 'filename|md5', 'to_ids': True, + 'categories': ['Artifacts dropped', 'Payload delivery'], 'comment': ''}) + + else: + # print("stored_created_file filename: {} IDS:yes".format( + # stored_created_file['@filename']) # filename + # ) # => filename + yield({'values': stored_created_file['@filename'], + 'type': 'filename', 'to_ids': True, + 'categories': ['Artifacts dropped', 'Payload delivery'], 'comment': ''}) + + if 'stored_files' in process and 'stored_modified_file' in process['stored_files']: + for stored_modified_file in process['stored_files']['stored_modified_file']: + stored_modified_file['@filename'] = cleanup_filepath(stored_modified_file['@filename']) + if stored_modified_file['@filename']: + if stored_modified_file['@filesize'] is not '0': + val = '{}|{}'.format(stored_modified_file['@filename'], stored_modified_file['@md5']) + # print("stored_modified_file MODIFY FILE: {}\t{}".format( + # stored_modified_file['@filename'], # filename + # stored_modified_file['@md5']) # md5 + # ) # => filename|md5 + yield({'values': val, 'type': 'filename|md5', 'to_ids': True, + 'categories': ['Artifacts dropped', 'Payload delivery'], + 'comment': 'modified'}) + else: + # print("stored_modified_file MODIFY FILE: {}\t{}".format( + # stored_modified_file['@filename']) # filename + # ) # => filename + yield({'values': stored_modified_file['@filename'], 'type': 'filename', 'to_ids': True, + 'categories': ['Artifacts dropped', 'Payload delivery'], + 'comment': 'modified'}) + + +def add_file(filename, results, hash, index, filedata=None): + pass + # results.append({'values': filename, 'data': "{}|{}".format(filename, filedata.decode()), 'type': 'malware-sample', + # 'categories': ['Artifacts dropped', 'Payload delivery']}) + + +def add_file_zip(): + # if 'malware-sample' in request: + # sample_filename = request.get("malware-sample").split("|", 1)[0] + # data = base64.b64decode(data) + # fl = io.BytesIO(data) + # zf = zipfile.ZipFile(fl) + # sample_hashname = zf.namelist()[0] + # data = zf.read(sample_hashname, b"infected") + # zf.close() + pass + + +def print_json(data): + print(json.dumps(data, sort_keys=True, indent=4, separators=(',', ': '))) + + +def list_in_string(lst, data, regex=False): + for item in lst: + if regex: + if re.search(item, data, flags=re.IGNORECASE): + return True + else: + if item in data: + return True + + +def cleanup_ip(item): + # you should exclude private IP ranges via import regexes + noise_substrings = { + '224.0.0.', + '127.0.0.', + '8.8.8.8', + '8.8.4.4', + '0.0.0.0', + 'NONE' + } + if list_in_string(noise_substrings, item): + return None + try: + ipaddress.ip_address(item) + return item + except ValueError: + return None + + +def cleanup_hostname(item): + noise_substrings = { + 'wpad', + 'teredo.ipv6.microsoft.com', + 'WIN7SP1-x64-UNP' + } + # take away common known bad + if list_in_string(noise_substrings, item): + return None + # eliminate IP addresses + try: + ipaddress.ip_address(item) + except ValueError: + # this is not an IP, so continue + return item + return None + + +def cleanup_url(item): + if item in ['/']: + return None + return item + + +def cleanup_filepath(item): + noise_substrings = { + 'C:\\Windows\\Prefetch\\', + '\\AppData\\Roaming\\Microsoft\\Windows\\Recent\\', + '\\AppData\\Roaming\\Microsoft\\Office\\Recent\\', + 'C:\\ProgramData\\Microsoft\\OfficeSoftwareProtectionPlatform\\Cache\\cache.dat', + '\\AppData\\Local\\Microsoft\\Windows\\Temporary Internet Files\\Content.', + '\\AppData\\Local\\Microsoft\\Internet Explorer\\Recovery\\High\\', + '\\AppData\\Local\\Microsoft\\Internet Explorer\\DOMStore\\', + '\\AppData\\LocalLow\\Microsoft\\Internet Explorer\\Services\\search_', + '\\AppData\\Local\\Microsoft\\Windows\\History\\History.', + '\\AppData\\Roaming\\Microsoft\\Windows\\Cookies\\', + '\\AppData\\LocalLow\\Microsoft\\CryptnetUrlCache\\', + '\\AppData\\Local\\Microsoft\\Windows\\Caches\\', + '\\AppData\\Local\\Microsoft\\Windows\WebCache\\', + '\\AppData\\Local\\Microsoft\\Windows\\Explorer\\thumbcache', + + '\\AppData\\Roaming\\Adobe\\Acrobat\\9.0\\SharedDataEvents-journal', + '\\AppData\\Roaming\\Adobe\\Acrobat\\9.0\\UserCache.bin', + + '\\AppData\\Roaming\\Macromedia\\Flash Player\\macromedia.com\\support\\flashplayer\\sys\\settings.sol', + '\\AppData\\Roaming\Adobe\\Flash Player\\NativeCache\\', + 'C:\\Windows\\AppCompat\\Programs\\', + 'C:\~' # caused by temp file created by MS Office when opening malicious doc/xls/... + } + if list_in_string(noise_substrings, item): + return None + return item + + +def cleanup_regkey(item): + noise_substrings = { + r'\\Software\\Microsoft\\Windows\\CurrentVersion\\Installer\\UserData\\', + r'\\Software\\Microsoft\\Windows\\CurrentVersion\\Internet Settings\\', + r'\\CurrentVersion\\Explorer\\RecentDocs\\', + r'\\CurrentVersion\\Explorer\\UserAssist\\', + r'\\CurrentVersion\\Explorer\\FileExts\\[a-z\.]+\\OpenWith', + r'\\Software\\Microsoft\\Internet Explorer\\Main\\WindowsSearch', + r'\\Software\\Microsoft\\Office\\[0-9\.]+\\', + r'\\SOFTWARE\\Microsoft\\OfficeSoftwareProtectionPlatform\\', + r'\\Software\\Microsoft\\Office\\Common\\Smart Tag\\', + r'\\Usage\\SpellingAndGrammarFiles', + r'^HKLM\\Software\\Microsoft\\Tracing\\', + r'\\Software\\Classes\\CLSID\\', + r'\\Software\\Classes\\Local Settings\\MuiCache\\', + r'\\Local Settings\\Software\\Microsoft\\Windows\\Shell\\Bag', + r'\\Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\RunMRU\\' + } + item = item.replace('\\REGISTRY\\MACHINE\\', 'HKLM\\') + item = item.replace('\\REGISTRY\\USER\\', 'HKCU\\') + if list_in_string(noise_substrings, item, regex=True): + return None + return item + + +def cleanup_regdata(item): + if not item: + return None + item = item.replace('(UNICODE_0x00000000)', '') + return item + + +def get_zipped_contents(filename, data, password=None): + with zipfile.ZipFile(io.BytesIO(data), 'r') as zf: + unzipped_files = [] + if password is not None: + password = str.encode(password) # Byte encoded password required + for zip_file_name in zf.namelist(): # Get all files in the zip file + # print(zip_file_name) + with zf.open(zip_file_name, mode='r', pwd=password) as fp: + file_data = fp.read() + unzipped_files.append({'values': zip_file_name, + 'data': file_data, + 'comment': 'Extracted from {0}'.format(filename)}) + # print("{} : {}".format(zip_file_name, len(file_data))) + return unzipped_files + + +def introspection(): + modulesetup = {} + try: + userConfig + modulesetup['userConfig'] = userConfig + except NameError: + pass + try: + inputSource + modulesetup['inputSource'] = inputSource + except NameError: + pass + return modulesetup + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo From 0be188644453e983dafa8894e30492b4a7f52e98 Mon Sep 17 00:00:00 2001 From: Christophe Vandeplas Date: Tue, 16 Jan 2018 15:13:17 +0100 Subject: [PATCH 14/66] fix farsight_passivedns - rdata 404 not found --- misp_modules/modules/expansion/farsight_passivedns.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/misp_modules/modules/expansion/farsight_passivedns.py b/misp_modules/modules/expansion/farsight_passivedns.py index 7c0ad23..c76c752 100755 --- a/misp_modules/modules/expansion/farsight_passivedns.py +++ b/misp_modules/modules/expansion/farsight_passivedns.py @@ -50,10 +50,10 @@ def lookup_name(client, name): for i in item.get('rdata'): # grab email field and replace first dot by @ to convert to an email address yield(i.split(' ')[1].rstrip('.').replace('.', '@', 1)) - res = client.query_rdata_name(name) # RDATA = entries on the right-hand side of the domain name related labels - for item in res: - if item.get('rrtype') in ['A', 'AAAA', 'CNAME']: - yield(item.get('rrname').rstrip('.')) + # res = client.query_rdata_name(name) # RDATA = entries on the right-hand side of the domain name related labels + # for item in res: + # if item.get('rrtype') in ['A', 'AAAA', 'CNAME']: + # yield(item.get('rrname').rstrip('.')) def lookup_ip(client, ip): From 18523c4ada89baadf68cc9a6b16292ae4bba3c16 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Tue, 16 Jan 2018 17:08:44 +0100 Subject: [PATCH 15/66] Check an IPv4 address against known RBLs --- misp_modules/modules/expansion/rbl.py | 117 ++++++++++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 misp_modules/modules/expansion/rbl.py diff --git a/misp_modules/modules/expansion/rbl.py b/misp_modules/modules/expansion/rbl.py new file mode 100644 index 0000000..994682a --- /dev/null +++ b/misp_modules/modules/expansion/rbl.py @@ -0,0 +1,117 @@ +import json +import datetime + +try: + import dns.resolver + resolver = dns.resolver.Resolver() + resolver.timeout = 0.2 + resolver.lifetime = 0.2 +except: + print("dnspython3 is missing, use 'pip install dnspython3' to install it.") + sys.exit(0) + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['ip-src', 'ip-dst'], 'output': ['text']} +moduleinfo = {'version': '0.1', 'author': 'Christian Studer', + 'description': 'Check an IPv4 address against known RBLs.', + 'module-type': ['expansion', 'hover']} +moduleconfig = [] + +rbls = { + 'spam.spamrats.com': 'http://www.spamrats.com', + 'spamguard.leadmon.net': 'http://www.leadmon.net/SpamGuard/', + 'rbl-plus.mail-abuse.org': 'http://www.mail-abuse.com/lookup.html', + 'web.dnsbl.sorbs.net': 'http://www.sorbs.net', + 'ix.dnsbl.manitu.net': 'http://www.dnsbl.manitu.net', + 'virus.rbl.jp': 'http://www.rbl.jp', + 'dul.dnsbl.sorbs.net': 'http://www.sorbs.net', + 'bogons.cymru.com': 'http://www.team-cymru.org/Services/Bogons/', + 'psbl.surriel.com': 'http://psbl.surriel.com', + 'misc.dnsbl.sorbs.net': 'http://www.sorbs.net', + 'httpbl.abuse.ch': 'http://dnsbl.abuse.ch', + 'combined.njabl.org': 'http://combined.njabl.org', + 'smtp.dnsbl.sorbs.net': 'http://www.sorbs.net', + 'korea.services.net': 'http://korea.services.net', + 'drone.abuse.ch': 'http://dnsbl.abuse.ch', + 'rbl.efnetrbl.org': 'http://rbl.efnetrbl.org', + 'cbl.anti-spam.org.cn': 'http://www.anti-spam.org.cn/?Locale=en_US', + 'b.barracudacentral.org': 'http://www.barracudacentral.org/rbl/removal-request', + 'bl.spamcannibal.org': 'http://www.spamcannibal.org', + 'xbl.spamhaus.org': 'http://www.spamhaus.org/xbl/', + 'zen.spamhaus.org': 'http://www.spamhaus.org/zen/', + 'rbl.suresupport.com': 'http://suresupport.com/postmaster', + 'db.wpbl.info': 'http://www.wpbl.info', + 'sbl.spamhaus.org': 'http://www.spamhaus.org/sbl/', + 'http.dnsbl.sorbs.net': 'http://www.sorbs.net', + 'csi.cloudmark.com': 'http://www.cloudmark.com/en/products/cloudmark-sender-intelligence/index', + 'rbl.interserver.net': 'http://rbl.interserver.net', + 'ubl.unsubscore.com': 'http://www.lashback.com/blacklist/', + 'dnsbl.sorbs.net': 'http://www.sorbs.net', + 'virbl.bit.nl': 'http://virbl.bit.nl', + 'pbl.spamhaus.org': 'http://www.spamhaus.org/pbl/', + 'socks.dnsbl.sorbs.net': 'http://www.sorbs.net', + 'short.rbl.jp': 'http://www.rbl.jp', + 'dnsbl.dronebl.org': 'http://www.dronebl.org', + 'blackholes.mail-abuse.org': 'http://www.mail-abuse.com/lookup.html', + 'truncate.gbudb.net': 'http://www.gbudb.com/truncate/index.jsp', + 'dyna.spamrats.com': 'http://www.spamrats.com', + 'spamrbl.imp.ch': 'http://antispam.imp.ch', + 'spam.dnsbl.sorbs.net': 'http://www.sorbs.net', + 'wormrbl.imp.ch': 'http://antispam.imp.ch', + 'query.senderbase.org': 'http://www.senderbase.org/about', + 'opm.tornevall.org': 'http://dnsbl.tornevall.org', + 'netblock.pedantic.org': 'http://pedantic.org', + 'access.redhawk.org': 'http://www.redhawk.org/index.php?option=com_wrapper&Itemid=33', + 'cdl.anti-spam.org.cn': 'http://www.anti-spam.org.cn/?Locale=en_US', + 'multi.surbl.org': 'http://www.surbl.org', + 'noptr.spamrats.com': 'http://www.spamrats.com', + 'dnsbl.inps.de': 'http://dnsbl.inps.de/index.cgi?lang=en', + 'bl.spamcop.net': 'http://bl.spamcop.net', + 'cbl.abuseat.org': 'http://cbl.abuseat.org', + 'dsn.rfc-ignorant.org': 'http://www.rfc-ignorant.org/policy-dsn.php', + 'zombie.dnsbl.sorbs.net': 'http://www.sorbs.net', + 'dnsbl.njabl.org': 'http://dnsbl.njabl.org', + 'relays.mail-abuse.org': 'http://www.mail-abuse.com/lookup.html', + 'rbl.spamlab.com': 'http://tools.appriver.com/index.aspx?tool=rbl', + 'all.bl.blocklist.de': 'http://www.blocklist.de/en/rbldns.html' +} + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + if request.get('ip-src'): + ip = request['ip-src'] + elif request.get('ip-dst'): + ip = request['ip-dst'] + else: + misperrors['error'] = "Unsupported attributes type" + return misperrors + results = {} + results['query'] = ip + results['date'] = datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc).isoformat() + results['listed'] = [] + results['info'] = [] + results['not_listed'] = [] + for rbl in rbls: + ipRev = '.'.join(ip.split('.')[::-1]) + query = '{}.{}'.format(ipRev, rbl) + try: + resolver.query(query,'A') + try: + txt = resolver.query(query, 'TXT') + except: + results['listed'].append(query) + results['listed'].append(query) + results['info'].append(str(txt[0])) + except: + results['not_listed'].append(query) + r = {'results': [{'types': mispattributes.get('output'), 'values': json.dumps(results)}]} + return r + +def introspection(): + return mispattributes + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo From d045cf7d5f4acd466cf9cfb43a51915d0b3e7784 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Tue, 16 Jan 2018 19:46:52 +0100 Subject: [PATCH 16/66] chg: Modified output format --- misp_modules/modules/expansion/rbl.py | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/misp_modules/modules/expansion/rbl.py b/misp_modules/modules/expansion/rbl.py index 994682a..da8c5fb 100644 --- a/misp_modules/modules/expansion/rbl.py +++ b/misp_modules/modules/expansion/rbl.py @@ -87,26 +87,21 @@ def handler(q=False): else: misperrors['error'] = "Unsupported attributes type" return misperrors - results = {} - results['query'] = ip - results['date'] = datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc).isoformat() - results['listed'] = [] - results['info'] = [] - results['not_listed'] = [] + listed = [] + info = [] for rbl in rbls: ipRev = '.'.join(ip.split('.')[::-1]) query = '{}.{}'.format(ipRev, rbl) try: - resolver.query(query,'A') - try: - txt = resolver.query(query, 'TXT') - except: - results['listed'].append(query) - results['listed'].append(query) - results['info'].append(str(txt[0])) + txt = resolver.query(query,'TXT') + listed.append(query) + info.append(str(txt[0])) except: - results['not_listed'].append(query) - r = {'results': [{'types': mispattributes.get('output'), 'values': json.dumps(results)}]} + continue + result = {} + for l, i in zip(listed, info): + result[l] = i + r = {'results': [{'types': mispattributes.get('output'), 'values': json.dumps(result)}]} return r def introspection(): From 8008d62bc10c7ec4f5e7a4242029e7d5829fe950 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Tue, 16 Jan 2018 20:16:53 +0100 Subject: [PATCH 17/66] add: RBL added --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 339c56f..1097ed4 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,7 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/ * [iprep](misp-modules/modules/expansion/iprep.py) - an expansion module to get IP reputation from packetmail.net. * [OTX](misp_modules/modules/expansion/otx.py) - an expansion module for [OTX](https://otx.alienvault.com/). * [passivetotal](misp_modules/modules/expansion/passivetotal.py) - a [passivetotal](https://www.passivetotal.org/) module that queries a number of different PassiveTotal datasets. +* [rbl](misp_modules/modules/expansion/rbl.py) - a module to get RBL (Real-Time Blackhost List) values from an attribute. * [shodan](misp_modules/modules/expansion/shodan.py) - a minimal [shodan](https://www.shodan.io/) expansion module. * [sourcecache](misp_modules/modules/expansion/sourcecache.py) - a module to cache a specific link from a MISP instance. * [ThreatCrowd](misp_modules/modules/expansion/threatcrowd.py) - an expansion module for [ThreatCrowd](https://www.threatcrowd.org/). From 8a1a860cda7b722a315dd203baf8c669de9f7104 Mon Sep 17 00:00:00 2001 From: Christophe Vandeplas Date: Fri, 19 Jan 2018 14:42:25 +0100 Subject: [PATCH 18/66] added CrowdStrike Falcon Intel Indicators expansion module --- README.md | 1 + misp_modules/modules/expansion/__init__.py | 4 +- .../modules/expansion/crowdstrike_falcon.py | 128 ++++++++++++++++++ 3 files changed, 131 insertions(+), 2 deletions(-) create mode 100755 misp_modules/modules/expansion/crowdstrike_falcon.py diff --git a/README.md b/README.md index 1097ed4..1ba2c38 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/ * [CIRCL Passive DNS](misp_modules/modules/expansion/circl_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information. * [CIRCL Passive SSL](misp_modules/modules/expansion/circl_passivessl.py) - a hover and expansion module to expand IP addresses with the X.509 certificate seen. * [countrycode](misp_modules/modules/expansion/countrycode.py) - a hover module to tell you what country a URL belongs to. +* [CrowdStrike Falcon](misp_modules/modules/expansion/crowdstrike_falcon.py) - an expansion module to expand using CrowdStrike Falcon Intel Indicator API. * [CVE](misp_modules/modules/expansion/cve.py) - a hover module to give more information about a vulnerability (CVE). * [DNS](misp_modules/modules/expansion/dns.py) - a simple module to resolve MISP attributes like hostname and domain to expand IP addresses attributes. * [DomainTools](misp_modules/modules/expansion/domaintools.py) - a hover and expansion module to get information from [DomainTools](http://www.domaintools.com/) whois. diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py index 231d30e..7b56b62 100644 --- a/misp_modules/modules/expansion/__init__.py +++ b/misp_modules/modules/expansion/__init__.py @@ -2,5 +2,5 @@ from . import _vmray __all__ = ['vmray_submit', 'asn_history', 'circl_passivedns', 'circl_passivessl', 'countrycode', 'cve', 'dns', 'domaintools', 'eupi', 'farsight_passivedns', 'ipasn', 'passivetotal', 'sourcecache', - 'virustotal', 'whois', 'shodan', 'reversedns', 'geoip_country', 'wiki', 'iprep', 'threatminer' ,'otx', - 'threatcrowd','vulndb'] + 'virustotal', 'whois', 'shodan', 'reversedns', 'geoip_country', 'wiki', 'iprep', 'threatminer', 'otx', + 'threatcrowd', 'vulndb', 'crowdstrike_falcon'] diff --git a/misp_modules/modules/expansion/crowdstrike_falcon.py b/misp_modules/modules/expansion/crowdstrike_falcon.py new file mode 100755 index 0000000..1342e88 --- /dev/null +++ b/misp_modules/modules/expansion/crowdstrike_falcon.py @@ -0,0 +1,128 @@ +import json +import requests + +moduleinfo = {'version': '0.1', + 'author': 'Christophe Vandeplas', + 'description': 'Module to query CrowdStrike Falcon.', + 'module-type': ['expansion']} +moduleconfig = ['api_id', 'apikey'] +misperrors = {'error': 'Error'} +misp_types_in = ['domain', 'email-attachment', 'email-dst', 'email-reply-to', 'email-src', 'email-subject', + 'filename', 'hostname', 'ip', 'ip-src', 'ip-dst', 'md5', 'mutex', 'regkey', 'sha1', 'sha256', 'uri', 'url', + 'user-agent', 'whois-registrant-email', 'x509-fingerprint-md5'] +mapping_out = { # mapping between the MISP attributes types and the compatible CrowdStrike indicator types. + 'domain': {'types': 'hostname', 'to_ids': True}, + 'email_address': {'types': 'email-src', 'to_ids': True}, + 'email_subject': {'types': 'email-subject', 'to_ids': True}, + 'file_name': {'types': 'filename', 'to_ids': True}, + 'hash_md5': {'types': 'md5', 'to_ids': True}, + 'hash_sha1': {'types': 'sha1', 'to_ids': True}, + 'hash_sha256': {'types': 'sha256', 'to_ids': True}, + 'ip_address': {'types': 'ip-dst', 'to_ids': True}, + 'ip_address_block': {'types': 'ip-dst', 'to_ids': True}, + 'mutex_name': {'types': 'mutex', 'to_ids': True}, + 'registry': {'types': 'regkey', 'to_ids': True}, + 'url': {'types': 'url', 'to_ids': True}, + 'user_agent': {'types': 'user-agent', 'to_ids': True}, + 'x509_serial': {'types': 'x509-fingerprint-md5', 'to_ids': True}, + + 'actors': {'types': 'threat-actor'}, + 'malware_families': {'types': 'text', 'categories': 'Attribution'} +} +misp_types_out = [item['types'] for item in mapping_out.values()] +mispattributes = {'input': misp_types_in, 'output': misp_types_out} + + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + if (request.get('config')): + if (request['config'].get('apikey') is None): + misperrors['error'] = 'CrowdStrike apikey is missing' + return misperrors + if (request['config'].get('api_id') is None): + misperrors['error'] = 'CrowdStrike api_id is missing' + return misperrors + client = CSIntelAPI(request['config']['api_id'], request['config']['apikey']) + + r = {"results": []} + + valid_type = False + for k in misp_types_in: + if request.get(k): + # map the MISP typ to the CrowdStrike type + for item in lookup_indicator(client, request[k]): + r['results'].append(item) + valid_type = True + + if not valid_type: + misperrors['error'] = "Unsupported attributes type" + return misperrors + return r + + +def lookup_indicator(client, item): + result = client.search_indicator(item) + for item in result: + for relation in item['relations']: + if mapping_out.get(relation['type']): + r = mapping_out[relation['type']].copy() + r['values'] = relation['indicator'] + yield(r) + for actor in item['actors']: + r = mapping_out['actors'].copy() + r['values'] = actor + yield(r) + for malware_family in item['malware_families']: + r = mapping_out['malware_families'].copy() + r['values'] = malware_family + yield(r) + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo + + +class CSIntelAPI(): + def __init__(self, custid=None, custkey=None, perpage=100, page=1, baseurl="https://intelapi.crowdstrike.com/indicator/v2/search/"): + # customer id and key should be passed when obj is created + self.custid = custid + self.custkey = custkey + + self.baseurl = baseurl + self.perpage = perpage + self.page = page + + def request(self, query): + headers = {'X-CSIX-CUSTID': self.custid, + 'X-CSIX-CUSTKEY': self.custkey, + 'Content-Type': 'application/json'} + + full_query = self.baseurl + query + + r = requests.get(full_query, headers=headers) + # 400 - bad request + if r.status_code == 400: + raise Exception('HTTP Error 400 - Bad request.') + + # 404 - oh shit + if r.status_code == 404: + raise Exception('HTTP Error 404 - awww snap.') + + # catch all? + if r.status_code != 200: + raise Exception('HTTP Error: ' + str(r.status_code)) + + if r.text: + return r + + def search_indicator(self, item): + query = 'indicator?match=' + item + r = self.request(query) + return json.loads(r.text) From b9d72bb043ca7f0a707a7a062e3a554fc51461b2 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Thu, 25 Jan 2018 15:44:08 +0100 Subject: [PATCH 19/66] First version of csv import module - If more than 1 misp type is recognized, for each one an attribute is created - Needs to have header set by user as parameters of the module atm - Review needed to see the feasibility with fields that can create confusion and be interpreted both as misp type or attribute field (for instance comment is a misp type and an attribute field) --- misp_modules/modules/import_mod/csvimport.py | 113 +++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 misp_modules/modules/import_mod/csvimport.py diff --git a/misp_modules/modules/import_mod/csvimport.py b/misp_modules/modules/import_mod/csvimport.py new file mode 100644 index 0000000..fc5b4e7 --- /dev/null +++ b/misp_modules/modules/import_mod/csvimport.py @@ -0,0 +1,113 @@ +import json, os +import pymisp + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['file'], 'output': ['MISP attributes']} +moduleinfo = {'version': '0.1', 'author': 'Christian Studer', + 'description': 'Import Attributes from a csv file.', + 'module-type': ['import']} +moduleconfig = ['header'] + +duplicatedFields = {'mispType': {'mispComment': 'comment'}, + 'attrField': {'eventComment': 'comment'}} + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + if request.get('file'): + filename = request['file'] + else: + misperrors['error'] = "Unsupported attributes type" + return misperrors + if not request.get('config') and not request['config'].get('header'): + misperrors['error'] = "Configuration error" + return misperrors + config = request['config'].get('header') + header = [] + data = [] + with open(filename, 'r') as f: + for line in f: + # split comments from data + if line.startswith('#'): + header.append(line) + else: + data.append(line) + # find which delimiter is used + delimiter, length = findDelimiter(config, data) + # build the attributes + result = buildAttributes(config, data, delimiter, length) + r = {'results': [{'types': mispattributes['output'], 'values': result}]} + return r + +def findDelimiter(header, data): + n = len(header) + if n > 1: + for d in (';', ',', '|'): + if data[0].count(d) == (n-1): + return d, n + break + else: + return None, 1 + +def buildAttributes(header, dataValues, delimiter, length): + attributes = [] + # if there is only 1 field of data + if delimiter is None: + mispType = header[0] + for data in dataValues: + attributes.append({'type': mispType, 'value': data}) + else: + # split fields that should be recognized as misp attribute types from the others + list2pop, misp, head = findMispTypes(header) + # for each line of data + for data in dataValues: + datamisp = [] + datasplit = data.split(delimiter) + # in case there is an empty line or an error + if len(datasplit) != length: + continue + # pop from the line data that matches with a misp type, using the list of indexes + for l in list2pop: + datamisp.append(datasplit.pop(l).strip()) + # for each misp type, we create an attribute + for m, dm in zip(misp, datamisp): + attribute = {'type': m, 'value': dm} + for h, ds in zip(head, datasplit): + if h: + attribute[h] = ds.strip() + attributes.append(attribute) + return attributes + +def findMispTypes(header): + descFilename = os.path.join(pymisp.__path__[0], 'data/describeTypes.json') + with open(descFilename, 'r') as f: + MispTypes = json.loads(f.read())['result'].get('types') + list2pop = [] + misp = [] + head = [] + for h in reversed(header): + n = header.index(h) + # fields that are misp attribute types + if h in MispTypes: + list2pop.append(n) + misp.append(h) + # handle confusions between misp attribute types and attribute fields + elif h in duplicatedFields['mispType']: + # fields that should be considered as misp attribute types + list2pop.append(n) + misp.append(duplicatedFields['mispType'].get(h)) + elif h in duplicatedFields['attrField']: + # fields that should be considered as attribute fields + head.append(duplicatedFields['attrField'].get(h)) + else: + head.append(h) + # return list of indexes of the misp types, list of the misp types, remaining fields that will be attribute fields + return list2pop, misp, head + +def introspection(): + return mispattributes + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo From 4d846f968f4a1e2671b64a18a4b6b1371b8f5e41 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Fri, 26 Jan 2018 17:11:01 +0100 Subject: [PATCH 20/66] Updated delimiter parsing & data reading functions --- misp_modules/modules/import_mod/csvimport.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/misp_modules/modules/import_mod/csvimport.py b/misp_modules/modules/import_mod/csvimport.py index fc5b4e7..e5ff3b0 100644 --- a/misp_modules/modules/import_mod/csvimport.py +++ b/misp_modules/modules/import_mod/csvimport.py @@ -24,13 +24,17 @@ def handler(q=False): misperrors['error'] = "Configuration error" return misperrors config = request['config'].get('header') - header = [] + #header = [] data = [] with open(filename, 'r') as f: for line in f: # split comments from data - if line.startswith('#'): - header.append(line) + if '#' in line: + l = line.split('#')[0] + if l: + data.append(l) + #else: + #header.append(line) else: data.append(line) # find which delimiter is used @@ -43,10 +47,9 @@ def handler(q=False): def findDelimiter(header, data): n = len(header) if n > 1: - for d in (';', ',', '|'): + for d in (';', '|', '/', ','): if data[0].count(d) == (n-1): return d, n - break else: return None, 1 @@ -100,6 +103,7 @@ def findMispTypes(header): elif h in duplicatedFields['attrField']: # fields that should be considered as attribute fields head.append(duplicatedFields['attrField'].get(h)) + # otherwise, it is an attribute field else: head.append(h) # return list of indexes of the misp types, list of the misp types, remaining fields that will be attribute fields From 56cbd72b6526bf6b777cb62cea5f4be99b390463 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Sun, 28 Jan 2018 18:12:40 +0100 Subject: [PATCH 21/66] Fixed data treatment & other updates --- misp_modules/modules/import_mod/csvimport.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/misp_modules/modules/import_mod/csvimport.py b/misp_modules/modules/import_mod/csvimport.py index e5ff3b0..4076902 100644 --- a/misp_modules/modules/import_mod/csvimport.py +++ b/misp_modules/modules/import_mod/csvimport.py @@ -47,7 +47,7 @@ def handler(q=False): def findDelimiter(header, data): n = len(header) if n > 1: - for d in (';', '|', '/', ','): + for d in (';', '|', '/', ',', ' '): if data[0].count(d) == (n-1): return d, n else: @@ -59,7 +59,7 @@ def buildAttributes(header, dataValues, delimiter, length): if delimiter is None: mispType = header[0] for data in dataValues: - attributes.append({'type': mispType, 'value': data}) + attributes.append({'type': mispType, 'value': data.strip()}) else: # split fields that should be recognized as misp attribute types from the others list2pop, misp, head = findMispTypes(header) @@ -107,7 +107,7 @@ def findMispTypes(header): else: head.append(h) # return list of indexes of the misp types, list of the misp types, remaining fields that will be attribute fields - return list2pop, misp, head + return list2pop, misp, list(reversed(head)) def introspection(): return mispattributes From 529d22cca80820690183a10016f03f38d5047a6c Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Mon, 29 Jan 2018 09:19:58 +0100 Subject: [PATCH 22/66] fix: skipping empty lines --- misp_modules/modules/import_mod/csvimport.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/misp_modules/modules/import_mod/csvimport.py b/misp_modules/modules/import_mod/csvimport.py index 4076902..50e9837 100644 --- a/misp_modules/modules/import_mod/csvimport.py +++ b/misp_modules/modules/import_mod/csvimport.py @@ -59,7 +59,9 @@ def buildAttributes(header, dataValues, delimiter, length): if delimiter is None: mispType = header[0] for data in dataValues: - attributes.append({'type': mispType, 'value': data.strip()}) + d = data.strip() + if d: + attributes.append({'type': mispType, 'value': d}) else: # split fields that should be recognized as misp attribute types from the others list2pop, misp, head = findMispTypes(header) From b2ec186ccb14a48b3e4370925bc05fb439081d06 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Mon, 29 Jan 2018 17:04:32 +0100 Subject: [PATCH 23/66] Updated delimiter finder method --- misp_modules/modules/import_mod/csvimport.py | 23 ++++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/misp_modules/modules/import_mod/csvimport.py b/misp_modules/modules/import_mod/csvimport.py index 50e9837..7e0107f 100644 --- a/misp_modules/modules/import_mod/csvimport.py +++ b/misp_modules/modules/import_mod/csvimport.py @@ -30,13 +30,11 @@ def handler(q=False): for line in f: # split comments from data if '#' in line: - l = line.split('#')[0] - if l: - data.append(l) - #else: - #header.append(line) + l = line.split('#')[0].strip() else: - data.append(line) + l = line.strip() + if l: + data.append(l) # find which delimiter is used delimiter, length = findDelimiter(config, data) # build the attributes @@ -47,9 +45,16 @@ def handler(q=False): def findDelimiter(header, data): n = len(header) if n > 1: - for d in (';', '|', '/', ',', ' '): - if data[0].count(d) == (n-1): - return d, n + tmpData = [] + for da in data: + tmp = [] + for d in (';', '|', '/', ',', '\t', ' ',): + if da.count(d) == (n-1): + tmp.append(d) + if len(tmp) == 1 and tmp == tmpData: + return tmpData[0], n + else: + tmpData = tmp else: return None, 1 From 71c00954d08862f58de8d218fdf4407fb8104f6d Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Tue, 30 Jan 2018 11:20:28 +0100 Subject: [PATCH 24/66] fix: Solved reading problems for some files --- misp_modules/modules/import_mod/csvimport.py | 22 +++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/misp_modules/modules/import_mod/csvimport.py b/misp_modules/modules/import_mod/csvimport.py index 7e0107f..dc67eec 100644 --- a/misp_modules/modules/import_mod/csvimport.py +++ b/misp_modules/modules/import_mod/csvimport.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- import json, os import pymisp @@ -25,8 +26,20 @@ def handler(q=False): return misperrors config = request['config'].get('header') #header = [] + try: + data = readFile(filename, 'utf-8') + except: + data = readFile(filename, 'iso-8859-1') + # find which delimiter is used + delimiter, length = findDelimiter(config, data) + # build the attributes + result = buildAttributes(config, data, delimiter, length) + r = {'results': [{'types': mispattributes['output'], 'values': result}]} + return r + +def readFile(filename, encoding): data = [] - with open(filename, 'r') as f: + with open(filename, 'r', encoding=encoding) as f: for line in f: # split comments from data if '#' in line: @@ -35,12 +48,7 @@ def handler(q=False): l = line.strip() if l: data.append(l) - # find which delimiter is used - delimiter, length = findDelimiter(config, data) - # build the attributes - result = buildAttributes(config, data, delimiter, length) - r = {'results': [{'types': mispattributes['output'], 'values': result}]} - return r + return data def findDelimiter(header, data): n = len(header) From 545589373c8e93a3afd1dd53109faec4796e1bd7 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Tue, 30 Jan 2018 12:13:00 +0100 Subject: [PATCH 25/66] fix: Python version in Travis --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7a39758..96cd6bd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,8 @@ python: - "3.5-dev" - "3.6" - "3.6-dev" - - "nightly" + - "3.7" + - "3.7-dev" install: - pip install -U nose codecov pytest From 3a4c8ea9522a9d2ca5a98280997f87f8aa6c1f4a Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Tue, 30 Jan 2018 12:26:26 +0100 Subject: [PATCH 26/66] 3.7-alpha removed --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 96cd6bd..0a3a912 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,8 +11,6 @@ python: - "3.5-dev" - "3.6" - "3.6-dev" - - "3.7" - - "3.7-dev" install: - pip install -U nose codecov pytest From 48869335ee28179ebd21b31227bc91a4056c1695 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Wed, 31 Jan 2018 18:09:45 +0100 Subject: [PATCH 27/66] first tests for the GoAML export module --- misp_modules/modules/export_mod/goaml.py | 47 ++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 misp_modules/modules/export_mod/goaml.py diff --git a/misp_modules/modules/export_mod/goaml.py b/misp_modules/modules/export_mod/goaml.py new file mode 100644 index 0000000..f82590b --- /dev/null +++ b/misp_modules/modules/export_mod/goaml.py @@ -0,0 +1,47 @@ +import json +import pymisp +import base64 + +misperrors = {'error': 'Error'} +moduleinfo = {'version': '1', 'author': 'Christian Studer', + 'description': '', + 'module-type': ['export']} +moduleconfig = [] +mispattributes = {} + +objects_to_parse = ['bank-account', 'person'] + +class GoAmlGeneration(): + def __init__(self): + self.document = {} + + def from_event(self, event): + self.misp_event = pymisp.MISPEvent() + self.misp_event.load(event) + + def parse_objects(self): + for obj in self.misp_event.objects: + if obj.name in objects_to_parse: + obj_dict = {} + for attribute in obj.attributes: + obj_dict[attribute.object_relation] = attribute.value + self.document[obj.name] = obj_dict + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + if 'data' not in request: + return False + exp_doc = GoAmlGeneration() + exp_doc.from_event(request['data'][0]) + exp_doc.parse_objects() + return {'response': {}, 'data': exp_doc.document} + #return {'response': [], 'data': str(base64.b64encode(bytes(exp_doc.document, 'utf-8')), 'utf-8')} + +def introspection(): + return + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo From 8dce7935ae2679540f7eeb8a7d69472f85c7a15a Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Thu, 1 Feb 2018 14:55:48 +0100 Subject: [PATCH 28/66] Outputting xml format Also mapping MISP and GoAML types --- misp_modules/modules/export_mod/goaml.py | 65 ++++++++++++++++++++++-- 1 file changed, 61 insertions(+), 4 deletions(-) diff --git a/misp_modules/modules/export_mod/goaml.py b/misp_modules/modules/export_mod/goaml.py index f82590b..82375c2 100644 --- a/misp_modules/modules/export_mod/goaml.py +++ b/misp_modules/modules/export_mod/goaml.py @@ -1,5 +1,5 @@ import json -import pymisp +from pymisp import MISPEvent import base64 misperrors = {'error': 'Error'} @@ -8,15 +8,29 @@ moduleinfo = {'version': '1', 'author': 'Christian Studer', 'module-type': ['export']} moduleconfig = [] mispattributes = {} +outputFileExtension = "xml" +responseType = "application/xml" objects_to_parse = ['bank-account', 'person'] +goAMLmapping = {'bank-account': 't_account', 'text': 'institution_name', 'institution-code': 'institution_code', + 'swift': 'swift', 'branch': 'branch', 'non-banking-institution': 'non_bank_institution', + 'account': 'account', 'currency-code': 'currency_code', 'account-name': 'account_name', + 'iban': 'iban', 'client-number': 'client_number', 'personal-account-type': 'personal_account_type', + 'opened': 'opened', 'closed': 'closed', 'balance': 'balance', 'status-code': 'status_code', + 'beneficiary': 'beneficiary', 'beneficiary-comment': 'beneficiary_comment', 'comments': 'comments', + 'person': 't_person', 'text': 'comments', 'first-name': 'first_name', 'middle-name': 'middle_name', + 'last-name': 'last_name', 'mothers-name': 'mothers_name', 'title': 'title', 'alias': 'alias', + 'date-of-birth': 'birthdate', 'place-of-birth': 'birth_place', 'gender': 'gender', + 'passport-number': 'passport_number', 'passport-country': 'passport_country', + 'social-security-number': 'ssn', 'nationality': 'nationality1'} + class GoAmlGeneration(): def __init__(self): self.document = {} def from_event(self, event): - self.misp_event = pymisp.MISPEvent() + self.misp_event = MISPEvent() self.misp_event.load(event) def parse_objects(self): @@ -27,6 +41,27 @@ class GoAmlGeneration(): obj_dict[attribute.object_relation] = attribute.value self.document[obj.name] = obj_dict + def build_xml(self): + self.xml = "" + if 'bank-account' in self.document: + if 'report-code' in self.document['bank-account']: + self.xml += "{}".format(self.document['bank-account'].pop('report-code').split(' ')[0]) + for a in ('personal-account-type', 'status-code'): + if a in self.document['bank-account']: + self.document['bank-account'][a] = self.document['bank-account'][a].split(' - ')[0] + self.itterate() + self.xml += "" + + def itterate(self): + for t in self.document: + self.xml += "<{}>".format(goAMLmapping[t]) + for k in self.document[t]: + try: + self.xml += "<{0}>{1}".format(goAMLmapping[k], self.document[t][k]) + except KeyError: + pass + self.xml += "".format(goAMLmapping[t]) + def handler(q=False): if q is False: return False @@ -36,11 +71,33 @@ def handler(q=False): exp_doc = GoAmlGeneration() exp_doc.from_event(request['data'][0]) exp_doc.parse_objects() - return {'response': {}, 'data': exp_doc.document} + exp_doc.build_xml() + return {'response': {}, 'data': exp_doc.xml} #return {'response': [], 'data': str(base64.b64encode(bytes(exp_doc.document, 'utf-8')), 'utf-8')} def introspection(): - return + modulesetup = {} + try: + responseType + modulesetup['responseType'] = responseType + except NameError: + pass + try: + userConfig + modulesetup['userConfig'] = userConfig + except NameError: + pass + try: + outputFileExtension + modulesetup['outputFileExtension'] = outputFileExtension + except NameError: + pass + try: + inputSource + mmoduleSetup['inputSource'] = inputSource + except NameError: + pass + return modulesetup def version(): moduleinfo['config'] = moduleconfig From 69d733bb35ac95d19661cb48e051d43c23fc8dea Mon Sep 17 00:00:00 2001 From: Thomas Gardner Date: Thu, 1 Feb 2018 10:22:28 -0700 Subject: [PATCH 29/66] added csvimport to __init__.py --- misp_modules/modules/import_mod/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/import_mod/__init__.py b/misp_modules/modules/import_mod/__init__.py index e2941d7..8acccbd 100644 --- a/misp_modules/modules/import_mod/__init__.py +++ b/misp_modules/modules/import_mod/__init__.py @@ -1,4 +1,4 @@ from . import _vmray __all__ = ['vmray_import', 'testimport', 'ocr', 'stiximport', 'cuckooimport', - 'email_import', 'mispjson', 'openiocimport', 'threatanalyzer_import'] + 'email_import', 'mispjson', 'openiocimport', 'threatanalyzer_import', 'csvimport'] From deed898c0446e0b40d35a2b43c2cacb34e95ffb1 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Fri, 2 Feb 2018 07:16:44 +0100 Subject: [PATCH 30/66] add: CSV import module added --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 1ba2c38..dcf6c85 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,7 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/ ### Import modules +* [CSV import](misp_modules/modules/import_mod/csvimport.py) Customizable CSV import module. * [Cuckoo JSON](misp_modules/modules/import_mod/cuckooimport.py) Cuckoo JSON import. * [Email Import](misp_modules/modules/import_mod/email_import.py) Email import module for MISP to import basic metadata. * [OCR](misp_modules/modules/import_mod/ocr.py) Optical Character Recognition (OCR) module for MISP to import attributes from images, scan or faxes. From 8983ebc4b206106ad6b893f3c8a29e51605d0374 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Mon, 5 Feb 2018 15:51:03 +0100 Subject: [PATCH 31/66] wip: added location & signatory information --- misp_modules/modules/export_mod/goaml.py | 25 ++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/misp_modules/modules/export_mod/goaml.py b/misp_modules/modules/export_mod/goaml.py index 82375c2..ce29f8f 100644 --- a/misp_modules/modules/export_mod/goaml.py +++ b/misp_modules/modules/export_mod/goaml.py @@ -11,7 +11,7 @@ mispattributes = {} outputFileExtension = "xml" responseType = "application/xml" -objects_to_parse = ['bank-account', 'person'] +objects_to_parse = ['bank-account', 'person', 'geolocation'] goAMLmapping = {'bank-account': 't_account', 'text': 'institution_name', 'institution-code': 'institution_code', 'swift': 'swift', 'branch': 'branch', 'non-banking-institution': 'non_bank_institution', @@ -23,7 +23,9 @@ goAMLmapping = {'bank-account': 't_account', 'text': 'institution_name', 'instit 'last-name': 'last_name', 'mothers-name': 'mothers_name', 'title': 'title', 'alias': 'alias', 'date-of-birth': 'birthdate', 'place-of-birth': 'birth_place', 'gender': 'gender', 'passport-number': 'passport_number', 'passport-country': 'passport_country', - 'social-security-number': 'ssn', 'nationality': 'nationality1'} + 'social-security-number': 'ssn', 'nationality': 'nationality1', 'identity-card-number': 'id_number', + 'geolocation': 'location', 'city': 'city', 'region': 'state', 'country': 'country-code', + 'address': 'address', 'zipcode': 'zip'} class GoAmlGeneration(): def __init__(self): @@ -55,13 +57,20 @@ class GoAmlGeneration(): def itterate(self): for t in self.document: self.xml += "<{}>".format(goAMLmapping[t]) - for k in self.document[t]: - try: - self.xml += "<{0}>{1}".format(goAMLmapping[k], self.document[t][k]) - except KeyError: - pass + self.fill_xml(t) + if t == 'bank-account' and 'person' in self.document: + self.xml += "" + self.fill_xml('person') + self.xml += "" self.xml += "".format(goAMLmapping[t]) + def fill_xml(self, t): + for k in self.document[t]: + try: + self.xml += "<{0}>{1}".format(goAMLmapping[k], self.document[t][k]) + except KeyError: + pass + def handler(q=False): if q is False: return False @@ -73,7 +82,7 @@ def handler(q=False): exp_doc.parse_objects() exp_doc.build_xml() return {'response': {}, 'data': exp_doc.xml} - #return {'response': [], 'data': str(base64.b64encode(bytes(exp_doc.document, 'utf-8')), 'utf-8')} + #return {'response': [], 'data': str(base64.b64encode(bytes(exp_doc.xml, 'utf-8')), 'utf-8')} def introspection(): modulesetup = {} From 8569c3d70231f8ae79e436560194c50b6ba28664 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Mon, 12 Feb 2018 13:40:49 +0100 Subject: [PATCH 32/66] Suporting the recent objects added to misp-objects - Matching the aml documents structure - Some parts of the document still need to be added --- misp_modules/modules/export_mod/goaml.py | 115 ++++++++++++++++------- 1 file changed, 80 insertions(+), 35 deletions(-) diff --git a/misp_modules/modules/export_mod/goaml.py b/misp_modules/modules/export_mod/goaml.py index ce29f8f..05803d5 100644 --- a/misp_modules/modules/export_mod/goaml.py +++ b/misp_modules/modules/export_mod/goaml.py @@ -1,5 +1,6 @@ import json from pymisp import MISPEvent +from collections import defaultdict import base64 misperrors = {'error': 'Error'} @@ -11,12 +12,12 @@ mispattributes = {} outputFileExtension = "xml" responseType = "application/xml" -objects_to_parse = ['bank-account', 'person', 'geolocation'] +objects_to_parse = ['transaction', 'bank-account', 'person', 'entity', 'geolocation'] -goAMLmapping = {'bank-account': 't_account', 'text': 'institution_name', 'institution-code': 'institution_code', +goAMLmapping = {'bank-account': 't_account', 'institution-code': 'institution_code', 'iban': 'iban', 'swift': 'swift', 'branch': 'branch', 'non-banking-institution': 'non_bank_institution', 'account': 'account', 'currency-code': 'currency_code', 'account-name': 'account_name', - 'iban': 'iban', 'client-number': 'client_number', 'personal-account-type': 'personal_account_type', + 'client-number': 'client_number', 'personal-account-type': 'personal_account_type', 'opened': 'opened', 'closed': 'closed', 'balance': 'balance', 'status-code': 'status_code', 'beneficiary': 'beneficiary', 'beneficiary-comment': 'beneficiary_comment', 'comments': 'comments', 'person': 't_person', 'text': 'comments', 'first-name': 'first_name', 'middle-name': 'middle_name', @@ -25,64 +26,108 @@ goAMLmapping = {'bank-account': 't_account', 'text': 'institution_name', 'instit 'passport-number': 'passport_number', 'passport-country': 'passport_country', 'social-security-number': 'ssn', 'nationality': 'nationality1', 'identity-card-number': 'id_number', 'geolocation': 'location', 'city': 'city', 'region': 'state', 'country': 'country-code', - 'address': 'address', 'zipcode': 'zip'} + 'address': 'address', 'zipcode': 'zip', + 'transaction': 'transaction', 'transaction-number': 'transactionnumber', 'date': 'date_transaction', + 'location': 'transaction_location', 'transmode-code': 'transmode_code', 'amount': 'amount_local', + 'transmode-comment': 'transmode_comment', 'date-posting': 'date_posting', + 'entity': 'entity', 'name': 'name', 'commercial-name': 'commercial_name', 'business': 'business', + 'legal-form': 'incorporation_legal_form', 'registration-number': 'incorporation_number', + 'phone-number': 'phone'} -class GoAmlGeneration(): - def __init__(self): - self.document = {} +class GoAmlGeneration(object): + def __init__(self, config): + self.config = config def from_event(self, event): self.misp_event = MISPEvent() self.misp_event.load(event) def parse_objects(self): + uuids = defaultdict(list) + report_code = [] + currency_code = [] for obj in self.misp_event.objects: - if obj.name in objects_to_parse: - obj_dict = {} - for attribute in obj.attributes: - obj_dict[attribute.object_relation] = attribute.value - self.document[obj.name] = obj_dict + obj_type = obj.name + uuids[obj_type].append(obj.uuid) + if obj_type == 'bank-account': + try: + report_code.append(obj.get_attributes_by_relation('report-code')) + currency_code.append(obj.get_attributes_by_relation('currency-code')) + except: + print('non') + self.uuids, self.report_code, self.currency_code = uuids, report_code, currency_code def build_xml(self): - self.xml = "" - if 'bank-account' in self.document: - if 'report-code' in self.document['bank-account']: - self.xml += "{}".format(self.document['bank-account'].pop('report-code').split(' ')[0]) - for a in ('personal-account-type', 'status-code'): - if a in self.document['bank-account']: - self.document['bank-account'][a] = self.document['bank-account'][a].split(' - ')[0] - self.itterate() + self.xml = "{}".format(self.config) + for trans_uuid in self.uuids.get('transaction'): + self.itterate('transaction', 'transaction', trans_uuid) self.xml += "" - def itterate(self): - for t in self.document: - self.xml += "<{}>".format(goAMLmapping[t]) - self.fill_xml(t) - if t == 'bank-account' and 'person' in self.document: - self.xml += "" - self.fill_xml('person') - self.xml += "" - self.xml += "".format(goAMLmapping[t]) + def itterate(self, object_type, aml_type, uuid): + self.xml += "<{}>".format(aml_type) + obj = self.misp_event.get_object_by_uuid(uuid) + self.fill_xml(obj) + if obj.ObjectReference: + for ref in obj.ObjectReference: + uuid = ref.referenced_uuid + next_object_type = ref.Object.get('name') + relationship_type = ref.relationship_type + self.parse_references(object_type, next_object_type, uuid, relationship_type) + self.xml += "".format(aml_type) - def fill_xml(self, t): - for k in self.document[t]: + def fill_xml(self, obj): + for attribute in obj.attributes: + if obj.name == 'bank-account' and attribute.type in ('personal-account-type', 'status-code'): + attribute_value = attribute.value.split(' - ')[0] + else: + attribute_value = attribute.value try: - self.xml += "<{0}>{1}".format(goAMLmapping[k], self.document[t][k]) + self.xml += "<{0}>{1}".format(goAMLmapping[attribute.object_relation], attribute_value) except KeyError: pass + def parse_references(self, object_type, next_object_type, uuid, relationship_type): + if next_object_type == 'bank-account': + self.xml += "".format(relationship_type) + next_aml_type = "{}_account".format(relationship_type.split('_')[0]) + self.itterate(next_object_type, next_aml_type, uuid) + self.xml += "".format(relationship_type) + elif next_object_type == 'person': + if object_type == 'transaction': + self.xml += "".format(relationship_type) + next_aml_type = "{}_person".format(relationship_type.split('_')[0]) + self.itterate(next_object_type, next_aml_type, uuid) + self.xml += "".format(relationship_type) + elif object_type == 'bank-account': + self.xml += "" + next_aml_type = goAMLmapping[next_object_type] + self.itterate(next_object_type, next_aml_type, uuid) + self.xml += "" + elif next_object_type == 'legal-entity': + if object_type == 'transaction': + self.xml += "".format(relationship_type) + next_aml_type = "{}_entity".format(relationship_type.split('_')[0]) + self.itterate(next_object_type, next_aml_type, uuid) + self.xml += "".format(relationship_type) + elif object_type == 'bank-account': + next_aml_type = goAMLmapping[next_object_type] + self.itterate(next_object_type, next_aml_type, uuid) + def handler(q=False): if q is False: return False request = json.loads(q) if 'data' not in request: return False - exp_doc = GoAmlGeneration() + if not request.get('config') and not request['config'].get('rentity_id'): + misperrors['error'] = "Configuration error." + return misperrors + config = request['config'].get('rentity_id') + exp_doc = GoAmlGeneration(config) exp_doc.from_event(request['data'][0]) exp_doc.parse_objects() exp_doc.build_xml() - return {'response': {}, 'data': exp_doc.xml} - #return {'response': [], 'data': str(base64.b64encode(bytes(exp_doc.xml, 'utf-8')), 'utf-8')} + return {'response': [], 'data': str(base64.b64encode(bytes(exp_doc.xml, 'utf-8')), 'utf-8')} def introspection(): modulesetup = {} From 43db92dbe60d21615793b3d4806d7c6ce830804e Mon Sep 17 00:00:00 2001 From: Dennis Rand Date: Mon, 12 Feb 2018 19:11:54 +0000 Subject: [PATCH 33/66] Added Yara syntax validation expansion module --- REQUIREMENTS | 1 + misp_modules/modules/expansion/__init__.py | 2 +- .../expansion/yara_syntax_validator.py | 38 +++++++++++++++++++ 3 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 misp_modules/modules/expansion/yara_syntax_validator.py diff --git a/REQUIREMENTS b/REQUIREMENTS index 3b31b16..9e383d4 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -21,3 +21,4 @@ domaintools_api pygeoip bs4 oauth2 +yara diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py index 7b56b62..e40e844 100644 --- a/misp_modules/modules/expansion/__init__.py +++ b/misp_modules/modules/expansion/__init__.py @@ -3,4 +3,4 @@ from . import _vmray __all__ = ['vmray_submit', 'asn_history', 'circl_passivedns', 'circl_passivessl', 'countrycode', 'cve', 'dns', 'domaintools', 'eupi', 'farsight_passivedns', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal', 'whois', 'shodan', 'reversedns', 'geoip_country', 'wiki', 'iprep', 'threatminer', 'otx', - 'threatcrowd', 'vulndb', 'crowdstrike_falcon'] + 'threatcrowd', 'vulndb', 'crowdstrike_falcon','yara_syntax_validator'] diff --git a/misp_modules/modules/expansion/yara_syntax_validator.py b/misp_modules/modules/expansion/yara_syntax_validator.py new file mode 100644 index 0000000..4953c41 --- /dev/null +++ b/misp_modules/modules/expansion/yara_syntax_validator.py @@ -0,0 +1,38 @@ +import json +import requests +try: + import yara +except: + print("yara is missing, use 'pip3 install yara' to install it.") + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['yara'], 'output': ['text']} +moduleinfo = {'version': '0.1', 'author': 'Dennis Rand', 'description': 'An expansion hover module to perform a syntax check on if yara rules are valid or not.', 'module-type': ['hover']} +moduleconfig = [] + + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + if not request.get('yara'): + misperrors['error'] = 'Yara rule missing' + return misperrors + + try: + rules = yara.compile(source=request.get('yara')) + summary = ("Syntax valid") + except Exception as e: + summary = ("Syntax error: " + str(e)) + + r = {'results': [{'types': mispattributes['output'], 'values': summary}]} + return r + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo From 37ea090cbaaf0060c01a40ec72295fbc01ca3583 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Mon, 12 Feb 2018 21:13:32 +0100 Subject: [PATCH 34/66] add: YARA syntax validator --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index dcf6c85..11c28b4 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,7 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/ * [virustotal](misp_modules/modules/expansion/virustotal.py) - an expansion module to pull known resolutions and malware samples related with an IP/Domain from virusTotal (this modules require a VirusTotal private API key) * [wikidata](misp_modules/modules/expansion/wiki.py) - a [wikidata](https://www.wikidata.org) expansion module. * [xforce](misp_modules/modules/expansion/xforceexchange.py) - an IBM X-Force Exchange expansion module. +* [YARA syntax validator](misp_modules/modules/expansion/yara_syntax_validator.py) - YARA syntax validator. ### Export modules From a97eeb44fef91677591c46c1b404e19f9d5389f2 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Tue, 13 Feb 2018 11:51:34 +0100 Subject: [PATCH 35/66] Added some report information Also changed the ObjectReference parser to replace all the if conditions by a dictionary reading --- misp_modules/modules/export_mod/goaml.py | 99 +++++++++++++----------- 1 file changed, 52 insertions(+), 47 deletions(-) diff --git a/misp_modules/modules/export_mod/goaml.py b/misp_modules/modules/export_mod/goaml.py index 05803d5..798cc00 100644 --- a/misp_modules/modules/export_mod/goaml.py +++ b/misp_modules/modules/export_mod/goaml.py @@ -30,13 +30,19 @@ goAMLmapping = {'bank-account': 't_account', 'institution-code': 'institution_co 'transaction': 'transaction', 'transaction-number': 'transactionnumber', 'date': 'date_transaction', 'location': 'transaction_location', 'transmode-code': 'transmode_code', 'amount': 'amount_local', 'transmode-comment': 'transmode_comment', 'date-posting': 'date_posting', - 'entity': 'entity', 'name': 'name', 'commercial-name': 'commercial_name', 'business': 'business', + 'legal-entity': 'entity', 'name': 'name', 'commercial-name': 'commercial_name', 'business': 'business', 'legal-form': 'incorporation_legal_form', 'registration-number': 'incorporation_number', 'phone-number': 'phone'} +referencesMapping = {'bank-account': {'aml_type': '{}_account', 'bracket': 't_{}'}, + 'person': {'transaction': {'aml_type': '{}_person', 'bracket': 't_{}'}, 'bank-account': {'aml_type': 't_person', 'bracket': 'signatory'}}, + 'legal-entity': {'transaction': {'aml_type': '{}_entity', 'bracket': 't_{}'}, 'bank-account': {'aml_type': 'entity'}}, + 'geolocation': {'aml_type': 'address', 'bracket': 'addresses'}} + class GoAmlGeneration(object): def __init__(self, config): self.config = config + self.parsed_uuids = defaultdict(list) def from_event(self, event): self.misp_event = MISPEvent() @@ -51,67 +57,65 @@ class GoAmlGeneration(object): uuids[obj_type].append(obj.uuid) if obj_type == 'bank-account': try: - report_code.append(obj.get_attributes_by_relation('report-code')) - currency_code.append(obj.get_attributes_by_relation('currency-code')) + report_code.append(obj.get_attributes_by_relation('report-code')[0]) + currency_code.append(obj.get_attributes_by_relation('currency-code')[0]) except: - print('non') - self.uuids, self.report_code, self.currency_code = uuids, report_code, currency_code + print('report_code or currency_code error') + self.uuids, self.report_codes, self.currency_codes = uuids, report_code, currency_code def build_xml(self): - self.xml = "{}".format(self.config) + self.xml = {'header': "{}".format(self.config), + 'data': ""} for trans_uuid in self.uuids.get('transaction'): - self.itterate('transaction', 'transaction', trans_uuid) - self.xml += "" + self.itterate('transaction', 'transaction', trans_uuid, 'data') + person_to_parse = [person_uuid for person_uuid in self.uuids.get('person') if person_uuid not in self.parsed_uuids.get('person')] + if len(person_to_parse) == 1: + self.itterate('person', 'reporting_person', person_to_parse[0], 'header') + location_to_parse = [location_uuid for location_uuid in self.uuids.get('geolocation') if location_uuid not in self.parsed_uuids.get('geolocation')] + if len(location_to_parse) == 1: + self.itterate('geolocation', 'location', location_to_parse[0], 'header') + self.xml['data'] += "" - def itterate(self, object_type, aml_type, uuid): - self.xml += "<{}>".format(aml_type) + def itterate(self, object_type, aml_type, uuid, xml_part): + self.xml[xml_part] += "<{}>".format(aml_type) obj = self.misp_event.get_object_by_uuid(uuid) - self.fill_xml(obj) + self.fill_xml(obj, xml_part) + self.parsed_uuids[object_type].append(uuid) if obj.ObjectReference: for ref in obj.ObjectReference: uuid = ref.referenced_uuid next_object_type = ref.Object.get('name') relationship_type = ref.relationship_type - self.parse_references(object_type, next_object_type, uuid, relationship_type) - self.xml += "".format(aml_type) + self.parse_references(object_type, next_object_type, uuid, relationship_type, xml_part) + self.xml[xml_part] += "".format(aml_type) - def fill_xml(self, obj): + def fill_xml(self, obj, xml_part): for attribute in obj.attributes: - if obj.name == 'bank-account' and attribute.type in ('personal-account-type', 'status-code'): + if obj.name == 'bank-account' and attribute.object_relation in ('personal-account-type', 'status-code'): attribute_value = attribute.value.split(' - ')[0] else: attribute_value = attribute.value try: - self.xml += "<{0}>{1}".format(goAMLmapping[attribute.object_relation], attribute_value) + self.xml[xml_part] += "<{0}>{1}".format(goAMLmapping[attribute.object_relation], attribute_value) except KeyError: pass - def parse_references(self, object_type, next_object_type, uuid, relationship_type): - if next_object_type == 'bank-account': - self.xml += "".format(relationship_type) - next_aml_type = "{}_account".format(relationship_type.split('_')[0]) - self.itterate(next_object_type, next_aml_type, uuid) - self.xml += "".format(relationship_type) - elif next_object_type == 'person': - if object_type == 'transaction': - self.xml += "".format(relationship_type) - next_aml_type = "{}_person".format(relationship_type.split('_')[0]) - self.itterate(next_object_type, next_aml_type, uuid) - self.xml += "".format(relationship_type) - elif object_type == 'bank-account': - self.xml += "" - next_aml_type = goAMLmapping[next_object_type] - self.itterate(next_object_type, next_aml_type, uuid) - self.xml += "" - elif next_object_type == 'legal-entity': - if object_type == 'transaction': - self.xml += "".format(relationship_type) - next_aml_type = "{}_entity".format(relationship_type.split('_')[0]) - self.itterate(next_object_type, next_aml_type, uuid) - self.xml += "".format(relationship_type) - elif object_type == 'bank-account': - next_aml_type = goAMLmapping[next_object_type] - self.itterate(next_object_type, next_aml_type, uuid) + def parse_references(self, object_type, next_object_type, uuid, relationship_type, xml_part): + try: + next_aml_type = referencesMapping[next_object_type][object_type].get('aml_type').format(relationship_type.split('_')[0]) + try: + bracket = referencesMapping[next_object_type][object_type].get('bracket').format(relationship_type) + self.xml[xml_part] += "<{}>".format(bracket) + self.itterate(next_object_type, next_aml_type, uuid, xml_part) + self.xml[xml_part] += "".format(bracket) + except KeyError: + self.itterate(next_object_type, next_aml_type, uuid, xml_part) + except KeyError: + next_aml_type = referencesMapping[next_object_type].get('aml_type').format(relationship_type.split('_')[0]) + bracket = referencesMapping[next_object_type].get('bracket').format(relationship_type) + self.xml[xml_part] += "<{}>".format(bracket) + self.itterate(next_object_type, next_aml_type, uuid, xml_part) + self.xml[xml_part] += "".format(bracket) def handler(q=False): if q is False: @@ -123,11 +127,12 @@ def handler(q=False): misperrors['error'] = "Configuration error." return misperrors config = request['config'].get('rentity_id') - exp_doc = GoAmlGeneration(config) - exp_doc.from_event(request['data'][0]) - exp_doc.parse_objects() - exp_doc.build_xml() - return {'response': [], 'data': str(base64.b64encode(bytes(exp_doc.xml, 'utf-8')), 'utf-8')} + export_doc = GoAmlGeneration(config) + export_doc.from_event(request['data'][0]) + export_doc.parse_objects() + export_doc.build_xml() + exp_doc = "{}{}".format(export_doc.xml.get('header'), export_doc.xml.get('data')) + return {'response': [], 'data': str(base64.b64encode(bytes(exp_doc, 'utf-8')), 'utf-8')} def introspection(): modulesetup = {} From d4538382d03bf59a6bdad76a24a9efb15b81cad2 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Tue, 13 Feb 2018 13:41:22 +0100 Subject: [PATCH 36/66] Simplified ObjectReference dictionary reading --- misp_modules/modules/export_mod/goaml.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/misp_modules/modules/export_mod/goaml.py b/misp_modules/modules/export_mod/goaml.py index 798cc00..fd5cde7 100644 --- a/misp_modules/modules/export_mod/goaml.py +++ b/misp_modules/modules/export_mod/goaml.py @@ -101,18 +101,19 @@ class GoAmlGeneration(object): pass def parse_references(self, object_type, next_object_type, uuid, relationship_type, xml_part): + reference = referencesMapping[next_object_type] try: - next_aml_type = referencesMapping[next_object_type][object_type].get('aml_type').format(relationship_type.split('_')[0]) + next_aml_type = reference[object_type].get('aml_type').format(relationship_type.split('_')[0]) try: - bracket = referencesMapping[next_object_type][object_type].get('bracket').format(relationship_type) + bracket = reference[object_type].get('bracket').format(relationship_type) self.xml[xml_part] += "<{}>".format(bracket) self.itterate(next_object_type, next_aml_type, uuid, xml_part) self.xml[xml_part] += "".format(bracket) except KeyError: self.itterate(next_object_type, next_aml_type, uuid, xml_part) except KeyError: - next_aml_type = referencesMapping[next_object_type].get('aml_type').format(relationship_type.split('_')[0]) - bracket = referencesMapping[next_object_type].get('bracket').format(relationship_type) + next_aml_type = reference.get('aml_type').format(relationship_type.split('_')[0]) + bracket = reference.get('bracket').format(relationship_type) self.xml[xml_part] += "<{}>".format(bracket) self.itterate(next_object_type, next_aml_type, uuid, xml_part) self.xml[xml_part] += "".format(bracket) From 43e9010858d4a8652c98d47e9cf11bb1efdb94e4 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Tue, 13 Feb 2018 16:39:19 +0100 Subject: [PATCH 37/66] Added report expected information fields --- misp_modules/modules/export_mod/goaml.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/misp_modules/modules/export_mod/goaml.py b/misp_modules/modules/export_mod/goaml.py index fd5cde7..23f9ba0 100644 --- a/misp_modules/modules/export_mod/goaml.py +++ b/misp_modules/modules/export_mod/goaml.py @@ -1,7 +1,6 @@ -import json +import json, datetime, base64 from pymisp import MISPEvent -from collections import defaultdict -import base64 +from collections import defaultdict, Counter misperrors = {'error': 'Error'} moduleinfo = {'version': '1', 'author': 'Christian Studer', @@ -57,15 +56,23 @@ class GoAmlGeneration(object): uuids[obj_type].append(obj.uuid) if obj_type == 'bank-account': try: - report_code.append(obj.get_attributes_by_relation('report-code')[0]) - currency_code.append(obj.get_attributes_by_relation('currency-code')[0]) + report_code.append(obj.get_attributes_by_relation('report-code')[0].value.split(' ')[0]) + currency_code.append(obj.get_attributes_by_relation('currency-code')[0].value) except: print('report_code or currency_code error') self.uuids, self.report_codes, self.currency_codes = uuids, report_code, currency_code def build_xml(self): - self.xml = {'header': "{}".format(self.config), + self.xml = {'header': "{}E".format(self.config), 'data': ""} + if "STR" in self.report_codes: + report_code = "STR" + else: + report_code = Counter(self.report_codes).most_common(1)[0][0] + self.xml['header'] += "{}".format(report_code) + submission_date = str(self.misp_event.timestamp).replace(' ', 'T') + self.xml['header'] += "{}".format(submission_date) + self.xml['header'] += "{}".format(Counter(self.currency_codes).most_common(1)[0][0]) for trans_uuid in self.uuids.get('transaction'): self.itterate('transaction', 'transaction', trans_uuid, 'data') person_to_parse = [person_uuid for person_uuid in self.uuids.get('person') if person_uuid not in self.parsed_uuids.get('person')] From be1b541966396e8e67fbcbdd4c508f8ac0010610 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Wed, 14 Feb 2018 12:18:12 +0100 Subject: [PATCH 38/66] Added a required field & the latest attributes in transaction --- misp_modules/modules/export_mod/goaml.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/misp_modules/modules/export_mod/goaml.py b/misp_modules/modules/export_mod/goaml.py index 23f9ba0..acddd43 100644 --- a/misp_modules/modules/export_mod/goaml.py +++ b/misp_modules/modules/export_mod/goaml.py @@ -28,7 +28,8 @@ goAMLmapping = {'bank-account': 't_account', 'institution-code': 'institution_co 'address': 'address', 'zipcode': 'zip', 'transaction': 'transaction', 'transaction-number': 'transactionnumber', 'date': 'date_transaction', 'location': 'transaction_location', 'transmode-code': 'transmode_code', 'amount': 'amount_local', - 'transmode-comment': 'transmode_comment', 'date-posting': 'date_posting', + 'transmode-comment': 'transmode_comment', 'date-posting': 'date_posting', 'teller': 'teller', + 'authorized': 'authorized', 'legal-entity': 'entity', 'name': 'name', 'commercial-name': 'commercial_name', 'business': 'business', 'legal-form': 'incorporation_legal_form', 'registration-number': 'incorporation_number', 'phone-number': 'phone'} @@ -102,6 +103,8 @@ class GoAmlGeneration(object): attribute_value = attribute.value.split(' - ')[0] else: attribute_value = attribute.value + if obj.name == 'transaction' and attribute.object_relation == 'date-posting': + self.xml[xml_part] += "True" try: self.xml[xml_part] += "<{0}>{1}".format(goAMLmapping[attribute.object_relation], attribute_value) except KeyError: From 92ab1d5c23e4c6b05da2d8caaa98a80c8eebf56e Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Wed, 14 Feb 2018 21:30:48 +0100 Subject: [PATCH 39/66] Added "t_to" and "t_from" required fields: funds code & country --- misp_modules/modules/export_mod/goaml.py | 80 +++++++++++++++++++----- 1 file changed, 63 insertions(+), 17 deletions(-) diff --git a/misp_modules/modules/export_mod/goaml.py b/misp_modules/modules/export_mod/goaml.py index acddd43..61311e6 100644 --- a/misp_modules/modules/export_mod/goaml.py +++ b/misp_modules/modules/export_mod/goaml.py @@ -4,10 +4,10 @@ from collections import defaultdict, Counter misperrors = {'error': 'Error'} moduleinfo = {'version': '1', 'author': 'Christian Studer', - 'description': '', + 'description': 'Export to GoAML', 'module-type': ['export']} -moduleconfig = [] -mispattributes = {} +moduleconfig = ['rentity_id'] +mispattributes = {'input': ['MISPEvent'], 'output': ['xml file']} outputFileExtension = "xml" responseType = "application/xml" @@ -85,30 +85,76 @@ class GoAmlGeneration(object): self.xml['data'] += "" def itterate(self, object_type, aml_type, uuid, xml_part): - self.xml[xml_part] += "<{}>".format(aml_type) obj = self.misp_event.get_object_by_uuid(uuid) + if object_type == 'transaction': + self.xml[xml_part] += "<{}>".format(aml_type) + self.fill_xml_transaction(obj.attributes, xml_part) + self.parsed_uuids[object_type].append(uuid) + if obj.ObjectReference: + self.parseObjectReferences(object_type, xml_part, obj.ObjectReference) + self.xml[xml_part] += "".format(aml_type) + else: + if 'to_' in aml_type or 'from_' in aml_type: + relation_type = aml_type.split('_')[0] + self.xml[xml_part] += "<{0}_funds_code>{1}".format(relation_type, self.from_and_to_fields[relation_type]['funds'].split(' ')[0]) + self.itterate_normal_case(object_type, obj, aml_type, uuid, xml_part) + self.xml[xml_part] += "<{0}_country>{1}".format(relation_type, self.from_and_to_fields[relation_type]['country']) + else: + self.itterate_normal_case(object_type, obj, aml_type, uuid, xml_part) + + def itterate_normal_case(self, object_type, obj, aml_type, uuid, xml_part): + self.xml[xml_part] += "<{}>".format(aml_type) self.fill_xml(obj, xml_part) self.parsed_uuids[object_type].append(uuid) if obj.ObjectReference: - for ref in obj.ObjectReference: - uuid = ref.referenced_uuid - next_object_type = ref.Object.get('name') - relationship_type = ref.relationship_type - self.parse_references(object_type, next_object_type, uuid, relationship_type, xml_part) + self.parseObjectReferences(object_type, xml_part, obj.ObjectReference) self.xml[xml_part] += "".format(aml_type) - def fill_xml(self, obj, xml_part): - for attribute in obj.attributes: - if obj.name == 'bank-account' and attribute.object_relation in ('personal-account-type', 'status-code'): - attribute_value = attribute.value.split(' - ')[0] - else: - attribute_value = attribute.value - if obj.name == 'transaction' and attribute.object_relation == 'date-posting': + def parseObjectReferences(self, object_type, xml_part, references): + for ref in references: + next_uuid = ref.referenced_uuid + next_object_type = ref.Object.get('name') + relationship_type = ref.relationship_type + self.parse_references(object_type, next_object_type, next_uuid, relationship_type, xml_part) + + def fill_xml_transaction(self, attributes, xml_part): + from_and_to_fields = {'from': {}, 'to': {}} + for attribute in attributes: + object_relation = attribute.object_relation + attribute_value = attribute.value + if object_relation == 'date-posting': self.xml[xml_part] += "True" + elif object_relation in ('from-funds-code', 'to-funds-code'): + relation_type, field, _ = object_relation.split('-') + from_and_to_fields[relation_type][field] = attribute_value + continue + elif object_relation in ('from-country', 'to-country'): + relation_type, field = object_relation.split('-') + from_and_to_fields[relation_type][field] = attribute_value + continue try: - self.xml[xml_part] += "<{0}>{1}".format(goAMLmapping[attribute.object_relation], attribute_value) + self.xml[xml_part] += "<{0}>{1}".format(goAMLmapping[object_relation], attribute_value) except KeyError: pass + self.from_and_to_fields = from_and_to_fields + + def fill_xml(self, obj, xml_part): + if obj.name == 'bank-account': + for attribute in obj.attributes: + if attribute.object_relation in ('personal-account-type', 'status-code'): + attribute_value = attribute.value.split(' - ')[0] + else: + attribute_value = attribute.value + try: + self.xml[xml_part] += "<{0}>{1}".format(goAMLmapping[attribute.object_relation], attribute_value) + except KeyError: + pass + else: + for attribute in obj.attributes: + try: + self.xml[xml_part] += "<{0}>{1}".format(goAMLmapping[attribute.object_relation], attribute.value) + except KeyError: + pass def parse_references(self, object_type, next_object_type, uuid, relationship_type, xml_part): reference = referencesMapping[next_object_type] From 978903f9111bd62bbc63d5150e89969bbb1b1856 Mon Sep 17 00:00:00 2001 From: Andras Iklody Date: Tue, 20 Feb 2018 14:08:14 +0100 Subject: [PATCH 40/66] Quick fix to the invalid hash types offered on all returned hashes, hopefully fixes #162 --- misp_modules/modules/expansion/virustotal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/virustotal.py b/misp_modules/modules/expansion/virustotal.py index 44199a1..3997ee6 100755 --- a/misp_modules/modules/expansion/virustotal.py +++ b/misp_modules/modules/expansion/virustotal.py @@ -152,7 +152,7 @@ def getMoreInfo(req, key): # Get all hashes first hashes = [] hashes = findAll(req, ["md5", "sha1", "sha256", "sha512"]) - r.append({"types": ["md5", "sha1", "sha256", "sha512"], "values": hashes}) + r.append({"types": ["freetext"], "values": hashes}) for hsh in hashes[:limit]: # Search VT for some juicy info try: From eb9e06f1cc4aa07cd35c3766cdcb83b7f580eab5 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Tue, 20 Feb 2018 15:15:54 +0100 Subject: [PATCH 41/66] explicit name Avoiding confusion with the coming import module for goaml --- misp_modules/modules/export_mod/{goaml.py => goamlexport.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename misp_modules/modules/export_mod/{goaml.py => goamlexport.py} (100%) diff --git a/misp_modules/modules/export_mod/goaml.py b/misp_modules/modules/export_mod/goamlexport.py similarity index 100% rename from misp_modules/modules/export_mod/goaml.py rename to misp_modules/modules/export_mod/goamlexport.py From 02b8938b2aefbe10d4eb5099206be492753fb136 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Tue, 20 Feb 2018 16:57:27 +0100 Subject: [PATCH 42/66] typo --- misp_modules/modules/export_mod/goamlexport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/export_mod/goamlexport.py b/misp_modules/modules/export_mod/goamlexport.py index 61311e6..2859b3c 100644 --- a/misp_modules/modules/export_mod/goamlexport.py +++ b/misp_modules/modules/export_mod/goamlexport.py @@ -210,7 +210,7 @@ def introspection(): pass try: inputSource - mmoduleSetup['inputSource'] = inputSource + moduleSetup['inputSource'] = inputSource except NameError: pass return modulesetup From f361fb4ee3f9de69bbe6d0c27c42af7e4f373769 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Tue, 20 Feb 2018 17:00:13 +0100 Subject: [PATCH 43/66] Reading the entire document, to create a big dictionary containing the data, as a beginning --- .../modules/import_mod/goamlimport.py | 99 +++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 misp_modules/modules/import_mod/goamlimport.py diff --git a/misp_modules/modules/import_mod/goamlimport.py b/misp_modules/modules/import_mod/goamlimport.py new file mode 100644 index 0000000..dbef826 --- /dev/null +++ b/misp_modules/modules/import_mod/goamlimport.py @@ -0,0 +1,99 @@ +import json, datetime +import xml.etree.ElementTree as ET +from collections import defaultdict + +misperrors = {'error': 'Error'} +moduleinfo = {'version': 1, 'author': 'Christian Studer', + 'description': 'Import from GoAML', + 'module-type': ['import']} +moduleconfig = [] +mispattributes = {'input': ['xml file'], 'output': ['MISPEvent']} + +t_from = {'nodes': ['from_person', 'from_account', 'from_entity'], + 'leaves': ['from_funds_code', 'from_country']} +t_to = {'nodes': ['to_person', 'to_account', 'to_entity'], + 'leaves': ['to_funds_code', 'to_country']} +t_person = {'nodes': ['addresses'], + 'leaves': ['first_name', 'middle_name', 'last_name', 'gender', 'title', 'mothers_name', 'birthdate', + 'passport_number', 'passport_country', 'id_number', 'birth_place', 'alias', 'nationality1']} +t_account = {'nodes': ['signatory'], + 'leaves': ['institution_name', 'institution_code', 'swift', 'branch', 'non_banking_insitution', + 'account', 'currency_code', 'account_name', 'iban', 'client_number', 'opened', 'closed', + 'personal_account_type', 'balance', 'date_balance', 'status_code', 'beneficiary', + 'beneficiary_comment', 'comments']} +entity = {'nodes': ['addresses'], + 'leaves': ['name', 'commercial_name', 'incorporation_legal_form', 'incorporation_number', 'business', 'phone']} + +goAMLobjects = {'report': {'nodes': ['reporting_person', 'location', 'transaction'], + 'leaves': ['rentity_id', 'submission_code', 'report_code', 'submission_date', + 'currency_code_local']}, + 'reporting_person': {'nodes': ['addresses'], + 'leaves': ['first_name', 'middle_name', 'last_name', 'title']}, + 'location': {'nodes': [], + 'leaves': ['address_type', 'address', 'city', 'zip', 'country_code', 'state']}, + 'transaction': {'nodes': ['t_from', 't_from_my_client', 't_to', 't_to_my_client'], + 'leaves': ['transactionnumber', 'transaction_location', 'date_transaction', + 'transmode_code', 'amount_local']}, + 't_from': t_from, + 't_from_my_client': t_from, + 't_to': t_to, + 't_to_my_client': t_to, + 'addresses': {'nodes': ['address'], 'leaves': []}, + 'address': {'nodes': [], + 'leaves': ['address_type', 'address', 'city', 'zip', 'country_code', 'state']}, + 'from_person': t_person, + 'to_person': t_person, + 't_person': t_person, + 'from_account': t_account, + 'to_account': t_account, + 'signatory': {'nodes': ['t_person'], 'leaves': []}, + 'from_entity': entity, + 'to_entity': entity, + } + +class GoAmlParser(): + def __init__(self): + self.dict = defaultdict(list) + + def readFile(self, filename): + self.tree = ET.parse(filename).getroot() + + def parse_xml(self): + self.itterate(self.tree, 'report') + + def itterate(self, tree, aml_type): + elementDict = {} + for element in tree: + tag = element.tag + print(tag) + mapping = goAMLobjects.get(aml_type) + if tag in mapping.get('nodes'): + self.itterate(element, tag) + elif tag in mapping.get('leaves'): + elementDict[tag] = element.text + self.dict[aml_type].append(elementDict) + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + if request.get('file'): + filename = request['file'] + else: + misperrors['error'] = "Unsupported attributes type" + return misperrors + aml_parser = GoAmlParser() + try: + aml_parser.readFile(filename) + except: + misperrors['error'] = "Impossible to read the file" + return misperrors + aml_parser.parse_xml() + return aml_parser.dict + +def introspection(): + return mispattributes + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo From c3ac53a069aa5d2ee95d7ef8d5528fd7f59b216f Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Tue, 20 Feb 2018 17:18:36 +0100 Subject: [PATCH 44/66] fix: goamlexport added --- misp_modules/modules/export_mod/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/export_mod/__init__.py b/misp_modules/modules/export_mod/__init__.py index fff02d7..0034f5d 100644 --- a/misp_modules/modules/export_mod/__init__.py +++ b/misp_modules/modules/export_mod/__init__.py @@ -1 +1 @@ -__all__ = ['testexport','cef_export','liteexport','threat_connect_export','pdfexport','threatStream_misp_export'] +__all__ = ['testexport','cef_export','liteexport','goamlexport','threat_connect_export','pdfexport','threatStream_misp_export'] From 5995458aab0179d50925a14bb12d1d3605176990 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Wed, 21 Feb 2018 17:14:26 +0100 Subject: [PATCH 45/66] fix: Added the moduleinfo field need to have MISP event in standard format --- misp_modules/modules/export_mod/goamlexport.py | 3 ++- misp_modules/modules/export_mod/pdfexport.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/export_mod/goamlexport.py b/misp_modules/modules/export_mod/goamlexport.py index 2859b3c..e678024 100644 --- a/misp_modules/modules/export_mod/goamlexport.py +++ b/misp_modules/modules/export_mod/goamlexport.py @@ -5,7 +5,8 @@ from collections import defaultdict, Counter misperrors = {'error': 'Error'} moduleinfo = {'version': '1', 'author': 'Christian Studer', 'description': 'Export to GoAML', - 'module-type': ['export']} + 'module-type': ['export'], + 'require_standard_format': True} moduleconfig = ['rentity_id'] mispattributes = {'input': ['MISPEvent'], 'output': ['xml file']} outputFileExtension = "xml" diff --git a/misp_modules/modules/export_mod/pdfexport.py b/misp_modules/modules/export_mod/pdfexport.py index 4ee7bd7..2aeaec7 100755 --- a/misp_modules/modules/export_mod/pdfexport.py +++ b/misp_modules/modules/export_mod/pdfexport.py @@ -15,7 +15,8 @@ misperrors = {'error': 'Error'} moduleinfo = {'version': '1', 'author': 'Raphaël Vinot', 'description': 'Simple export to PDF', - 'module-type': ['export']} + 'module-type': ['export'], + 'require_standard_format': True} moduleconfig = [] From 9b34602f73beac61083fa8669c0b40e6a2c0b787 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Thu, 22 Feb 2018 01:22:08 +0100 Subject: [PATCH 46/66] Added GoAML export module in description --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 11c28b4..67ba189 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,7 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/ ### Export modules * [CEF](misp_modules/modules/export_mod/cef_export.py) module to export Common Event Format (CEF). +* [GoAML export](misp_modules/modules/export_mod/goamlexport.py) module to export in GoAML format. * [Lite Export](misp_modules/modules/export_mod/liteexport.py) module to export a lite event. * [Simple PDF export](misp_modules/modules/export_mod/pdfexport.py) module to export in PDF (required: asciidoctor-pdf). * [ThreatConnect](misp_modules/modules/export_mod/threat_connect_export.py) module to export in ThreatConnect CSV format. From c942013812512184fb79e2bd20dc39c070580c18 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Thu, 22 Feb 2018 01:23:08 +0100 Subject: [PATCH 47/66] chg: Modified the mapping dictionary to support misp-objects updates --- .../modules/export_mod/goamlexport.py | 61 +++++++++++-------- 1 file changed, 34 insertions(+), 27 deletions(-) diff --git a/misp_modules/modules/export_mod/goamlexport.py b/misp_modules/modules/export_mod/goamlexport.py index e678024..e732584 100644 --- a/misp_modules/modules/export_mod/goamlexport.py +++ b/misp_modules/modules/export_mod/goamlexport.py @@ -14,26 +14,33 @@ responseType = "application/xml" objects_to_parse = ['transaction', 'bank-account', 'person', 'entity', 'geolocation'] -goAMLmapping = {'bank-account': 't_account', 'institution-code': 'institution_code', 'iban': 'iban', - 'swift': 'swift', 'branch': 'branch', 'non-banking-institution': 'non_bank_institution', - 'account': 'account', 'currency-code': 'currency_code', 'account-name': 'account_name', - 'client-number': 'client_number', 'personal-account-type': 'personal_account_type', - 'opened': 'opened', 'closed': 'closed', 'balance': 'balance', 'status-code': 'status_code', - 'beneficiary': 'beneficiary', 'beneficiary-comment': 'beneficiary_comment', 'comments': 'comments', - 'person': 't_person', 'text': 'comments', 'first-name': 'first_name', 'middle-name': 'middle_name', - 'last-name': 'last_name', 'mothers-name': 'mothers_name', 'title': 'title', 'alias': 'alias', - 'date-of-birth': 'birthdate', 'place-of-birth': 'birth_place', 'gender': 'gender', - 'passport-number': 'passport_number', 'passport-country': 'passport_country', - 'social-security-number': 'ssn', 'nationality': 'nationality1', 'identity-card-number': 'id_number', - 'geolocation': 'location', 'city': 'city', 'region': 'state', 'country': 'country-code', - 'address': 'address', 'zipcode': 'zip', - 'transaction': 'transaction', 'transaction-number': 'transactionnumber', 'date': 'date_transaction', - 'location': 'transaction_location', 'transmode-code': 'transmode_code', 'amount': 'amount_local', - 'transmode-comment': 'transmode_comment', 'date-posting': 'date_posting', 'teller': 'teller', - 'authorized': 'authorized', - 'legal-entity': 'entity', 'name': 'name', 'commercial-name': 'commercial_name', 'business': 'business', - 'legal-form': 'incorporation_legal_form', 'registration-number': 'incorporation_number', - 'phone-number': 'phone'} +goAMLmapping = {'bank-account': {'bank-account': 't_account', 'institution-name': 'institution_name', + 'institution-code': 'institution_code', 'iban': 'iban', 'swift': 'swift', + 'branch': 'branch', 'non-banking-institution': 'non_bank_institution', + 'account': 'account', 'currency-code': 'currency_code', + 'account-name': 'account_name', 'client-number': 'client_number', + 'personal-account-type': 'personal_account_type', 'opened': 'opened', + 'closed': 'closed', 'balance': 'balance', 'status-code': 'status_code', + 'beneficiary': 'beneficiary', 'beneficiary-comment': 'beneficiary_comment', + 'comments': 'comments'}, + 'person': {'person': 't_person', 'text': 'comments', 'first-name': 'first_name', + 'middle-name': 'middle_name', 'last-name': 'last_name', 'title': 'title', + 'mothers-name': 'mothers_name', 'alias': 'alias', 'date-of-birth': 'birthdate', + 'place-of-birth': 'birth_place', 'gender': 'gender','nationality': 'nationality1', + 'passport-number': 'passport_number', 'passport-country': 'passport_country', + 'social-security-number': 'ssn', 'identity-card-number': 'id_number'}, + 'geolocation': {'geolocation': 'location', 'city': 'city', 'region': 'state', + 'country': 'country-code', 'address': 'address', 'zipcode': 'zip'}, + 'transaction': {'transaction': 'transaction', 'transaction-number': 'transactionnumber', + 'date': 'date_transaction', 'location': 'transaction_location', + 'transmode-code': 'transmode_code', 'amount': 'amount_local', + 'transmode-comment': 'transmode_comment', 'date-posting': 'date_posting', + 'teller': 'teller', 'authorized': 'authorized', + 'text': 'transaction_description'}, + 'legal-enitty': {'legal-entity': 'entity', 'name': 'name', 'business': 'business', + 'commercial-name': 'commercial_name', 'phone-number': 'phone', + 'legal-form': 'incorporation_legal_form', + 'registration-number': 'incorporation_number'}} referencesMapping = {'bank-account': {'aml_type': '{}_account', 'bracket': 't_{}'}, 'person': {'transaction': {'aml_type': '{}_person', 'bracket': 't_{}'}, 'bank-account': {'aml_type': 't_person', 'bracket': 'signatory'}}, @@ -89,7 +96,7 @@ class GoAmlGeneration(object): obj = self.misp_event.get_object_by_uuid(uuid) if object_type == 'transaction': self.xml[xml_part] += "<{}>".format(aml_type) - self.fill_xml_transaction(obj.attributes, xml_part) + self.fill_xml_transaction(object_type, obj.attributes, xml_part) self.parsed_uuids[object_type].append(uuid) if obj.ObjectReference: self.parseObjectReferences(object_type, xml_part, obj.ObjectReference) @@ -105,7 +112,7 @@ class GoAmlGeneration(object): def itterate_normal_case(self, object_type, obj, aml_type, uuid, xml_part): self.xml[xml_part] += "<{}>".format(aml_type) - self.fill_xml(obj, xml_part) + self.fill_xml(object_type, obj, xml_part) self.parsed_uuids[object_type].append(uuid) if obj.ObjectReference: self.parseObjectReferences(object_type, xml_part, obj.ObjectReference) @@ -118,7 +125,7 @@ class GoAmlGeneration(object): relationship_type = ref.relationship_type self.parse_references(object_type, next_object_type, next_uuid, relationship_type, xml_part) - def fill_xml_transaction(self, attributes, xml_part): + def fill_xml_transaction(self, object_type, attributes, xml_part): from_and_to_fields = {'from': {}, 'to': {}} for attribute in attributes: object_relation = attribute.object_relation @@ -134,12 +141,12 @@ class GoAmlGeneration(object): from_and_to_fields[relation_type][field] = attribute_value continue try: - self.xml[xml_part] += "<{0}>{1}".format(goAMLmapping[object_relation], attribute_value) + self.xml[xml_part] += "<{0}>{1}".format(goAMLmapping[object_type][object_relation], attribute_value) except KeyError: pass self.from_and_to_fields = from_and_to_fields - def fill_xml(self, obj, xml_part): + def fill_xml(self, object_type, obj, xml_part): if obj.name == 'bank-account': for attribute in obj.attributes: if attribute.object_relation in ('personal-account-type', 'status-code'): @@ -147,13 +154,13 @@ class GoAmlGeneration(object): else: attribute_value = attribute.value try: - self.xml[xml_part] += "<{0}>{1}".format(goAMLmapping[attribute.object_relation], attribute_value) + self.xml[xml_part] += "<{0}>{1}".format(goAMLmapping[object_type][attribute.object_relation], attribute_value) except KeyError: pass else: for attribute in obj.attributes: try: - self.xml[xml_part] += "<{0}>{1}".format(goAMLmapping[attribute.object_relation], attribute.value) + self.xml[xml_part] += "<{0}>{1}".format(goAMLmapping[object_type][attribute.object_relation], attribute.value) except KeyError: pass From 694a63c8f3293f63bf0e241e6ad362f04e083ece Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Thu, 22 Feb 2018 10:29:05 +0100 Subject: [PATCH 48/66] add: Added an example file generated by GoAML export module --- tests/goamlexport.xml | 1 + 1 file changed, 1 insertion(+) create mode 100644 tests/goamlexport.xml diff --git a/tests/goamlexport.xml b/tests/goamlexport.xml new file mode 100644 index 0000000..ae3ea80 --- /dev/null +++ b/tests/goamlexport.xml @@ -0,0 +1 @@ +2510ESTR2018-02-22T08:34:16+00:00EURTW000009011 Manners Street WellingtonBG2015-12-01T10:03:0012345when it transactsEAAEUR31032027088ATTBVIThe bankNickPittSir1993-09-25Mulhouse, FranceMale
ParisFrance
FRAKMichelJeanHimselfPrefer not to say
LuxembourgLuxembourg
LUX From b2b0fccd47c42f54dff290d5b5a000cb0e134b93 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Thu, 22 Feb 2018 16:37:27 +0100 Subject: [PATCH 49/66] fix: Added an object checking - Checking if there are objects in the event, and then if there is at least 1 transaction object - This prevents the module from crashing, but does not guaranty having a valid GoAML file (depending on objects and their relations) --- misp_modules/modules/export_mod/goamlexport.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/misp_modules/modules/export_mod/goamlexport.py b/misp_modules/modules/export_mod/goamlexport.py index e732584..76bbdc8 100644 --- a/misp_modules/modules/export_mod/goamlexport.py +++ b/misp_modules/modules/export_mod/goamlexport.py @@ -194,6 +194,15 @@ def handler(q=False): config = request['config'].get('rentity_id') export_doc = GoAmlGeneration(config) export_doc.from_event(request['data'][0]) + if not export_doc.misp_event.Object: + misperrors['error'] = "There is no object in this event." + return misperrors + types = [] + for obj in export_doc.misp_event.Object: + types.append(obj.name) + if 'transaction' not in types: + misperrors['error'] = "There is no transaction object in this event." + return misperrors export_doc.parse_objects() export_doc.build_xml() exp_doc = "{}{}".format(export_doc.xml.get('header'), export_doc.xml.get('data')) From 983b7da7b71ed3e5d9f139167400ecf5e8c49e6c Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Thu, 22 Feb 2018 16:55:52 +0100 Subject: [PATCH 50/66] fix: Added an object checking - Checking if there are objects in the event, and then if there is at least 1 transaction object - This prevents the module from crashing, but does not guaranty having a valid GoAML file (depending on objects and their relations) --- misp_modules/modules/export_mod/goamlexport.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/misp_modules/modules/export_mod/goamlexport.py b/misp_modules/modules/export_mod/goamlexport.py index e732584..76bbdc8 100644 --- a/misp_modules/modules/export_mod/goamlexport.py +++ b/misp_modules/modules/export_mod/goamlexport.py @@ -194,6 +194,15 @@ def handler(q=False): config = request['config'].get('rentity_id') export_doc = GoAmlGeneration(config) export_doc.from_event(request['data'][0]) + if not export_doc.misp_event.Object: + misperrors['error'] = "There is no object in this event." + return misperrors + types = [] + for obj in export_doc.misp_event.Object: + types.append(obj.name) + if 'transaction' not in types: + misperrors['error'] = "There is no transaction object in this event." + return misperrors export_doc.parse_objects() export_doc.build_xml() exp_doc = "{}{}".format(export_doc.xml.get('header'), export_doc.xml.get('data')) From 359ac9100ebb9ed41c853d228cc84e2d4cb2441c Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Fri, 23 Feb 2018 15:58:04 +0100 Subject: [PATCH 51/66] fix: typo in references mapping dictionary --- misp_modules/modules/export_mod/goamlexport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/export_mod/goamlexport.py b/misp_modules/modules/export_mod/goamlexport.py index 76bbdc8..f6d3ff5 100644 --- a/misp_modules/modules/export_mod/goamlexport.py +++ b/misp_modules/modules/export_mod/goamlexport.py @@ -44,7 +44,7 @@ goAMLmapping = {'bank-account': {'bank-account': 't_account', 'institution-name' referencesMapping = {'bank-account': {'aml_type': '{}_account', 'bracket': 't_{}'}, 'person': {'transaction': {'aml_type': '{}_person', 'bracket': 't_{}'}, 'bank-account': {'aml_type': 't_person', 'bracket': 'signatory'}}, - 'legal-entity': {'transaction': {'aml_type': '{}_entity', 'bracket': 't_{}'}, 'bank-account': {'aml_type': 'entity'}}, + 'legal-entity': {'transaction': {'aml_type': '{}_entity', 'bracket': 't_{}'}, 'bank-account': {'aml_type': 't_entity'}}, 'geolocation': {'aml_type': 'address', 'bracket': 'addresses'}} class GoAmlGeneration(object): From 81a6be17d3f567ced1517b216d8c847cbd1e0267 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Mon, 26 Feb 2018 11:47:35 +0100 Subject: [PATCH 52/66] chg: Structurded data --- misp_modules/modules/import_mod/goamlimport.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/misp_modules/modules/import_mod/goamlimport.py b/misp_modules/modules/import_mod/goamlimport.py index dbef826..a2cda32 100644 --- a/misp_modules/modules/import_mod/goamlimport.py +++ b/misp_modules/modules/import_mod/goamlimport.py @@ -24,7 +24,7 @@ t_account = {'nodes': ['signatory'], entity = {'nodes': ['addresses'], 'leaves': ['name', 'commercial_name', 'incorporation_legal_form', 'incorporation_number', 'business', 'phone']} -goAMLobjects = {'report': {'nodes': ['reporting_person', 'location', 'transaction'], +goAMLobjects = {'report': {'nodes': ['reporting_person', 'location'], 'leaves': ['rentity_id', 'submission_code', 'report_code', 'submission_date', 'currency_code_local']}, 'reporting_person': {'nodes': ['addresses'], @@ -53,25 +53,27 @@ goAMLobjects = {'report': {'nodes': ['reporting_person', 'location', 'transactio class GoAmlParser(): def __init__(self): - self.dict = defaultdict(list) + self.dict = {} def readFile(self, filename): self.tree = ET.parse(filename).getroot() def parse_xml(self): - self.itterate(self.tree, 'report') + self.dict = self.itterate(self.tree, 'report') + self.dict['transaction'] = [] + for t in self.tree.findall('transaction'): + self.dict['transaction'].append(self.itterate(t, 'transaction')) def itterate(self, tree, aml_type): elementDict = {} for element in tree: tag = element.tag - print(tag) mapping = goAMLobjects.get(aml_type) if tag in mapping.get('nodes'): - self.itterate(element, tag) + elementDict[tag] = self.itterate(element, tag) elif tag in mapping.get('leaves'): elementDict[tag] = element.text - self.dict[aml_type].append(elementDict) + return elementDict def handler(q=False): if q is False: From 5df2d309a0e514b848746e9cb7ed5f1c9f11a9c8 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Mon, 26 Feb 2018 15:58:53 +0100 Subject: [PATCH 53/66] typo --- misp_modules/modules/export_mod/goamlexport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/export_mod/goamlexport.py b/misp_modules/modules/export_mod/goamlexport.py index f6d3ff5..f32aef2 100644 --- a/misp_modules/modules/export_mod/goamlexport.py +++ b/misp_modules/modules/export_mod/goamlexport.py @@ -37,7 +37,7 @@ goAMLmapping = {'bank-account': {'bank-account': 't_account', 'institution-name' 'transmode-comment': 'transmode_comment', 'date-posting': 'date_posting', 'teller': 'teller', 'authorized': 'authorized', 'text': 'transaction_description'}, - 'legal-enitty': {'legal-entity': 'entity', 'name': 'name', 'business': 'business', + 'legal-entity': {'legal-entity': 'entity', 'name': 'name', 'business': 'business', 'commercial-name': 'commercial_name', 'phone-number': 'phone', 'legal-form': 'incorporation_legal_form', 'registration-number': 'incorporation_number'}} From 478cd53912238c820928ebb615efb62dd987736e Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Mon, 26 Feb 2018 18:13:43 +0100 Subject: [PATCH 54/66] add: Added dictionary to map aml types into MISP types --- .../modules/import_mod/goamlimport.py | 71 ++++++++++++------- 1 file changed, 47 insertions(+), 24 deletions(-) diff --git a/misp_modules/modules/import_mod/goamlimport.py b/misp_modules/modules/import_mod/goamlimport.py index a2cda32..ea6a3cb 100644 --- a/misp_modules/modules/import_mod/goamlimport.py +++ b/misp_modules/modules/import_mod/goamlimport.py @@ -1,6 +1,7 @@ import json, datetime import xml.etree.ElementTree as ET from collections import defaultdict +from pymisp import MISPEvent misperrors = {'error': 'Error'} moduleinfo = {'version': 1, 'author': 'Christian Studer', @@ -9,51 +10,72 @@ moduleinfo = {'version': 1, 'author': 'Christian Studer', moduleconfig = [] mispattributes = {'input': ['xml file'], 'output': ['MISPEvent']} -t_from = {'nodes': ['from_person', 'from_account', 'from_entity'], +t_from_objects = {'nodes': ['from_person', 'from_account', 'from_entity'], 'leaves': ['from_funds_code', 'from_country']} -t_to = {'nodes': ['to_person', 'to_account', 'to_entity'], +t_to_objects = {'nodes': ['to_person', 'to_account', 'to_entity'], 'leaves': ['to_funds_code', 'to_country']} -t_person = {'nodes': ['addresses'], +t_person_objects = {'nodes': ['addresses'], 'leaves': ['first_name', 'middle_name', 'last_name', 'gender', 'title', 'mothers_name', 'birthdate', 'passport_number', 'passport_country', 'id_number', 'birth_place', 'alias', 'nationality1']} -t_account = {'nodes': ['signatory'], +t_account_objects = {'nodes': ['signatory'], 'leaves': ['institution_name', 'institution_code', 'swift', 'branch', 'non_banking_insitution', 'account', 'currency_code', 'account_name', 'iban', 'client_number', 'opened', 'closed', 'personal_account_type', 'balance', 'date_balance', 'status_code', 'beneficiary', 'beneficiary_comment', 'comments']} -entity = {'nodes': ['addresses'], +entity_objects = {'nodes': ['addresses'], 'leaves': ['name', 'commercial_name', 'incorporation_legal_form', 'incorporation_number', 'business', 'phone']} goAMLobjects = {'report': {'nodes': ['reporting_person', 'location'], - 'leaves': ['rentity_id', 'submission_code', 'report_code', 'submission_date', - 'currency_code_local']}, - 'reporting_person': {'nodes': ['addresses'], - 'leaves': ['first_name', 'middle_name', 'last_name', 'title']}, - 'location': {'nodes': [], - 'leaves': ['address_type', 'address', 'city', 'zip', 'country_code', 'state']}, + 'leaves': ['rentity_id', 'submission_code', 'report_code', 'submission_date', 'currency_code_local']}, + 'reporting_person': {'nodes': ['addresses'], 'leaves': ['first_name', 'middle_name', 'last_name', 'title']}, + 'location': {'nodes': [], 'leaves': ['address_type', 'address', 'city', 'zip', 'country_code', 'state']}, 'transaction': {'nodes': ['t_from', 't_from_my_client', 't_to', 't_to_my_client'], 'leaves': ['transactionnumber', 'transaction_location', 'date_transaction', 'transmode_code', 'amount_local']}, - 't_from': t_from, - 't_from_my_client': t_from, - 't_to': t_to, - 't_to_my_client': t_to, + 't_from': t_from_objects, 't_from_my_client': t_from_objects, + 't_to': t_to_objects, 't_to_my_client': t_to_objects, 'addresses': {'nodes': ['address'], 'leaves': []}, - 'address': {'nodes': [], - 'leaves': ['address_type', 'address', 'city', 'zip', 'country_code', 'state']}, - 'from_person': t_person, - 'to_person': t_person, - 't_person': t_person, - 'from_account': t_account, - 'to_account': t_account, + 'address': {'nodes': [], 'leaves': ['address_type', 'address', 'city', 'zip', 'country_code', 'state']}, + 'from_person': t_person_objects, 'to_person': t_person_objects, 't_person': t_person_objects, + 'from_account': t_account_objects, 'to_account': t_account_objects, 'signatory': {'nodes': ['t_person'], 'leaves': []}, - 'from_entity': entity, - 'to_entity': entity, + 'from_entity': entity_objects, 'to_entity': entity_objects, } +t_account_mapping = {'t_account': 'bank-account', 'institution_name': 'institution-name', 'institution_code': 'institution-code', + 'iban': 'iban', 'swift': 'swift', 'branch': 'branch', 'non_banking_institution': 'non-bank-institution', + 'account': 'account', 'currency_code': 'currency-code', 'account_name': 'account-name', + 'client_number': 'client-number', 'personal_account_type': 'personal-account-type', 'opened': 'opened', + 'closed': 'closed', 'balance': 'balance', 'status_code': 'status-code', 'beneficiary': 'beneficiary', + 'beneficiary_comment': 'beneficiary-comment', 'comments': 'comments'} + +t_person_mapping = {'t_person': 'person', 'comments': 'text', 'first_name': 'first-name', 'middle_name': 'middle-name', + 'last_name': 'last-name', 'title': 'title', 'mothers_name': 'mothers-name', 'alias': 'alias', + 'birthdate': 'date-of-birth', 'birth_place': 'place-of-birth', 'gender': 'gender','nationality1': 'nationality', + 'passport_number': 'passport-number', 'passport_country': 'passport-country', 'ssn': 'social-security-number', + 'id_number': 'identity-card-number'} + +location_mapping = {'location': 'geolocation', 'city': 'city', 'state': 'region', 'country-code': 'country', 'address': 'address', + 'zip': 'zipcode'} + +t_entity_mapping = {'entity': 'legal-entity', 'name': 'name', 'business': 'business', 'commercial_name': 'commercial-name', + 'phone': 'phone-number', 'incorporation_legal_form': 'legal-form', 'incorporation_number': 'registration-number'} + +goAMLmapping = {'from_account': t_account_mapping, 'to_account': t_account_mapping, + 'from_person': t_person_mapping, 'to_person': t_person_mapping, 'reporting_person': t_person_mapping, + 'from_entity': t_entity_mapping, 'to_entity': t_entity_mapping, + 'location': location_mapping, 'address': location_mapping, + 'transaction': {'transaction': 'transaction', 'transactionnumber': 'transaction-number', 'date_transaction': 'date', + 'transaction_location': 'location', 'transmode_code': 'transmode-code', 'amount_local': 'amount', + 'transmode_comment': 'transmode-comment', 'date_posting': 'date-posting', 'teller': 'teller', + 'authorized': 'authorized', 'transaction_description': 'text'}} + +nodes_to_ignore = ['addresses', 'signatory'] + class GoAmlParser(): def __init__(self): self.dict = {} + self.misp_event = MISPEvent() def readFile(self, filename): self.tree = ET.parse(filename).getroot() @@ -63,6 +85,7 @@ class GoAmlParser(): self.dict['transaction'] = [] for t in self.tree.findall('transaction'): self.dict['transaction'].append(self.itterate(t, 'transaction')) + self.misp_event.timestamp = self.dict.get('submission_date') def itterate(self, tree, aml_type): elementDict = {} From a02dbd6a8dc595f734fb289e651dea8ee8465e40 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Mon, 26 Feb 2018 18:44:44 +0100 Subject: [PATCH 55/66] fix: Fixed typo of the aml type for country codes --- misp_modules/modules/export_mod/goamlexport.py | 2 +- tests/goamlexport.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/export_mod/goamlexport.py b/misp_modules/modules/export_mod/goamlexport.py index f32aef2..c277640 100644 --- a/misp_modules/modules/export_mod/goamlexport.py +++ b/misp_modules/modules/export_mod/goamlexport.py @@ -30,7 +30,7 @@ goAMLmapping = {'bank-account': {'bank-account': 't_account', 'institution-name' 'passport-number': 'passport_number', 'passport-country': 'passport_country', 'social-security-number': 'ssn', 'identity-card-number': 'id_number'}, 'geolocation': {'geolocation': 'location', 'city': 'city', 'region': 'state', - 'country': 'country-code', 'address': 'address', 'zipcode': 'zip'}, + 'country': 'country_code', 'address': 'address', 'zipcode': 'zip'}, 'transaction': {'transaction': 'transaction', 'transaction-number': 'transactionnumber', 'date': 'date_transaction', 'location': 'transaction_location', 'transmode-code': 'transmode_code', 'amount': 'amount_local', diff --git a/tests/goamlexport.xml b/tests/goamlexport.xml index ae3ea80..4a001b9 100644 --- a/tests/goamlexport.xml +++ b/tests/goamlexport.xml @@ -1 +1 @@ -2510ESTR2018-02-22T08:34:16+00:00EURTW000009011 Manners Street WellingtonBG2015-12-01T10:03:0012345when it transactsEAAEUR31032027088ATTBVIThe bankNickPittSir1993-09-25Mulhouse, FranceMale
ParisFrance
FRAKMichelJeanHimselfPrefer not to say
LuxembourgLuxembourg
LUX +2510ESTR2018-02-22T08:34:16+00:00EURTW000009011 Manners Street WellingtonBG2015-12-01T10:03:0012345when it transactsEAAEUR31032027088ATTBVIThe bankNickPittSir1993-09-25Mulhouse, FranceMale
ParisFrance
FRAKMichelJeanHimselfPrefer not to say
LuxembourgLuxembourg
LUX From cad62464c5f19aefba17605315cad9d196cbc874 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Tue, 27 Feb 2018 11:08:37 +0100 Subject: [PATCH 56/66] Now parsing all the transaction attributes --- .../modules/import_mod/goamlimport.py | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/misp_modules/modules/import_mod/goamlimport.py b/misp_modules/modules/import_mod/goamlimport.py index ea6a3cb..4cda375 100644 --- a/misp_modules/modules/import_mod/goamlimport.py +++ b/misp_modules/modules/import_mod/goamlimport.py @@ -88,15 +88,29 @@ class GoAmlParser(): self.misp_event.timestamp = self.dict.get('submission_date') def itterate(self, tree, aml_type): - elementDict = {} + element_dict = {} for element in tree: tag = element.tag mapping = goAMLobjects.get(aml_type) if tag in mapping.get('nodes'): - elementDict[tag] = self.itterate(element, tag) + if aml_type == 'transaction': + self.fill_transaction(element, element_dict, tag) + element_dict[tag] = self.itterate(element, tag) elif tag in mapping.get('leaves'): - elementDict[tag] = element.text - return elementDict + try: + element_dict[goAMLmapping[aml_type][tag]] = element.text + except KeyError: + pass + return element_dict + + @staticmethod + def fill_transaction(element, element_dict, tag): + if 't_from' in tag: + element_dict['from-funds-code'] = element.find('from_funds_code').text + element_dict['from-country'] = element.find('from_country').text + if 't_to' in tag: + element_dict['to-funds-code'] = element.find('to_funds_code').text + element_dict['to-country'] = element.find('to_country').text def handler(q=False): if q is False: From 8f5c08e2c6c52e5f06f6cbc1ff037fe86d354856 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Wed, 28 Feb 2018 15:07:55 +0100 Subject: [PATCH 57/66] Converting GoAML into MISPEvent --- .../modules/import_mod/goamlimport.py | 95 +++++++++++-------- 1 file changed, 58 insertions(+), 37 deletions(-) diff --git a/misp_modules/modules/import_mod/goamlimport.py b/misp_modules/modules/import_mod/goamlimport.py index 4cda375..a9174db 100644 --- a/misp_modules/modules/import_mod/goamlimport.py +++ b/misp_modules/modules/import_mod/goamlimport.py @@ -1,7 +1,7 @@ import json, datetime import xml.etree.ElementTree as ET from collections import defaultdict -from pymisp import MISPEvent +from pymisp import MISPEvent, MISPObject misperrors = {'error': 'Error'} moduleinfo = {'version': 1, 'author': 'Christian Studer', @@ -18,12 +18,12 @@ t_person_objects = {'nodes': ['addresses'], 'leaves': ['first_name', 'middle_name', 'last_name', 'gender', 'title', 'mothers_name', 'birthdate', 'passport_number', 'passport_country', 'id_number', 'birth_place', 'alias', 'nationality1']} t_account_objects = {'nodes': ['signatory'], - 'leaves': ['institution_name', 'institution_code', 'swift', 'branch', 'non_banking_insitution', - 'account', 'currency_code', 'account_name', 'iban', 'client_number', 'opened', 'closed', - 'personal_account_type', 'balance', 'date_balance', 'status_code', 'beneficiary', - 'beneficiary_comment', 'comments']} + 'leaves': ['institution_name', 'institution_code', 'swift', 'branch', 'non_banking_insitution', + 'account', 'currency_code', 'account_name', 'iban', 'client_number', 'opened', 'closed', + 'personal_account_type', 'balance', 'date_balance', 'status_code', 'beneficiary', + 'beneficiary_comment', 'comments']} entity_objects = {'nodes': ['addresses'], - 'leaves': ['name', 'commercial_name', 'incorporation_legal_form', 'incorporation_number', 'business', 'phone']} + 'leaves': ['name', 'commercial_name', 'incorporation_legal_form', 'incorporation_number', 'business', 'phone']} goAMLobjects = {'report': {'nodes': ['reporting_person', 'location'], 'leaves': ['rentity_id', 'submission_code', 'report_code', 'submission_date', 'currency_code_local']}, @@ -42,30 +42,30 @@ goAMLobjects = {'report': {'nodes': ['reporting_person', 'location'], 'from_entity': entity_objects, 'to_entity': entity_objects, } -t_account_mapping = {'t_account': 'bank-account', 'institution_name': 'institution-name', 'institution_code': 'institution-code', +t_account_mapping = {'misp_name': 'bank-account', 'institution_name': 'institution-name', 'institution_code': 'institution-code', 'iban': 'iban', 'swift': 'swift', 'branch': 'branch', 'non_banking_institution': 'non-bank-institution', 'account': 'account', 'currency_code': 'currency-code', 'account_name': 'account-name', 'client_number': 'client-number', 'personal_account_type': 'personal-account-type', 'opened': 'opened', 'closed': 'closed', 'balance': 'balance', 'status_code': 'status-code', 'beneficiary': 'beneficiary', 'beneficiary_comment': 'beneficiary-comment', 'comments': 'comments'} -t_person_mapping = {'t_person': 'person', 'comments': 'text', 'first_name': 'first-name', 'middle_name': 'middle-name', +t_person_mapping = {'misp_name': 'person', 'comments': 'text', 'first_name': 'first-name', 'middle_name': 'middle-name', 'last_name': 'last-name', 'title': 'title', 'mothers_name': 'mothers-name', 'alias': 'alias', 'birthdate': 'date-of-birth', 'birth_place': 'place-of-birth', 'gender': 'gender','nationality1': 'nationality', 'passport_number': 'passport-number', 'passport_country': 'passport-country', 'ssn': 'social-security-number', 'id_number': 'identity-card-number'} -location_mapping = {'location': 'geolocation', 'city': 'city', 'state': 'region', 'country-code': 'country', 'address': 'address', - 'zip': 'zipcode'} +location_mapping = {'misp_name': 'geolocation', 'city': 'city', 'state': 'region', 'country_code': 'country', 'address': 'address', + 'zip': 'zipcode'} -t_entity_mapping = {'entity': 'legal-entity', 'name': 'name', 'business': 'business', 'commercial_name': 'commercial-name', +t_entity_mapping = {'misp_name': 'legal-entity', 'name': 'name', 'business': 'business', 'commercial_name': 'commercial-name', 'phone': 'phone-number', 'incorporation_legal_form': 'legal-form', 'incorporation_number': 'registration-number'} -goAMLmapping = {'from_account': t_account_mapping, 'to_account': t_account_mapping, +goAMLmapping = {'from_account': t_account_mapping, 'to_account': t_account_mapping, 't_person': t_person_mapping, 'from_person': t_person_mapping, 'to_person': t_person_mapping, 'reporting_person': t_person_mapping, 'from_entity': t_entity_mapping, 'to_entity': t_entity_mapping, 'location': location_mapping, 'address': location_mapping, - 'transaction': {'transaction': 'transaction', 'transactionnumber': 'transaction-number', 'date_transaction': 'date', + 'transaction': {'misp_name': 'transaction', 'transactionnumber': 'transaction-number', 'date_transaction': 'date', 'transaction_location': 'location', 'transmode_code': 'transmode-code', 'amount_local': 'amount', 'transmode_comment': 'transmode-comment', 'date_posting': 'date-posting', 'teller': 'teller', 'authorized': 'authorized', 'transaction_description': 'text'}} @@ -74,43 +74,64 @@ nodes_to_ignore = ['addresses', 'signatory'] class GoAmlParser(): def __init__(self): - self.dict = {} self.misp_event = MISPEvent() def readFile(self, filename): self.tree = ET.parse(filename).getroot() def parse_xml(self): - self.dict = self.itterate(self.tree, 'report') - self.dict['transaction'] = [] + self.first_itteration() for t in self.tree.findall('transaction'): - self.dict['transaction'].append(self.itterate(t, 'transaction')) - self.misp_event.timestamp = self.dict.get('submission_date') + self.itterate(t, 'transaction') + + def first_itteration(self): + self.misp_event.timestamp = self.tree.find('submission_date').text + for node in goAMLobjects['report']['nodes']: + element = self.tree.find(node) + if element is not None: + self.itterate(element, element.tag) def itterate(self, tree, aml_type): - element_dict = {} - for element in tree: - tag = element.tag - mapping = goAMLobjects.get(aml_type) - if tag in mapping.get('nodes'): + objects = goAMLobjects[aml_type] + if aml_type not in nodes_to_ignore: + try: + mapping = goAMLmapping[aml_type] + misp_object = MISPObject(name=mapping['misp_name']) + for leaf in objects['leaves']: + element = tree.find(leaf) + if element is not None: + object_relation = mapping[element.tag] + attribute = {'object_relation': object_relation, 'value': element.text} + misp_object.add_attribute(**attribute) if aml_type == 'transaction': - self.fill_transaction(element, element_dict, tag) - element_dict[tag] = self.itterate(element, tag) - elif tag in mapping.get('leaves'): - try: - element_dict[goAMLmapping[aml_type][tag]] = element.text - except KeyError: - pass - return element_dict + for node in objects['nodes']: + element = tree.find(node) + if element is not None: + self.fill_transaction(element, element.tag, misp_object) + self.misp_event.add_object(misp_object) + except KeyError: + pass + for node in objects['nodes']: + element = tree.find(node) + if element is not None: + self.itterate(element, element.tag) @staticmethod - def fill_transaction(element, element_dict, tag): + def fill_transaction(element, tag, misp_object): if 't_from' in tag: - element_dict['from-funds-code'] = element.find('from_funds_code').text - element_dict['from-country'] = element.find('from_country').text + from_funds = element.find('from_funds_code').text + from_funds_attribute = {'object_relation': 'from-funds-code', 'value': from_funds} + misp_object.add_attribute(**from_funds_attribute) + from_country = element.find('from_country').text + from_country_attribute = {'object_relation': 'from-country', 'value': from_country} + misp_object.add_attribute(**from_country_attribute) if 't_to' in tag: - element_dict['to-funds-code'] = element.find('to_funds_code').text - element_dict['to-country'] = element.find('to_country').text + to_funds = element.find('to_funds_code').text + to_funds_attribute = {'object_relation': 'to-funds-code', 'value': to_funds} + misp_object.add_attribute(**to_funds_attribute) + to_country = element.find('to_country').text + to_country_attribute = {'object_relation': 'to-country', 'value': to_country} + misp_object.add_attribute(**to_country_attribute) def handler(q=False): if q is False: @@ -128,7 +149,7 @@ def handler(q=False): misperrors['error'] = "Impossible to read the file" return misperrors aml_parser.parse_xml() - return aml_parser.dict + return aml_parser.misp_event.to_json() def introspection(): return mispattributes From 323f71cdd3f1a253a520af07b6f7a56a262be693 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Wed, 28 Feb 2018 17:41:45 +0100 Subject: [PATCH 58/66] Fixed some details about the module output --- misp_modules/modules/import_mod/goamlimport.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/misp_modules/modules/import_mod/goamlimport.py b/misp_modules/modules/import_mod/goamlimport.py index a9174db..10bdaab 100644 --- a/misp_modules/modules/import_mod/goamlimport.py +++ b/misp_modules/modules/import_mod/goamlimport.py @@ -1,4 +1,4 @@ -import json, datetime +import json, datetime, time import xml.etree.ElementTree as ET from collections import defaultdict from pymisp import MISPEvent, MISPObject @@ -85,7 +85,8 @@ class GoAmlParser(): self.itterate(t, 'transaction') def first_itteration(self): - self.misp_event.timestamp = self.tree.find('submission_date').text + submission_date = self.tree.find('submission_date').text.split('+')[0] + self.misp_event.timestamp = int(time.mktime(time.strptime(submission_date, "%Y-%m-%dT%H:%M:%S"))) for node in goAMLobjects['report']['nodes']: element = self.tree.find(node) if element is not None: @@ -149,7 +150,8 @@ def handler(q=False): misperrors['error'] = "Impossible to read the file" return misperrors aml_parser.parse_xml() - return aml_parser.misp_event.to_json() + r = {'results': [{'types': mispattributes['output'], 'values': aml_parser.misp_event.to_json()}]} + return r def introspection(): return mispattributes From 03d20856d9c71b2778e4b5d13b29099c0b0a0f56 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Wed, 28 Feb 2018 22:46:39 +0100 Subject: [PATCH 59/66] add: added goamlimport --- misp_modules/modules/import_mod/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/import_mod/__init__.py b/misp_modules/modules/import_mod/__init__.py index 8acccbd..886eaf7 100644 --- a/misp_modules/modules/import_mod/__init__.py +++ b/misp_modules/modules/import_mod/__init__.py @@ -1,4 +1,4 @@ from . import _vmray -__all__ = ['vmray_import', 'testimport', 'ocr', 'stiximport', 'cuckooimport', +__all__ = ['vmray_import', 'testimport', 'ocr', 'stiximport', 'cuckooimport', 'goamlimport', 'email_import', 'mispjson', 'openiocimport', 'threatanalyzer_import', 'csvimport'] From e6c55f5ddec38bf56df2a9ff6ba65b8ff213170e Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Fri, 2 Mar 2018 09:03:51 +0100 Subject: [PATCH 60/66] fix: Fixed input & output of the module Also updated some functions --- misp_modules/modules/import_mod/csvimport.py | 43 ++++++++------------ 1 file changed, 18 insertions(+), 25 deletions(-) diff --git a/misp_modules/modules/import_mod/csvimport.py b/misp_modules/modules/import_mod/csvimport.py index dc67eec..3773530 100644 --- a/misp_modules/modules/import_mod/csvimport.py +++ b/misp_modules/modules/import_mod/csvimport.py @@ -1,9 +1,9 @@ # -*- coding: utf-8 -*- -import json, os +import json, os, base64 import pymisp misperrors = {'error': 'Error'} -mispattributes = {'input': ['file'], 'output': ['MISP attributes']} +mispattributes = {'inputSource': ['file'], 'output': ['MISP attributes']} moduleinfo = {'version': '0.1', 'author': 'Christian Studer', 'description': 'Import Attributes from a csv file.', 'module-type': ['import']} @@ -16,39 +16,32 @@ def handler(q=False): if q is False: return False request = json.loads(q) - if request.get('file'): - filename = request['file'] + if request.get('data'): + data = base64.b64decode(request['data']).decode('utf-8') else: misperrors['error'] = "Unsupported attributes type" return misperrors if not request.get('config') and not request['config'].get('header'): misperrors['error'] = "Configuration error" return misperrors - config = request['config'].get('header') - #header = [] - try: - data = readFile(filename, 'utf-8') - except: - data = readFile(filename, 'iso-8859-1') + config = request['config'].get('header').split(',') + config = [c.strip() for c in config] + data = parse_data(data.split('\n')) # find which delimiter is used delimiter, length = findDelimiter(config, data) # build the attributes result = buildAttributes(config, data, delimiter, length) - r = {'results': [{'types': mispattributes['output'], 'values': result}]} + r = {'results': result} return r -def readFile(filename, encoding): - data = [] - with open(filename, 'r', encoding=encoding) as f: - for line in f: - # split comments from data - if '#' in line: - l = line.split('#')[0].strip() - else: - l = line.strip() - if l: - data.append(l) - return data +def parse_data(data): + return_data = [] + for line in data: + l = line.split('#')[0].strip() if '#' in line else line.strip() + if l: + return_data.append(l) + print(len(return_data)) + return return_data def findDelimiter(header, data): n = len(header) @@ -74,7 +67,7 @@ def buildAttributes(header, dataValues, delimiter, length): for data in dataValues: d = data.strip() if d: - attributes.append({'type': mispType, 'value': d}) + attributes.append({'types': mispType, 'values': d}) else: # split fields that should be recognized as misp attribute types from the others list2pop, misp, head = findMispTypes(header) @@ -90,7 +83,7 @@ def buildAttributes(header, dataValues, delimiter, length): datamisp.append(datasplit.pop(l).strip()) # for each misp type, we create an attribute for m, dm in zip(misp, datamisp): - attribute = {'type': m, 'value': dm} + attribute = {'types': m, 'values': dm} for h, ds in zip(head, datasplit): if h: attribute[h] = ds.strip() From c9ef57826219878acab6fa69a2e643086c35b818 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Fri, 2 Mar 2018 09:09:12 +0100 Subject: [PATCH 61/66] Removed print --- misp_modules/modules/import_mod/csvimport.py | 1 - 1 file changed, 1 deletion(-) diff --git a/misp_modules/modules/import_mod/csvimport.py b/misp_modules/modules/import_mod/csvimport.py index 3773530..5cfbc67 100644 --- a/misp_modules/modules/import_mod/csvimport.py +++ b/misp_modules/modules/import_mod/csvimport.py @@ -40,7 +40,6 @@ def parse_data(data): l = line.split('#')[0].strip() if '#' in line else line.strip() if l: return_data.append(l) - print(len(return_data)) return return_data def findDelimiter(header, data): From 82fe8ba78ca5d46643f3ef2c29ed51d82084cfd1 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Fri, 2 Mar 2018 11:03:21 +0100 Subject: [PATCH 62/66] fix: Fixed input & output of the module --- misp_modules/modules/import_mod/goamlimport.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/misp_modules/modules/import_mod/goamlimport.py b/misp_modules/modules/import_mod/goamlimport.py index 10bdaab..9b2a34c 100644 --- a/misp_modules/modules/import_mod/goamlimport.py +++ b/misp_modules/modules/import_mod/goamlimport.py @@ -1,4 +1,4 @@ -import json, datetime, time +import json, datetime, time, base64 import xml.etree.ElementTree as ET from collections import defaultdict from pymisp import MISPEvent, MISPObject @@ -8,7 +8,7 @@ moduleinfo = {'version': 1, 'author': 'Christian Studer', 'description': 'Import from GoAML', 'module-type': ['import']} moduleconfig = [] -mispattributes = {'input': ['xml file'], 'output': ['MISPEvent']} +mispattributes = {'inputSource': ['file'], 'output': ['MISP objects']} t_from_objects = {'nodes': ['from_person', 'from_account', 'from_entity'], 'leaves': ['from_funds_code', 'from_country']} @@ -76,8 +76,8 @@ class GoAmlParser(): def __init__(self): self.misp_event = MISPEvent() - def readFile(self, filename): - self.tree = ET.parse(filename).getroot() + def read_xml(self, data): + self.tree = ET.fromstring(data) def parse_xml(self): self.first_itteration() @@ -138,19 +138,19 @@ def handler(q=False): if q is False: return False request = json.loads(q) - if request.get('file'): - filename = request['file'] + if request.get('data'): + data = base64.b64decode(request['data']).decode('utf-8') else: misperrors['error'] = "Unsupported attributes type" return misperrors aml_parser = GoAmlParser() try: - aml_parser.readFile(filename) + aml_parser.read_xml(data) except: - misperrors['error'] = "Impossible to read the file" + misperrors['error'] = "Impossible to read XML data" return misperrors aml_parser.parse_xml() - r = {'results': [{'types': mispattributes['output'], 'values': aml_parser.misp_event.to_json()}]} + r = {'results': [obj.to_json() for obj in aml_parser.misp_event.objects]} return r def introspection(): From 4d7642ac91e6f8694ee83b050368e8a6bfff944e Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Mon, 5 Mar 2018 14:58:31 +0100 Subject: [PATCH 63/66] add: Added Object References in the objects imported --- misp_modules/modules/import_mod/goamlimport.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/import_mod/goamlimport.py b/misp_modules/modules/import_mod/goamlimport.py index 9b2a34c..ecb0a2d 100644 --- a/misp_modules/modules/import_mod/goamlimport.py +++ b/misp_modules/modules/import_mod/goamlimport.py @@ -71,6 +71,7 @@ goAMLmapping = {'from_account': t_account_mapping, 'to_account': t_account_mappi 'authorized': 'authorized', 'transaction_description': 'text'}} nodes_to_ignore = ['addresses', 'signatory'] +relationship_to_keep = ['signatory', 't_from', 't_from_my_client', 't_to', 't_to_my_client', 'address'] class GoAmlParser(): def __init__(self): @@ -92,8 +93,10 @@ class GoAmlParser(): if element is not None: self.itterate(element, element.tag) - def itterate(self, tree, aml_type): + def itterate(self, tree, aml_type, referencing_uuid=None, relationship_type=None): objects = goAMLobjects[aml_type] + referenced_uuid = referencing_uuid + rel = relationship_type if aml_type not in nodes_to_ignore: try: mapping = goAMLmapping[aml_type] @@ -110,12 +113,20 @@ class GoAmlParser(): if element is not None: self.fill_transaction(element, element.tag, misp_object) self.misp_event.add_object(misp_object) + last_object = self.misp_event.objects[-1] + referenced_uuid = last_object.uuid + if referencing_uuid and relationship_type: + referencing_object = self.misp_event.get_object_by_uuid(referencing_uuid) + referencing_object.add_reference(referenced_uuid, rel, None, **last_object) except KeyError: pass for node in objects['nodes']: element = tree.find(node) if element is not None: - self.itterate(element, element.tag) + tag = element.tag + if tag in relationship_to_keep: + rel = tag[2:] if tag.startswith('t_') else tag + self.itterate(element, element.tag, referencing_uuid=referenced_uuid, relationship_type=rel) @staticmethod def fill_transaction(element, tag, misp_object): From d8852867927399372911221756a9b38774af90d7 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Mon, 5 Mar 2018 19:59:30 +0100 Subject: [PATCH 64/66] Clarified functions arguments using a class --- misp_modules/modules/import_mod/csvimport.py | 180 ++++++++++--------- 1 file changed, 92 insertions(+), 88 deletions(-) diff --git a/misp_modules/modules/import_mod/csvimport.py b/misp_modules/modules/import_mod/csvimport.py index 5cfbc67..543d67b 100644 --- a/misp_modules/modules/import_mod/csvimport.py +++ b/misp_modules/modules/import_mod/csvimport.py @@ -10,7 +10,94 @@ moduleinfo = {'version': '0.1', 'author': 'Christian Studer', moduleconfig = ['header'] duplicatedFields = {'mispType': {'mispComment': 'comment'}, - 'attrField': {'eventComment': 'comment'}} + 'attrField': {'attrComment': 'comment'}} + +class CsvParser(): + def __init__(self, header): + self.header = header + self.attributes = [] + + def parse_data(self, data): + return_data = [] + for line in data: + l = line.split('#')[0].strip() if '#' in line else line.strip() + if l: + return_data.append(l) + self.data = return_data + # find which delimiter is used + self.delimiter, self.length = self.findDelimiter() + + def findDelimiter(self): + n = len(self.header) + if n > 1: + tmpData = [] + for da in self.data: + tmp = [] + for d in (';', '|', '/', ',', '\t', ' ',): + if da.count(d) == (n-1): + tmp.append(d) + if len(tmp) == 1 and tmp == tmpData: + return tmpData[0], n + else: + tmpData = tmp + else: + return None, 1 + + def buildAttributes(self): + # if there is only 1 field of data + if self.delimiter is None: + mispType = self.header[0] + for data in self.data: + d = data.strip() + if d: + self.attributes.append({'types': mispType, 'values': d}) + else: + # split fields that should be recognized as misp attribute types from the others + list2pop, misp, head = self.findMispTypes() + # for each line of data + for data in self.data: + datamisp = [] + datasplit = data.split(self.delimiter) + # in case there is an empty line or an error + if len(datasplit) != self.length: + continue + # pop from the line data that matches with a misp type, using the list of indexes + for l in list2pop: + datamisp.append(datasplit.pop(l).strip()) + # for each misp type, we create an attribute + for m, dm in zip(misp, datamisp): + attribute = {'types': m, 'values': dm} + for h, ds in zip(head, datasplit): + if h: + attribute[h] = ds.strip() + self.attributes.append(attribute) + + def findMispTypes(self): + descFilename = os.path.join(pymisp.__path__[0], 'data/describeTypes.json') + with open(descFilename, 'r') as f: + MispTypes = json.loads(f.read())['result'].get('types') + list2pop = [] + misp = [] + head = [] + for h in reversed(self.header): + n = self.header.index(h) + # fields that are misp attribute types + if h in MispTypes: + list2pop.append(n) + misp.append(h) + # handle confusions between misp attribute types and attribute fields + elif h in duplicatedFields['mispType']: + # fields that should be considered as misp attribute types + list2pop.append(n) + misp.append(duplicatedFields['mispType'].get(h)) + elif h in duplicatedFields['attrField']: + # fields that should be considered as attribute fields + head.append(duplicatedFields['attrField'].get(h)) + # otherwise, it is an attribute field + else: + head.append(h) + # return list of indexes of the misp types, list of the misp types, remaining fields that will be attribute fields + return list2pop, misp, list(reversed(head)) def handler(q=False): if q is False: @@ -26,96 +113,13 @@ def handler(q=False): return misperrors config = request['config'].get('header').split(',') config = [c.strip() for c in config] - data = parse_data(data.split('\n')) - # find which delimiter is used - delimiter, length = findDelimiter(config, data) + csv_parser = CsvParser(config) + csv_parser.parse_data(data.split('\n')) # build the attributes - result = buildAttributes(config, data, delimiter, length) - r = {'results': result} + csv_parser.buildAttributes() + r = {'results': csv_parser.attributes} return r -def parse_data(data): - return_data = [] - for line in data: - l = line.split('#')[0].strip() if '#' in line else line.strip() - if l: - return_data.append(l) - return return_data - -def findDelimiter(header, data): - n = len(header) - if n > 1: - tmpData = [] - for da in data: - tmp = [] - for d in (';', '|', '/', ',', '\t', ' ',): - if da.count(d) == (n-1): - tmp.append(d) - if len(tmp) == 1 and tmp == tmpData: - return tmpData[0], n - else: - tmpData = tmp - else: - return None, 1 - -def buildAttributes(header, dataValues, delimiter, length): - attributes = [] - # if there is only 1 field of data - if delimiter is None: - mispType = header[0] - for data in dataValues: - d = data.strip() - if d: - attributes.append({'types': mispType, 'values': d}) - else: - # split fields that should be recognized as misp attribute types from the others - list2pop, misp, head = findMispTypes(header) - # for each line of data - for data in dataValues: - datamisp = [] - datasplit = data.split(delimiter) - # in case there is an empty line or an error - if len(datasplit) != length: - continue - # pop from the line data that matches with a misp type, using the list of indexes - for l in list2pop: - datamisp.append(datasplit.pop(l).strip()) - # for each misp type, we create an attribute - for m, dm in zip(misp, datamisp): - attribute = {'types': m, 'values': dm} - for h, ds in zip(head, datasplit): - if h: - attribute[h] = ds.strip() - attributes.append(attribute) - return attributes - -def findMispTypes(header): - descFilename = os.path.join(pymisp.__path__[0], 'data/describeTypes.json') - with open(descFilename, 'r') as f: - MispTypes = json.loads(f.read())['result'].get('types') - list2pop = [] - misp = [] - head = [] - for h in reversed(header): - n = header.index(h) - # fields that are misp attribute types - if h in MispTypes: - list2pop.append(n) - misp.append(h) - # handle confusions between misp attribute types and attribute fields - elif h in duplicatedFields['mispType']: - # fields that should be considered as misp attribute types - list2pop.append(n) - misp.append(duplicatedFields['mispType'].get(h)) - elif h in duplicatedFields['attrField']: - # fields that should be considered as attribute fields - head.append(duplicatedFields['attrField'].get(h)) - # otherwise, it is an attribute field - else: - head.append(h) - # return list of indexes of the misp types, list of the misp types, remaining fields that will be attribute fields - return list2pop, misp, list(reversed(head)) - def introspection(): return mispattributes From 0436118747d4a199e8b42e0b19840767dc824859 Mon Sep 17 00:00:00 2001 From: "x41\\x43" Date: Tue, 6 Mar 2018 18:12:36 +0100 Subject: [PATCH 65/66] Improving regex (validating e-mail) Line 48: The previous regex ` ^[\w\.\+\-]+\@[\w]+\.[a-z]{2,3}$ ` matched only a small subset of valid e-mail address (e.g.: didn't match domain names longer than 3 chars or user@this-domain.de or user@multiple.level.dom) and needed to be with start (^) and end ($). This ` [a-zA-Z0-9!#$%&'*+\/=?^_`{|}~-]+(?:\.[a-zA-Z0-9!#$%&'*+\/=?^_`{|}~-]+)*@(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?\.)+[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])? ` is not perfect (e.g: can't match oriental chars), but imho is much more complete. Regex tested with several e-mail addresses with Python 3.6.4 and Python 2.7.14 on Linux 4.14. --- misp_modules/modules/expansion/otx.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/otx.py b/misp_modules/modules/expansion/otx.py index ad9da2f..214e7f0 100755 --- a/misp_modules/modules/expansion/otx.py +++ b/misp_modules/modules/expansion/otx.py @@ -45,7 +45,7 @@ def findAll(data, keys): return a def valid_email(email): - return bool(re.search(r"^[\w\.\+\-]+\@[\w]+\.[a-z]{2,3}$", email)) + return bool(re.search(r"[a-zA-Z0-9!#$%&'*+\/=?^_`{|}~-]+(?:\.[a-zA-Z0-9!#$%&'*+\/=?^_`{|}~-]+)*@(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?\.)+[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?", email)) def handler(q=False): if q is False: From d0f618b6480a306743e864f7f9cfb769c676c5eb Mon Sep 17 00:00:00 2001 From: Fred Morris Date: Thu, 8 Mar 2018 15:26:39 -0800 Subject: [PATCH 66/66] Add exception blocks for query errors. --- .../modules/expansion/farsight_passivedns.py | 45 +++++++++++-------- 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/misp_modules/modules/expansion/farsight_passivedns.py b/misp_modules/modules/expansion/farsight_passivedns.py index c76c752..2518771 100755 --- a/misp_modules/modules/expansion/farsight_passivedns.py +++ b/misp_modules/modules/expansion/farsight_passivedns.py @@ -1,5 +1,5 @@ import json -from ._dnsdb_query.dnsdb_query import DnsdbClient +from ._dnsdb_query.dnsdb_query import DnsdbClient, QueryError misperrors = {'error': 'Error'} @@ -41,26 +41,35 @@ def handler(q=False): def lookup_name(client, name): - res = client.query_rrset(name) # RRSET = entries in the left-hand side of the domain name related labels - for item in res: - if item.get('rrtype') in ['A', 'AAAA', 'CNAME']: - for i in item.get('rdata'): - yield(i.rstrip('.')) - if item.get('rrtype') in ['SOA']: - for i in item.get('rdata'): - # grab email field and replace first dot by @ to convert to an email address - yield(i.split(' ')[1].rstrip('.').replace('.', '@', 1)) - # res = client.query_rdata_name(name) # RDATA = entries on the right-hand side of the domain name related labels - # for item in res: - # if item.get('rrtype') in ['A', 'AAAA', 'CNAME']: - # yield(item.get('rrname').rstrip('.')) + try: + res = client.query_rrset(name) # RRSET = entries in the left-hand side of the domain name related labels + for item in res: + if item.get('rrtype') in ['A', 'AAAA', 'CNAME']: + for i in item.get('rdata'): + yield(i.rstrip('.')) + if item.get('rrtype') in ['SOA']: + for i in item.get('rdata'): + # grab email field and replace first dot by @ to convert to an email address + yield(i.split(' ')[1].rstrip('.').replace('.', '@', 1)) + except QueryError as e: + pass + + try: + res = client.query_rdata_name(name) # RDATA = entries on the right-hand side of the domain name related labels + for item in res: + if item.get('rrtype') in ['A', 'AAAA', 'CNAME']: + yield(item.get('rrname').rstrip('.')) + except QueryError as e: + pass def lookup_ip(client, ip): - res = client.query_rdata_ip(ip) - for item in res: - print(item) - yield(item['rrname'].rstrip('.')) + try: + res = client.query_rdata_ip(ip) + for item in res: + yield(item['rrname'].rstrip('.')) + except QueryError as e: + pass def introspection():