diff --git a/Makefile b/Makefile index 2d07388..b37670e 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ prepare_docs: cd documentation; cp -R ./logos/* ../docs/export_mod/logos cd documentation; cp -R ./logos/* ../docs/import_mod/logos cp ./documentation/mkdocs/*.md ./docs - cp LICENSE ../docs/license.md + cp LICENSE ./docs/license.md install_requirements: pip install -r docs/REQUIREMENTS.txt diff --git a/REQUIREMENTS b/REQUIREMENTS index 34c15f3..0fcfcb2 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -59,6 +59,7 @@ isodate==0.6.0 jbxapi==3.14.0 json-log-formatter==0.3.0 jsonschema==3.2.0 +ndjson==0.3.1 lark-parser==0.11.1 lief==0.11.0 lxml==4.6.2 diff --git a/documentation/logos/yeti.png b/documentation/logos/yeti.png new file mode 100644 index 0000000..46b77da Binary files /dev/null and b/documentation/logos/yeti.png differ diff --git a/documentation/website/expansion/yeti.json b/documentation/website/expansion/yeti.json index 3ec7789..93341dc 100644 --- a/documentation/website/expansion/yeti.json +++ b/documentation/website/expansion/yeti.json @@ -1,6 +1,6 @@ { "description": "Module to process a query on Yeti.", - "logo": "", + "logo": "yeti.png", "requirements": ["pyeti", "API key "], "input": "A domain, hostname,IP, sha256,sha1, md5, url of MISP attribute.", "output": "MISP attributes and objects fetched from the Yeti instances.", diff --git a/misp_modules/lib/__init__.py b/misp_modules/lib/__init__.py index c078cf7..d92e989 100644 --- a/misp_modules/lib/__init__.py +++ b/misp_modules/lib/__init__.py @@ -1,3 +1,3 @@ from .vt_graph_parser import * # noqa -all = ['joe_parser', 'lastline_api'] +all = ['joe_parser', 'lastline_api', 'cof2misp'] diff --git a/misp_modules/lib/cof2misp/LICENSE-2.0.txt b/misp_modules/lib/cof2misp/LICENSE-2.0.txt new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/misp_modules/lib/cof2misp/LICENSE-2.0.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/misp_modules/lib/cof2misp/__init__.py b/misp_modules/lib/cof2misp/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/misp_modules/lib/cof2misp/cof.py b/misp_modules/lib/cof2misp/cof.py new file mode 100644 index 0000000..395569e --- /dev/null +++ b/misp_modules/lib/cof2misp/cof.py @@ -0,0 +1,122 @@ +""" +Common Output Format for passive DNS library. + +Copyright 2021: Farsight Security (https://www.farsightsecurity.com/) + +Author: Aaron Kaplan + +Released under the Apache 2.0 license. +See: https://www.apache.org/licenses/LICENSE-2.0.txt + +""" + +import ipaddress +import sys +import ndjson + + +def is_valid_ip(ip: str) -> bool: + """Check if an IP address given as string would be convertible to + an ipaddress object (and thus if it is a valid IP). + + Returns + -------- + True on success, False on validation failure. + """ + + try: + ipaddress.ip_address(ip) + except Exception as ex: + print("is_valid_ip(%s) returned False. Reason: %s" % (ip, str(ex)), file=sys.stderr) + return False + return True + + +def is_cof_valid_strict(d: dict) -> bool: + """Check the COF - do the full JSON schema validation. + + Returns + -------- + True on success, False on validation failure. + """ + return True # FIXME + + +def is_cof_valid_simple(d: dict) -> bool: + """Check MANDATORY fields according to COF - simple check, do not do the full JSON schema validation. + + Returns + -------- + True on success, False on validation failure. + """ + + if "rrname" not in d: + print("Missing MANDATORY field 'rrname'", file=sys.stderr) + return False + if not isinstance(d['rrname'], str): + print("Type error: 'rrname' is not a JSON string", file=sys.stderr) + return False + if "rrtype" not in d: + print("Missing MANDATORY field 'rrtype'", file=sys.stderr) + return False + if not isinstance(d['rrtype'], str): + print("Type error: 'rrtype' is not a JSON string", file=sys.stderr) + return False + if "rdata" not in d: + print("Missing MANDATORY field 'rdata'", file=sys.stderr) + return False + if "rdata" not in d: + print("Missing MANDATORY field 'rdata'", file=sys.stderr) + return False + if not isinstance(d['rdata'], str) and not isinstance(d['rdata'], list): + print("'rdata' is not a list and not a string.", file=sys.stderr) + return False + if not ("time_first" in d and "time_last" in d) or ("zone_time_first" in d and "zone_time_last" in d): + print("We are missing EITHER ('first_seen' and 'last_seen') OR ('zone_time_first' and zone_time_last') fields", file=sys.stderr) + return False + # currently we don't check the OPTIONAL fields. Sorry... to be done later. + return True + + +def validate_cof(d: dict, strict=True) -> bool: + """Validate an input passive DNS COF (given as dict). + strict might be set to False in order to loosen the checking. + With strict==True, a full JSON Schema validation will happen. + + + Returns + -------- + True on success, False on validation failure. + """ + if not strict: + return is_cof_valid_simple(d) + else: + return is_cof_valid_strict(d) + + +if __name__ == "__main__": + # simple, poor man's unit tests. + + print(80 * "=", file=sys.stderr) + print("Unit Tests:", file=sys.stderr) + assert not is_valid_ip("a.2.3.4") + assert is_valid_ip("99.88.77.6") + assert is_valid_ip("2a0c:88:77:6::1") + + # COF validation + mock_input = """{"count":1909,"rdata":["cpa.circl.lu"],"rrname":"www.circl.lu","rrtype":"CNAME","time_first":"1315586409","time_last":"1449566799"} +{"count":2560,"rdata":["cpab.circl.lu"],"rrname":"www.circl.lu","rrtype":"CNAME","time_first":"1449584660","time_last":"1617676151"}""" + + i = 0 + for entry in ndjson.loads(mock_input): + retval = validate_cof(entry, strict=False) + assert retval + print("line %d is valid: %s" % (i, retval)) + i += 1 + + test2 = '{"count": 2, "time_first": 1619556027, "time_last": 1619556034, "rrname": "westernunion.com.ph.unblock-all.com.beta.opera-mini.net.", "rrtype": "A", "bailiwick": "beta.opera-mini.net.", "rdata": ["185.26.181.253"]}' + for entry in ndjson.loads(test2): + assert validate_cof(entry) + + print(80 * "=", file=sys.stderr) + print("Unit Tests DONE", file=sys.stderr) diff --git a/misp_modules/modules/expansion/farsight_passivedns.py b/misp_modules/modules/expansion/farsight_passivedns.py index de18735..47e7eaa 100755 --- a/misp_modules/modules/expansion/farsight_passivedns.py +++ b/misp_modules/modules/expansion/farsight_passivedns.py @@ -85,7 +85,7 @@ class FarsightDnsdbParser(): self.misp_event = MISPEvent() self.misp_event.add_attribute(**attribute) self.passivedns_mapping = { - 'bailiwick': {'type': 'text', 'object_relation': 'bailiwick'}, + 'bailiwick': {'type': 'domain', 'object_relation': 'bailiwick'}, 'count': {'type': 'counter', 'object_relation': 'count'}, 'raw_rdata': {'type': 'text', 'object_relation': 'raw_rdata'}, 'rdata': {'type': 'text', 'object_relation': 'rdata'}, @@ -148,6 +148,8 @@ def handler(q=False): response = to_query(client, *args) except dnsdb2.DnsdbException as e: return {'error': e.__str__()} + except dnsdb2.exceptions.QueryError: + return {'error': 'Communication error occurs while executing a query, or the server reports an error due to invalid arguments.'} if not response: return {'error': f"Empty results on Farsight DNSDB for the {TYPE_TO_FEATURE[attribute['type']]}: {attribute['value']}."} parser = FarsightDnsdbParser(attribute) diff --git a/misp_modules/modules/import_mod/__init__.py b/misp_modules/modules/import_mod/__init__.py index 694a434..71ae7fa 100644 --- a/misp_modules/modules/import_mod/__init__.py +++ b/misp_modules/modules/import_mod/__init__.py @@ -13,5 +13,6 @@ __all__ = [ 'openiocimport', 'threatanalyzer_import', 'csvimport', + 'cof2misp', 'joe_import', ] diff --git a/misp_modules/modules/import_mod/cof2misp.py b/misp_modules/modules/import_mod/cof2misp.py new file mode 100755 index 0000000..16bf259 --- /dev/null +++ b/misp_modules/modules/import_mod/cof2misp.py @@ -0,0 +1,224 @@ +""" PassiveDNS Common Output Format (COF) MISP importer. + +Takes as input a valid COF file or the output of the dnsdbflex utility +and creates MISP objects for the input. + +Copyright 2021: Farsight Security (https://www.farsightsecurity.com/) + +Author: Aaron Kaplan + +Released under the Apache 2.0 license. +See: https://www.apache.org/licenses/LICENSE-2.0.txt + +""" + +import sys +import json +import base64 + + +import ndjson + +# from pymisp import MISPObject, MISPEvent, PyMISP +from pymisp import MISPObject + +from cof2misp.cof import validate_cof + + +create_specific_attributes = False # this is for https://github.com/MISP/misp-objects/pull/314 + + +misperrors = {'error': 'Error'} +userConfig = {} + +inputSource = ['file'] + +mispattributes = {'inputSource': ['file'], 'output': ['MISP objects'], + 'format': 'misp_standard'} + + +moduleinfo = {'version': '0.2', 'author': 'Aaron Kaplan', + 'description': 'Module to import the passive DNS Common Output Format (COF) and merge as a MISP objet into a MISP event.', + 'module-type': ['import']} + +moduleconfig = [] + + +# misp = PyMISP() + + +def parse_and_insert_cof(data: str) -> dict: + """Parse and validate the COF data. + + Parameters + ---------- + data as a string + + Returns + ------- + A dict with either the error message or the data which may be sent off the the caller of handler() + + Raises + -------- + none. All Exceptions will be handled here. On error, a misperror is returned. + """ + + objects = [] + try: + entries = ndjson.loads(data) + for entry in entries: # iterate over all ndjson lines + + # validate here (simple validation or full JSON Schema validation) + if not validate_cof(entry): + return {"error": "Could not validate the COF input '%s'" % entry} + + # Next, extract some fields + rrtype = entry['rrtype'].upper() + rrname = entry['rrname'].rstrip('.') + rdata = [x.rstrip('.') for x in entry['rdata']] + + # create a new MISP object, based on the passive-dns object for each nd-JSON line + o = MISPObject(name='passive-dns', standalone=False, comment='created by cof2misp') + + # o.add_tag('tlp:amber') # FIXME: we'll want to add a tlp: tag to the object + o.add_attribute('bailiwick', value=entry['bailiwick'].rstrip('.')) + + # + # handle the combinations of rrtype (domain, ip) on both left and right side + # + + if create_specific_attributes: + if rrtype in ['A', 'AAAA', 'A6']: # address type + # address type + o.add_attribute('rrname_domain', value=rrname) + for r in rdata: + o.add_attribute('rdata_ip', value=r) + elif rrtype in ['CNAME', 'DNAME', 'NS']: # both sides are domains + o.add_attribute('rrname_domain', value=rrname) + for r in rdata: + o.add_attribute('rdata_domain', value=r) + elif rrtype in ['SOA']: # left side is a domain, right side is text + o.add_attribute('rrname_domain', value=rrname) + + # + # now do the regular filling up of rrname, rrtype, time_first, etc. + # + o.add_attribute('rrname', value=rrname) + o.add_attribute('rrtype', value=rrtype) + for r in rdata: + o.add_attribute('rdata', value=r) + o.add_attribute('raw_rdata', value=json.dumps(rdata)) # FIXME: do we need to hex encode it? + o.add_attribute('time_first', value=entry['time_first']) + o.add_attribute('time_last', value=entry['time_last']) + o.first_seen = entry['time_first'] # is this redundant? + o.last_seen = entry['time_last'] + + # + # Now add the other optional values. # FIXME: how about a map() other function. DNRY + # + for k in ['count', 'sensor_id', 'origin', 'text', 'time_first_ms', 'time_last_ms', 'zone_time_first', 'zone_time_last']: + if k in entry and entry[k]: + o.add_attribute(k, value=entry[k]) + + # + # add COF entry to MISP object + # + objects.append(o.to_json()) + + r = {'results': {'Object': [json.loads(o) for o in objects]}} + except Exception as ex: + misperrors["error"] = "An error occured during parsing of input: '%s'" % (str(ex),) + return misperrors + return r + + +def parse_and_insert_dnsdbflex(data: str): + """Parse and validate the more simplier dndsdbflex output data. + + Parameters + ---------- + data as a string + + Returns + ------- + A dict with either the error message or the data which may be sent off the the caller of handler() + + Raises + -------- + none + """ + return {"error": "NOT IMPLEMENTED YET"} # XXX FIXME: need a MISP object for dnsdbflex + + +def is_dnsdbflex(data: str) -> bool: + """Check if the supplied data conforms to the dnsdbflex output (which only contains rrname and rrtype) + + Parameters + ---------- + ndjson data as a string + + Returns + ------- + True or False + + Raises + -------- + none + """ + + try: + j = ndjson.loads(data) + for line in j: + if not set(line.keys()) == {'rrname', 'rrtype'}: + return False # shortcut. We assume it's not if a single line does not conform + return True + except Exception as ex: + print("oops, this should not have happened. Maybe not an ndjson file? Reason: %s" % (str(ex),), file=sys.stderr) + return False + + +def is_cof(data: str) -> bool: + return True + + +def handler(q=False): + if q is False: + return False + + request = json.loads(q) + # Parse the json, determine which type of JSON it is (dnsdbflex or COF?) + # Validate it + # transform into MISP object + # push to MISP + # event_id = request['event_id'] + # event = misp.get_event(event_id) + # print("event_id = %s" % event_id, file=sys.stderr) + try: + data = base64.b64decode(request["data"]).decode('utf-8') + if not data: + return json.dumps({'success': 0}) # empty file is ok + if is_dnsdbflex(data): + return parse_and_insert_dnsdbflex(data) + elif is_cof(data): + # check if it's valid COF format + return parse_and_insert_cof(data) + else: + return {'error': 'Could not find any valid COF input nor dnsdbflex input. Please have a loot at: https://datatracker.ietf.org/doc/draft-dulaunoy-dnsop-passive-dns-cof/'} + except Exception as ex: + print("oops, got exception %s" % str(ex), file=sys.stderr) + return {'error': "Got exception %s" % str(ex)} + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo + + +if __name__ == '__main__': + x = open('test.json', 'r') + r = handler(q=x.read()) + print(json.dumps(r))