From 0ec8339d7ae40e2e97749aa391bd2cb97aa3d707 Mon Sep 17 00:00:00 2001 From: Christophe Vandeplas Date: Tue, 5 Dec 2017 16:41:41 +0100 Subject: [PATCH 1/2] New Farsight DNSDB Passive DNS expansion module --- README.md | 1 + misp_modules/modules/expansion/__init__.py | 2 +- .../modules/expansion/_dnsdb_query/COPYRIGHT | 27 ++ .../modules/expansion/_dnsdb_query/LICENSE | 202 +++++++++++ .../modules/expansion/_dnsdb_query/README.md | 162 +++++++++ .../expansion/_dnsdb_query/dnsdb_query.py | 323 ++++++++++++++++++ .../modules/expansion/farsight_passivedns.py | 72 ++++ 7 files changed, 788 insertions(+), 1 deletion(-) create mode 100755 misp_modules/modules/expansion/_dnsdb_query/COPYRIGHT create mode 100755 misp_modules/modules/expansion/_dnsdb_query/LICENSE create mode 100755 misp_modules/modules/expansion/_dnsdb_query/README.md create mode 100755 misp_modules/modules/expansion/_dnsdb_query/dnsdb_query.py create mode 100755 misp_modules/modules/expansion/farsight_passivedns.py diff --git a/README.md b/README.md index 1eafb9b..f630b7b 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/ * [DNS](misp_modules/modules/expansion/dns.py) - a simple module to resolve MISP attributes like hostname and domain to expand IP addresses attributes. * [DomainTools](misp_modules/modules/expansion/domaintools.py) - a hover and expansion module to get information from [DomainTools](http://www.domaintools.com/) whois. * [EUPI](misp_modules/modules/expansion/eupi.py) - a hover and expansion module to get information about an URL from the [Phishing Initiative project](https://phishing-initiative.eu/?lang=en). +* [Farsight DNSDB Passive DNS](misp_modules/modules/expansion/farsight_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information. * [GeoIP](misp_modules/modules/expansion/geoip_country.py) - a hover and expansion module to get GeoIP information from geolite/maxmind. * [IPASN](misp_modules/modules/expansion/ipasn.py) - a hover and expansion to get the BGP ASN of an IP address. * [iprep](misp-modules/modules/expansion/iprep.py) - an expansion module to get IP reputation from packetmail.net. diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py index 48117e1..231d30e 100644 --- a/misp_modules/modules/expansion/__init__.py +++ b/misp_modules/modules/expansion/__init__.py @@ -1,6 +1,6 @@ from . import _vmray __all__ = ['vmray_submit', 'asn_history', 'circl_passivedns', 'circl_passivessl', - 'countrycode', 'cve', 'dns', 'domaintools', 'eupi', 'ipasn', 'passivetotal', 'sourcecache', + 'countrycode', 'cve', 'dns', 'domaintools', 'eupi', 'farsight_passivedns', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal', 'whois', 'shodan', 'reversedns', 'geoip_country', 'wiki', 'iprep', 'threatminer' ,'otx', 'threatcrowd','vulndb'] diff --git a/misp_modules/modules/expansion/_dnsdb_query/COPYRIGHT b/misp_modules/modules/expansion/_dnsdb_query/COPYRIGHT new file mode 100755 index 0000000..9606e4a --- /dev/null +++ b/misp_modules/modules/expansion/_dnsdb_query/COPYRIGHT @@ -0,0 +1,27 @@ +Copyright (c) 2013 by Farsight Security, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Copyright (c) 2010-2012 by Internet Systems Consortium, Inc. ("ISC") + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT +OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/misp_modules/modules/expansion/_dnsdb_query/LICENSE b/misp_modules/modules/expansion/_dnsdb_query/LICENSE new file mode 100755 index 0000000..d645695 --- /dev/null +++ b/misp_modules/modules/expansion/_dnsdb_query/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/misp_modules/modules/expansion/_dnsdb_query/README.md b/misp_modules/modules/expansion/_dnsdb_query/README.md new file mode 100755 index 0000000..af97937 --- /dev/null +++ b/misp_modules/modules/expansion/_dnsdb_query/README.md @@ -0,0 +1,162 @@ +dnsdb-query +=========== + +These clients are reference implementations of the [DNSDB HTTP API](https://api.dnsdb.info/). Output is +compliant with the [Passive DNS Common Output Format](http://tools.ietf.org/html/draft-dulaunoy-kaplan-passive-dns-cof-01). + +Please see https://www.dnsdb.info/ for more information. + +Requirements +------------ + * Linux, BSD, OS X + * Curl + * Python 2.7.x + * Farsight DNSDB API key + +Installation +------------ +1. Create a directory + + ``` + mkdir ~/dnsdb + ``` +1. Download the software + + ``` + curl https://codeload.github.com/dnsdb/dnsdb-query/tar.gz/debian/0.2-1 -o ~/dnsdb/0.2-1.tar.gz + ``` +1. Extract the software + + ``` + tar xzvf ~/dnsdb/0.2-1.tar.gz -C ~/dnsdb/ --strip-components=1 + ``` +1. Create a API key file + + ``` + nano ~/.dnsdb-query.conf + ``` +1. Cut and paste the following and replace '\' with your API Key + + ``` + APIKEY="" + ``` +1. Test the Python client + + ``` + $ python dnsdb/dnsdb_query.py -i 104.244.13.104 + ``` + ``` + ... + www.farsightsecurity.com. IN A 104.244.13.104 + ``` + +dnsdb_query.py +-------------- + +dnsdb_query.py is a Python client for the DNSDB HTTP API. It is similar +to the dnsdb-query shell script but supports some additional features +like sorting and setting the result limit parameter. It is also embeddable +as a Python module. + +``` +Usage: dnsdb_query.py [options] + +Options: + -h, --help show this help message and exit + -c CONFIG, --config=CONFIG + config file + -r RRSET, --rrset=RRSET + rrset [/[/BAILIWICK]] + -n RDATA_NAME, --rdataname=RDATA_NAME + rdata name [/] + -i RDATA_IP, --rdataip=RDATA_IP + rdata ip + -s SORT, --sort=SORT sort key + -R, --reverse reverse sort + -j, --json output in JSON format + -l LIMIT, --limit=LIMIT + limit number of results + --before=BEFORE only output results seen before this time + --after=AFTER only output results seen after this time + +Time formats are: "%Y-%m-%d", "%Y-%m-%d %H:%M:%S", "%d" (UNIX timestamp), +"-%d" (Relative time in seconds), BIND format relative timestamp (e.g. 1w1h, +(w)eek, (d)ay, (h)our, (m)inute, (s)econd) +``` + +Or, from Python: + +``` +from dnsdb_query import DnsdbClient + +server='https://api.dnsdb.info' +apikey='d41d8cd98f00b204e9800998ecf8427e' + +client = DnsdbClient(server,apikey) +for rrset in client.query_rrset('www.dnsdb.info'): + # rrset is a decoded JSON blob + print repr(rrset) +``` + +Other configuration options that may be set: + +`DNSDB_SERVER` +The base URL of the DNSDB HTTP API, minus the /lookup component. Defaults to +`https://api.dnsdb.info.` + +`HTTP_PROXY` +The URL of the HTTP proxy that you wish to use. + +`HTTPS_PROXY` +The URL of the HTTPS proxy that you wish to use. + +dnsdb-query +----------- + +dnsdb-query is a simple curl-based wrapper for the DNSDB HTTP API. + +The script sources the config file `/etc/dnsdb-query.conf` as a shell fragment. +If the config file is not present in `/etc`, the file `$HOME/.dnsdb-query.conf` +is sourced instead. + +The config file MUST set the value of the APIKEY shell variable to the API +key provided to you by Farsight Security. + +For example, if your API key is d41d8cd98f00b204e9800998ecf8427e, place the +following line in `/etc/dnsdb-query.conf` or `$HOME/.dnsdb-query.conf`: + +``` +APIKEY="d41d8cd98f00b204e9800998ecf8427e" +``` + +Other shell variables that may be set via the config file or command line +are: + +`DNSDB_SERVER` +The base URL of the DNSDB HTTP API, minus the /lookup component. Defaults to +`https://api.dnsdb.info.` + +`DNSDB_FORMAT` +The result format to use, either text or json. Defaults to text. + +`HTTP_PROXY` +The URL of the HTTP proxy that you wish to use. + +`HTTPS_PROXY` +The URL of the HTTPS proxy that you wish to use. + +dnsdb-query supports the following usages: + +``` +Usage: dnsdb-query rrset [/[/]] +Usage: dnsdb-query rdata ip +Usage: dnsdb-query rdata name [/] +Usage: dnsdb-query rdata raw [/] +``` + +If your rrname, bailiwick or rdata contains the `/` character you +will need to escape it to `%2F` on the command line. eg: + +`./dnsdb_query -r 1.0%2F1.0.168.192.in-addr.arpa` + +retrieves the rrsets for `1.0/1.0.168.192.in-addr.arpa`. diff --git a/misp_modules/modules/expansion/_dnsdb_query/dnsdb_query.py b/misp_modules/modules/expansion/_dnsdb_query/dnsdb_query.py new file mode 100755 index 0000000..0ab58e8 --- /dev/null +++ b/misp_modules/modules/expansion/_dnsdb_query/dnsdb_query.py @@ -0,0 +1,323 @@ +#!/usr/bin/env python +# +# Note: This file is NOT the official one from dnsdb, as it has a python3 cherry-picked pull-request applied for python3 compatibility +# See https://github.com/dnsdb/dnsdb-query/pull/30 +# +# Copyright (c) 2013 by Farsight Security, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import print_function + +import calendar +import errno +import locale +import optparse +import os +import re +import sys +import time +import json +from io import StringIO + +try: + from urllib2 import build_opener, Request, ProxyHandler, HTTPError, URLError + from urllib import quote as urllib_quote, urlencode +except ImportError: + from urllib.request import build_opener, Request, ProxyHandler, HTTPError, URLError + from urllib.parse import quote as urllib_quote, urlencode + + +DEFAULT_CONFIG_FILES = filter(os.path.isfile, ('/etc/dnsdb-query.conf', os.path.expanduser('~/.dnsdb-query.conf'))) +DEFAULT_DNSDB_SERVER = 'https://api.dnsdb.info' +DEFAULT_HTTP_PROXY = '' +DEFAULT_HTTPS_PROXY = '' + +cfg = None +options = None + +locale.setlocale(locale.LC_ALL, '') + +class QueryError(Exception): + pass + +class DnsdbClient(object): + def __init__(self, server, apikey, limit=None, http_proxy=None, https_proxy=None): + self.server = server + self.apikey = apikey + self.limit = limit + self.http_proxy = http_proxy + self.https_proxy = https_proxy + + def query_rrset(self, oname, rrtype=None, bailiwick=None, before=None, after=None): + if bailiwick: + if not rrtype: + rrtype = 'ANY' + path = 'rrset/name/%s/%s/%s' % (quote(oname), rrtype, quote(bailiwick)) + elif rrtype: + path = 'rrset/name/%s/%s' % (quote(oname), rrtype) + else: + path = 'rrset/name/%s' % quote(oname) + return self._query(path, before, after) + + def query_rdata_name(self, rdata_name, rrtype=None, before=None, after=None): + if rrtype: + path = 'rdata/name/%s/%s' % (quote(rdata_name), rrtype) + else: + path = 'rdata/name/%s' % quote(rdata_name) + return self._query(path, before, after) + + def query_rdata_ip(self, rdata_ip, before=None, after=None): + path = 'rdata/ip/%s' % rdata_ip.replace('/', ',') + return self._query(path, before, after) + + def _query(self, path, before=None, after=None): + res = [] + url = '%s/lookup/%s' % (self.server, path) + + params = {} + if self.limit: + params['limit'] = self.limit + if before and after: + params['time_first_after'] = after + params['time_last_before'] = before + else: + if before: + params['time_first_before'] = before + if after: + params['time_last_after'] = after + if params: + url += '?{0}'.format(urlencode(params)) + + req = Request(url) + req.add_header('Accept', 'application/json') + req.add_header('X-Api-Key', self.apikey) + + proxy_args = {} + if self.http_proxy: + proxy_args['http'] = self.http_proxy + if self.https_proxy: + proxy_args['https'] = self.https_proxy + proxy_handler = ProxyHandler(proxy_args) + opener = build_opener(proxy_handler) + + try: + http = opener.open(req) + while True: + line = http.readline() + if not line: + break + yield json.loads(line.decode('ascii')) + except (HTTPError, URLError) as e: + raise QueryError(str(e), sys.exc_traceback) + +def quote(path): + return urllib_quote(path, safe='') + +def sec_to_text(ts): + return time.strftime('%Y-%m-%d %H:%M:%S -0000', time.gmtime(ts)) + +def rrset_to_text(m): + s = StringIO() + + try: + if 'bailiwick' in m: + s.write(';; bailiwick: %s\n' % m['bailiwick']) + + if 'count' in m: + s.write(';; count: %s\n' % locale.format('%d', m['count'], True)) + + if 'time_first' in m: + s.write(';; first seen: %s\n' % sec_to_text(m['time_first'])) + if 'time_last' in m: + s.write(';; last seen: %s\n' % sec_to_text(m['time_last'])) + + if 'zone_time_first' in m: + s.write(';; first seen in zone file: %s\n' % sec_to_text(m['zone_time_first'])) + if 'zone_time_last' in m: + s.write(';; last seen in zone file: %s\n' % sec_to_text(m['zone_time_last'])) + + if 'rdata' in m: + for rdata in m['rdata']: + s.write('%s IN %s %s\n' % (m['rrname'], m['rrtype'], rdata)) + + s.seek(0) + return s.read() + finally: + s.close() + +def rdata_to_text(m): + return '%s IN %s %s' % (m['rrname'], m['rrtype'], m['rdata']) + +def parse_config(cfg_files): + config = {} + + if not cfg_files: + raise IOError(errno.ENOENT, 'dnsdb_query: No config files found') + + for fname in cfg_files: + for line in open(fname): + key, eq, val = line.strip().partition('=') + val = val.strip('"') + config[key] = val + + return config + +def time_parse(s): + try: + epoch = int(s) + return epoch + except ValueError: + pass + + try: + epoch = int(calendar.timegm(time.strptime(s, '%Y-%m-%d'))) + return epoch + except ValueError: + pass + + try: + epoch = int(calendar.timegm(time.strptime(s, '%Y-%m-%d %H:%M:%S'))) + return epoch + except ValueError: + pass + + m = re.match(r'^(?=\d)(?:(\d+)w)?(?:(\d+)d)?(?:(\d+)h)?(?:(\d+)m)?(?:(\d+)s?)?$', s, re.I) + if m: + return -1*(int(m.group(1) or 0)*604800 + + int(m.group(2) or 0)*86400+ + int(m.group(3) or 0)*3600+ + int(m.group(4) or 0)*60+ + int(m.group(5) or 0)) + + raise ValueError('Invalid time: "%s"' % s) + +def epipe_wrapper(func): + def f(*args, **kwargs): + try: + return func(*args, **kwargs) + except IOError as e: + if e.errno == errno.EPIPE: + sys.exit(e.errno) + raise + return f + +@epipe_wrapper +def main(): + global cfg + global options + + parser = optparse.OptionParser(epilog='Time formats are: "%Y-%m-%d", "%Y-%m-%d %H:%M:%S", "%d" (UNIX timestamp), "-%d" (Relative time in seconds), BIND format (e.g. 1w1h, (w)eek, (d)ay, (h)our, (m)inute, (s)econd)') + parser.add_option('-c', '--config', dest='config', + help='config file', action='append') + parser.add_option('-r', '--rrset', dest='rrset', type='string', + help='rrset [/[/BAILIWICK]]') + parser.add_option('-n', '--rdataname', dest='rdata_name', type='string', + help='rdata name [/]') + parser.add_option('-i', '--rdataip', dest='rdata_ip', type='string', + help='rdata ip ') + parser.add_option('-t', '--rrtype', dest='rrtype', type='string', + help='rrset or rdata rrtype') + parser.add_option('-b', '--bailiwick', dest='bailiwick', type='string', + help='rrset bailiwick') + parser.add_option('-s', '--sort', dest='sort', type='string', help='sort key') + parser.add_option('-R', '--reverse', dest='reverse', action='store_true', default=False, + help='reverse sort') + parser.add_option('-j', '--json', dest='json', action='store_true', default=False, + help='output in JSON format') + parser.add_option('-l', '--limit', dest='limit', type='int', default=0, + help='limit number of results') + + parser.add_option('', '--before', dest='before', type='string', help='only output results seen before this time') + parser.add_option('', '--after', dest='after', type='string', help='only output results seen after this time') + + options, args = parser.parse_args() + if args: + parser.print_help() + sys.exit(1) + + try: + if options.before: + options.before = time_parse(options.before) + except ValueError: + print('Could not parse before: {}'.format(options.before)) + + try: + if options.after: + options.after = time_parse(options.after) + except ValueError: + print('Could not parse after: {}'.format(options.after)) + + try: + cfg = parse_config(options.config or DEFAULT_CONFIG_FILES) + except IOError as e: + print(str(e), file=sys.stderr) + sys.exit(1) + + if not 'DNSDB_SERVER' in cfg: + cfg['DNSDB_SERVER'] = DEFAULT_DNSDB_SERVER + if not 'HTTP_PROXY' in cfg: + cfg['HTTP_PROXY'] = DEFAULT_HTTP_PROXY + if not 'HTTPS_PROXY' in cfg: + cfg['HTTPS_PROXY'] = DEFAULT_HTTPS_PROXY + if not 'APIKEY' in cfg: + sys.stderr.write('dnsdb_query: APIKEY not defined in config file\n') + sys.exit(1) + + client = DnsdbClient(cfg['DNSDB_SERVER'], cfg['APIKEY'], + limit=options.limit, + http_proxy=cfg['HTTP_PROXY'], + https_proxy=cfg['HTTPS_PROXY']) + if options.rrset: + if options.rrtype or options.bailiwick: + qargs = (options.rrset, options.rrtype, options.bailiwick) + else: + qargs = (options.rrset.split('/', 2)) + + results = client.query_rrset(*qargs, before=options.before, after=options.after) + fmt_func = rrset_to_text + elif options.rdata_name: + if options.rrtype: + qargs = (options.rdata_name, options.rrtype) + else: + qargs = (options.rdata_name.split('/', 1)) + + results = client.query_rdata_name(*qargs, before=options.before, after=options.after) + fmt_func = rdata_to_text + elif options.rdata_ip: + results = client.query_rdata_ip(options.rdata_ip, before=options.before, after=options.after) + fmt_func = rdata_to_text + else: + parser.print_help() + sys.exit(1) + + if options.json: + fmt_func = json.dumps + + try: + if options.sort: + results = list(results) + if len(results) > 0: + if not options.sort in results[0]: + sort_keys = results[0].keys() + sort_keys.sort() + sys.stderr.write('dnsdb_query: invalid sort key "%s". valid sort keys are %s\n' % (options.sort, ', '.join(sort_keys))) + sys.exit(1) + results.sort(key=lambda r: r[options.sort], reverse=options.reverse) + for res in results: + sys.stdout.write('%s\n' % fmt_func(res)) + except QueryError as e: + print(e.message, file=sys.stderr) + sys.exit(1) + +if __name__ == '__main__': + main() diff --git a/misp_modules/modules/expansion/farsight_passivedns.py b/misp_modules/modules/expansion/farsight_passivedns.py new file mode 100755 index 0000000..7c0ad23 --- /dev/null +++ b/misp_modules/modules/expansion/farsight_passivedns.py @@ -0,0 +1,72 @@ +import json +from ._dnsdb_query.dnsdb_query import DnsdbClient + + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['hostname', 'domain', 'ip-src', 'ip-dst'], 'output': ['freetext']} +moduleinfo = {'version': '0.1', 'author': 'Christophe Vandeplas', 'description': 'Module to access Farsight DNSDB Passive DNS', 'module-type': ['expansion', 'hover']} +moduleconfig = ['apikey'] + +server = 'https://api.dnsdb.info' + +# TODO return a MISP object with the different attributes + + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + if (request.get('config')): + if (request['config'].get('apikey') is None): + misperrors['error'] = 'Farsight DNSDB apikey is missing' + return misperrors + client = DnsdbClient(server, request['config']['apikey']) + if request.get('hostname'): + res = lookup_name(client, request['hostname']) + elif request.get('domain'): + res = lookup_name(client, request['domain']) + elif request.get('ip-src'): + res = lookup_ip(client, request['ip-src']) + elif request.get('ip-dst'): + res = lookup_ip(client, request['ip-dst']) + else: + misperrors['error'] = "Unsupported attributes type" + return misperrors + + out = '' + for v in set(res): # uniquify entries + out = out + "{} ".format(v) + r = {'results': [{'types': mispattributes['output'], 'values': out}]} + return r + + +def lookup_name(client, name): + res = client.query_rrset(name) # RRSET = entries in the left-hand side of the domain name related labels + for item in res: + if item.get('rrtype') in ['A', 'AAAA', 'CNAME']: + for i in item.get('rdata'): + yield(i.rstrip('.')) + if item.get('rrtype') in ['SOA']: + for i in item.get('rdata'): + # grab email field and replace first dot by @ to convert to an email address + yield(i.split(' ')[1].rstrip('.').replace('.', '@', 1)) + res = client.query_rdata_name(name) # RDATA = entries on the right-hand side of the domain name related labels + for item in res: + if item.get('rrtype') in ['A', 'AAAA', 'CNAME']: + yield(item.get('rrname').rstrip('.')) + + +def lookup_ip(client, ip): + res = client.query_rdata_ip(ip) + for item in res: + print(item) + yield(item['rrname'].rstrip('.')) + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo From 4cdb143733c22c80802e8cdd8aad3f7bc8754ab8 Mon Sep 17 00:00:00 2001 From: Christophe Vandeplas Date: Wed, 6 Dec 2017 09:23:44 +0100 Subject: [PATCH 2/2] fixes missing init file in dnsdb library folder --- misp_modules/modules/expansion/_dnsdb_query/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 misp_modules/modules/expansion/_dnsdb_query/__init__.py diff --git a/misp_modules/modules/expansion/_dnsdb_query/__init__.py b/misp_modules/modules/expansion/_dnsdb_query/__init__.py new file mode 100644 index 0000000..e69de29