mirror of https://github.com/MISP/misp-modules
Merge branch 'cvandeplas-master'
commit
d35a2ee4ae
|
|
@ -24,6 +24,7 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/
|
||||||
* [DNS](misp_modules/modules/expansion/dns.py) - a simple module to resolve MISP attributes like hostname and domain to expand IP addresses attributes.
|
* [DNS](misp_modules/modules/expansion/dns.py) - a simple module to resolve MISP attributes like hostname and domain to expand IP addresses attributes.
|
||||||
* [DomainTools](misp_modules/modules/expansion/domaintools.py) - a hover and expansion module to get information from [DomainTools](http://www.domaintools.com/) whois.
|
* [DomainTools](misp_modules/modules/expansion/domaintools.py) - a hover and expansion module to get information from [DomainTools](http://www.domaintools.com/) whois.
|
||||||
* [EUPI](misp_modules/modules/expansion/eupi.py) - a hover and expansion module to get information about an URL from the [Phishing Initiative project](https://phishing-initiative.eu/?lang=en).
|
* [EUPI](misp_modules/modules/expansion/eupi.py) - a hover and expansion module to get information about an URL from the [Phishing Initiative project](https://phishing-initiative.eu/?lang=en).
|
||||||
|
* [Farsight DNSDB Passive DNS](misp_modules/modules/expansion/farsight_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information.
|
||||||
* [GeoIP](misp_modules/modules/expansion/geoip_country.py) - a hover and expansion module to get GeoIP information from geolite/maxmind.
|
* [GeoIP](misp_modules/modules/expansion/geoip_country.py) - a hover and expansion module to get GeoIP information from geolite/maxmind.
|
||||||
* [IPASN](misp_modules/modules/expansion/ipasn.py) - a hover and expansion to get the BGP ASN of an IP address.
|
* [IPASN](misp_modules/modules/expansion/ipasn.py) - a hover and expansion to get the BGP ASN of an IP address.
|
||||||
* [iprep](misp-modules/modules/expansion/iprep.py) - an expansion module to get IP reputation from packetmail.net.
|
* [iprep](misp-modules/modules/expansion/iprep.py) - an expansion module to get IP reputation from packetmail.net.
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
from . import _vmray
|
from . import _vmray
|
||||||
|
|
||||||
__all__ = ['vmray_submit', 'asn_history', 'circl_passivedns', 'circl_passivessl',
|
__all__ = ['vmray_submit', 'asn_history', 'circl_passivedns', 'circl_passivessl',
|
||||||
'countrycode', 'cve', 'dns', 'domaintools', 'eupi', 'ipasn', 'passivetotal', 'sourcecache',
|
'countrycode', 'cve', 'dns', 'domaintools', 'eupi', 'farsight_passivedns', 'ipasn', 'passivetotal', 'sourcecache',
|
||||||
'virustotal', 'whois', 'shodan', 'reversedns', 'geoip_country', 'wiki', 'iprep', 'threatminer' ,'otx',
|
'virustotal', 'whois', 'shodan', 'reversedns', 'geoip_country', 'wiki', 'iprep', 'threatminer' ,'otx',
|
||||||
'threatcrowd','vulndb']
|
'threatcrowd','vulndb']
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,27 @@
|
||||||
|
Copyright (c) 2013 by Farsight Security, Inc.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
|
||||||
|
Copyright (c) 2010-2012 by Internet Systems Consortium, Inc. ("ISC")
|
||||||
|
|
||||||
|
Permission to use, copy, modify, and/or distribute this software for any
|
||||||
|
purpose with or without fee is hereby granted, provided that the above
|
||||||
|
copyright notice and this permission notice appear in all copies.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES
|
||||||
|
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
||||||
|
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR
|
||||||
|
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||||
|
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||||
|
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
|
||||||
|
OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||||
|
|
@ -0,0 +1,202 @@
|
||||||
|
|
||||||
|
Apache License
|
||||||
|
Version 2.0, January 2004
|
||||||
|
http://www.apache.org/licenses/
|
||||||
|
|
||||||
|
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||||
|
|
||||||
|
1. Definitions.
|
||||||
|
|
||||||
|
"License" shall mean the terms and conditions for use, reproduction,
|
||||||
|
and distribution as defined by Sections 1 through 9 of this document.
|
||||||
|
|
||||||
|
"Licensor" shall mean the copyright owner or entity authorized by
|
||||||
|
the copyright owner that is granting the License.
|
||||||
|
|
||||||
|
"Legal Entity" shall mean the union of the acting entity and all
|
||||||
|
other entities that control, are controlled by, or are under common
|
||||||
|
control with that entity. For the purposes of this definition,
|
||||||
|
"control" means (i) the power, direct or indirect, to cause the
|
||||||
|
direction or management of such entity, whether by contract or
|
||||||
|
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||||
|
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||||
|
|
||||||
|
"You" (or "Your") shall mean an individual or Legal Entity
|
||||||
|
exercising permissions granted by this License.
|
||||||
|
|
||||||
|
"Source" form shall mean the preferred form for making modifications,
|
||||||
|
including but not limited to software source code, documentation
|
||||||
|
source, and configuration files.
|
||||||
|
|
||||||
|
"Object" form shall mean any form resulting from mechanical
|
||||||
|
transformation or translation of a Source form, including but
|
||||||
|
not limited to compiled object code, generated documentation,
|
||||||
|
and conversions to other media types.
|
||||||
|
|
||||||
|
"Work" shall mean the work of authorship, whether in Source or
|
||||||
|
Object form, made available under the License, as indicated by a
|
||||||
|
copyright notice that is included in or attached to the work
|
||||||
|
(an example is provided in the Appendix below).
|
||||||
|
|
||||||
|
"Derivative Works" shall mean any work, whether in Source or Object
|
||||||
|
form, that is based on (or derived from) the Work and for which the
|
||||||
|
editorial revisions, annotations, elaborations, or other modifications
|
||||||
|
represent, as a whole, an original work of authorship. For the purposes
|
||||||
|
of this License, Derivative Works shall not include works that remain
|
||||||
|
separable from, or merely link (or bind by name) to the interfaces of,
|
||||||
|
the Work and Derivative Works thereof.
|
||||||
|
|
||||||
|
"Contribution" shall mean any work of authorship, including
|
||||||
|
the original version of the Work and any modifications or additions
|
||||||
|
to that Work or Derivative Works thereof, that is intentionally
|
||||||
|
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||||
|
or by an individual or Legal Entity authorized to submit on behalf of
|
||||||
|
the copyright owner. For the purposes of this definition, "submitted"
|
||||||
|
means any form of electronic, verbal, or written communication sent
|
||||||
|
to the Licensor or its representatives, including but not limited to
|
||||||
|
communication on electronic mailing lists, source code control systems,
|
||||||
|
and issue tracking systems that are managed by, or on behalf of, the
|
||||||
|
Licensor for the purpose of discussing and improving the Work, but
|
||||||
|
excluding communication that is conspicuously marked or otherwise
|
||||||
|
designated in writing by the copyright owner as "Not a Contribution."
|
||||||
|
|
||||||
|
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||||
|
on behalf of whom a Contribution has been received by Licensor and
|
||||||
|
subsequently incorporated within the Work.
|
||||||
|
|
||||||
|
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||||
|
this License, each Contributor hereby grants to You a perpetual,
|
||||||
|
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||||
|
copyright license to reproduce, prepare Derivative Works of,
|
||||||
|
publicly display, publicly perform, sublicense, and distribute the
|
||||||
|
Work and such Derivative Works in Source or Object form.
|
||||||
|
|
||||||
|
3. Grant of Patent License. Subject to the terms and conditions of
|
||||||
|
this License, each Contributor hereby grants to You a perpetual,
|
||||||
|
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||||
|
(except as stated in this section) patent license to make, have made,
|
||||||
|
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||||
|
where such license applies only to those patent claims licensable
|
||||||
|
by such Contributor that are necessarily infringed by their
|
||||||
|
Contribution(s) alone or by combination of their Contribution(s)
|
||||||
|
with the Work to which such Contribution(s) was submitted. If You
|
||||||
|
institute patent litigation against any entity (including a
|
||||||
|
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||||
|
or a Contribution incorporated within the Work constitutes direct
|
||||||
|
or contributory patent infringement, then any patent licenses
|
||||||
|
granted to You under this License for that Work shall terminate
|
||||||
|
as of the date such litigation is filed.
|
||||||
|
|
||||||
|
4. Redistribution. You may reproduce and distribute copies of the
|
||||||
|
Work or Derivative Works thereof in any medium, with or without
|
||||||
|
modifications, and in Source or Object form, provided that You
|
||||||
|
meet the following conditions:
|
||||||
|
|
||||||
|
(a) You must give any other recipients of the Work or
|
||||||
|
Derivative Works a copy of this License; and
|
||||||
|
|
||||||
|
(b) You must cause any modified files to carry prominent notices
|
||||||
|
stating that You changed the files; and
|
||||||
|
|
||||||
|
(c) You must retain, in the Source form of any Derivative Works
|
||||||
|
that You distribute, all copyright, patent, trademark, and
|
||||||
|
attribution notices from the Source form of the Work,
|
||||||
|
excluding those notices that do not pertain to any part of
|
||||||
|
the Derivative Works; and
|
||||||
|
|
||||||
|
(d) If the Work includes a "NOTICE" text file as part of its
|
||||||
|
distribution, then any Derivative Works that You distribute must
|
||||||
|
include a readable copy of the attribution notices contained
|
||||||
|
within such NOTICE file, excluding those notices that do not
|
||||||
|
pertain to any part of the Derivative Works, in at least one
|
||||||
|
of the following places: within a NOTICE text file distributed
|
||||||
|
as part of the Derivative Works; within the Source form or
|
||||||
|
documentation, if provided along with the Derivative Works; or,
|
||||||
|
within a display generated by the Derivative Works, if and
|
||||||
|
wherever such third-party notices normally appear. The contents
|
||||||
|
of the NOTICE file are for informational purposes only and
|
||||||
|
do not modify the License. You may add Your own attribution
|
||||||
|
notices within Derivative Works that You distribute, alongside
|
||||||
|
or as an addendum to the NOTICE text from the Work, provided
|
||||||
|
that such additional attribution notices cannot be construed
|
||||||
|
as modifying the License.
|
||||||
|
|
||||||
|
You may add Your own copyright statement to Your modifications and
|
||||||
|
may provide additional or different license terms and conditions
|
||||||
|
for use, reproduction, or distribution of Your modifications, or
|
||||||
|
for any such Derivative Works as a whole, provided Your use,
|
||||||
|
reproduction, and distribution of the Work otherwise complies with
|
||||||
|
the conditions stated in this License.
|
||||||
|
|
||||||
|
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||||
|
any Contribution intentionally submitted for inclusion in the Work
|
||||||
|
by You to the Licensor shall be under the terms and conditions of
|
||||||
|
this License, without any additional terms or conditions.
|
||||||
|
Notwithstanding the above, nothing herein shall supersede or modify
|
||||||
|
the terms of any separate license agreement you may have executed
|
||||||
|
with Licensor regarding such Contributions.
|
||||||
|
|
||||||
|
6. Trademarks. This License does not grant permission to use the trade
|
||||||
|
names, trademarks, service marks, or product names of the Licensor,
|
||||||
|
except as required for reasonable and customary use in describing the
|
||||||
|
origin of the Work and reproducing the content of the NOTICE file.
|
||||||
|
|
||||||
|
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||||
|
agreed to in writing, Licensor provides the Work (and each
|
||||||
|
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||||
|
implied, including, without limitation, any warranties or conditions
|
||||||
|
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||||
|
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||||
|
appropriateness of using or redistributing the Work and assume any
|
||||||
|
risks associated with Your exercise of permissions under this License.
|
||||||
|
|
||||||
|
8. Limitation of Liability. In no event and under no legal theory,
|
||||||
|
whether in tort (including negligence), contract, or otherwise,
|
||||||
|
unless required by applicable law (such as deliberate and grossly
|
||||||
|
negligent acts) or agreed to in writing, shall any Contributor be
|
||||||
|
liable to You for damages, including any direct, indirect, special,
|
||||||
|
incidental, or consequential damages of any character arising as a
|
||||||
|
result of this License or out of the use or inability to use the
|
||||||
|
Work (including but not limited to damages for loss of goodwill,
|
||||||
|
work stoppage, computer failure or malfunction, or any and all
|
||||||
|
other commercial damages or losses), even if such Contributor
|
||||||
|
has been advised of the possibility of such damages.
|
||||||
|
|
||||||
|
9. Accepting Warranty or Additional Liability. While redistributing
|
||||||
|
the Work or Derivative Works thereof, You may choose to offer,
|
||||||
|
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||||
|
or other liability obligations and/or rights consistent with this
|
||||||
|
License. However, in accepting such obligations, You may act only
|
||||||
|
on Your own behalf and on Your sole responsibility, not on behalf
|
||||||
|
of any other Contributor, and only if You agree to indemnify,
|
||||||
|
defend, and hold each Contributor harmless for any liability
|
||||||
|
incurred by, or claims asserted against, such Contributor by reason
|
||||||
|
of your accepting any such warranty or additional liability.
|
||||||
|
|
||||||
|
END OF TERMS AND CONDITIONS
|
||||||
|
|
||||||
|
APPENDIX: How to apply the Apache License to your work.
|
||||||
|
|
||||||
|
To apply the Apache License to your work, attach the following
|
||||||
|
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||||
|
replaced with your own identifying information. (Don't include
|
||||||
|
the brackets!) The text should be enclosed in the appropriate
|
||||||
|
comment syntax for the file format. We also recommend that a
|
||||||
|
file or class name and description of purpose be included on the
|
||||||
|
same "printed page" as the copyright notice for easier
|
||||||
|
identification within third-party archives.
|
||||||
|
|
||||||
|
Copyright [yyyy] [name of copyright owner]
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
|
@ -0,0 +1,162 @@
|
||||||
|
dnsdb-query
|
||||||
|
===========
|
||||||
|
|
||||||
|
These clients are reference implementations of the [DNSDB HTTP API](https://api.dnsdb.info/). Output is
|
||||||
|
compliant with the [Passive DNS Common Output Format](http://tools.ietf.org/html/draft-dulaunoy-kaplan-passive-dns-cof-01).
|
||||||
|
|
||||||
|
Please see https://www.dnsdb.info/ for more information.
|
||||||
|
|
||||||
|
Requirements
|
||||||
|
------------
|
||||||
|
* Linux, BSD, OS X
|
||||||
|
* Curl
|
||||||
|
* Python 2.7.x
|
||||||
|
* Farsight DNSDB API key
|
||||||
|
|
||||||
|
Installation
|
||||||
|
------------
|
||||||
|
1. Create a directory
|
||||||
|
|
||||||
|
```
|
||||||
|
mkdir ~/dnsdb
|
||||||
|
```
|
||||||
|
1. Download the software
|
||||||
|
|
||||||
|
```
|
||||||
|
curl https://codeload.github.com/dnsdb/dnsdb-query/tar.gz/debian/0.2-1 -o ~/dnsdb/0.2-1.tar.gz
|
||||||
|
```
|
||||||
|
1. Extract the software
|
||||||
|
|
||||||
|
```
|
||||||
|
tar xzvf ~/dnsdb/0.2-1.tar.gz -C ~/dnsdb/ --strip-components=1
|
||||||
|
```
|
||||||
|
1. Create a API key file
|
||||||
|
|
||||||
|
```
|
||||||
|
nano ~/.dnsdb-query.conf
|
||||||
|
```
|
||||||
|
1. Cut and paste the following and replace '\<apikey\>' with your API Key
|
||||||
|
|
||||||
|
```
|
||||||
|
APIKEY="<apikey>"
|
||||||
|
```
|
||||||
|
1. Test the Python client
|
||||||
|
|
||||||
|
```
|
||||||
|
$ python dnsdb/dnsdb_query.py -i 104.244.13.104
|
||||||
|
```
|
||||||
|
```
|
||||||
|
...
|
||||||
|
www.farsightsecurity.com. IN A 104.244.13.104
|
||||||
|
```
|
||||||
|
|
||||||
|
dnsdb_query.py
|
||||||
|
--------------
|
||||||
|
|
||||||
|
dnsdb_query.py is a Python client for the DNSDB HTTP API. It is similar
|
||||||
|
to the dnsdb-query shell script but supports some additional features
|
||||||
|
like sorting and setting the result limit parameter. It is also embeddable
|
||||||
|
as a Python module.
|
||||||
|
|
||||||
|
```
|
||||||
|
Usage: dnsdb_query.py [options]
|
||||||
|
|
||||||
|
Options:
|
||||||
|
-h, --help show this help message and exit
|
||||||
|
-c CONFIG, --config=CONFIG
|
||||||
|
config file
|
||||||
|
-r RRSET, --rrset=RRSET
|
||||||
|
rrset <ONAME>[/<RRTYPE>[/BAILIWICK]]
|
||||||
|
-n RDATA_NAME, --rdataname=RDATA_NAME
|
||||||
|
rdata name <NAME>[/<RRTYPE>]
|
||||||
|
-i RDATA_IP, --rdataip=RDATA_IP
|
||||||
|
rdata ip <IPADDRESS|IPRANGE|IPNETWORK>
|
||||||
|
-s SORT, --sort=SORT sort key
|
||||||
|
-R, --reverse reverse sort
|
||||||
|
-j, --json output in JSON format
|
||||||
|
-l LIMIT, --limit=LIMIT
|
||||||
|
limit number of results
|
||||||
|
--before=BEFORE only output results seen before this time
|
||||||
|
--after=AFTER only output results seen after this time
|
||||||
|
|
||||||
|
Time formats are: "%Y-%m-%d", "%Y-%m-%d %H:%M:%S", "%d" (UNIX timestamp),
|
||||||
|
"-%d" (Relative time in seconds), BIND format relative timestamp (e.g. 1w1h,
|
||||||
|
(w)eek, (d)ay, (h)our, (m)inute, (s)econd)
|
||||||
|
```
|
||||||
|
|
||||||
|
Or, from Python:
|
||||||
|
|
||||||
|
```
|
||||||
|
from dnsdb_query import DnsdbClient
|
||||||
|
|
||||||
|
server='https://api.dnsdb.info'
|
||||||
|
apikey='d41d8cd98f00b204e9800998ecf8427e'
|
||||||
|
|
||||||
|
client = DnsdbClient(server,apikey)
|
||||||
|
for rrset in client.query_rrset('www.dnsdb.info'):
|
||||||
|
# rrset is a decoded JSON blob
|
||||||
|
print repr(rrset)
|
||||||
|
```
|
||||||
|
|
||||||
|
Other configuration options that may be set:
|
||||||
|
|
||||||
|
`DNSDB_SERVER`
|
||||||
|
The base URL of the DNSDB HTTP API, minus the /lookup component. Defaults to
|
||||||
|
`https://api.dnsdb.info.`
|
||||||
|
|
||||||
|
`HTTP_PROXY`
|
||||||
|
The URL of the HTTP proxy that you wish to use.
|
||||||
|
|
||||||
|
`HTTPS_PROXY`
|
||||||
|
The URL of the HTTPS proxy that you wish to use.
|
||||||
|
|
||||||
|
dnsdb-query
|
||||||
|
-----------
|
||||||
|
|
||||||
|
dnsdb-query is a simple curl-based wrapper for the DNSDB HTTP API.
|
||||||
|
|
||||||
|
The script sources the config file `/etc/dnsdb-query.conf` as a shell fragment.
|
||||||
|
If the config file is not present in `/etc`, the file `$HOME/.dnsdb-query.conf`
|
||||||
|
is sourced instead.
|
||||||
|
|
||||||
|
The config file MUST set the value of the APIKEY shell variable to the API
|
||||||
|
key provided to you by Farsight Security.
|
||||||
|
|
||||||
|
For example, if your API key is d41d8cd98f00b204e9800998ecf8427e, place the
|
||||||
|
following line in `/etc/dnsdb-query.conf` or `$HOME/.dnsdb-query.conf`:
|
||||||
|
|
||||||
|
```
|
||||||
|
APIKEY="d41d8cd98f00b204e9800998ecf8427e"
|
||||||
|
```
|
||||||
|
|
||||||
|
Other shell variables that may be set via the config file or command line
|
||||||
|
are:
|
||||||
|
|
||||||
|
`DNSDB_SERVER`
|
||||||
|
The base URL of the DNSDB HTTP API, minus the /lookup component. Defaults to
|
||||||
|
`https://api.dnsdb.info.`
|
||||||
|
|
||||||
|
`DNSDB_FORMAT`
|
||||||
|
The result format to use, either text or json. Defaults to text.
|
||||||
|
|
||||||
|
`HTTP_PROXY`
|
||||||
|
The URL of the HTTP proxy that you wish to use.
|
||||||
|
|
||||||
|
`HTTPS_PROXY`
|
||||||
|
The URL of the HTTPS proxy that you wish to use.
|
||||||
|
|
||||||
|
dnsdb-query supports the following usages:
|
||||||
|
|
||||||
|
```
|
||||||
|
Usage: dnsdb-query rrset <ONAME>[/<RRTYPE>[/<BAILIWICK>]]
|
||||||
|
Usage: dnsdb-query rdata ip <IPADDRESS>
|
||||||
|
Usage: dnsdb-query rdata name <NAME>[/<RRTYPE>]
|
||||||
|
Usage: dnsdb-query rdata raw <HEX>[/<RRTYPE>]
|
||||||
|
```
|
||||||
|
|
||||||
|
If your rrname, bailiwick or rdata contains the `/` character you
|
||||||
|
will need to escape it to `%2F` on the command line. eg:
|
||||||
|
|
||||||
|
`./dnsdb_query -r 1.0%2F1.0.168.192.in-addr.arpa`
|
||||||
|
|
||||||
|
retrieves the rrsets for `1.0/1.0.168.192.in-addr.arpa`.
|
||||||
|
|
@ -0,0 +1,323 @@
|
||||||
|
#!/usr/bin/env python
|
||||||
|
#
|
||||||
|
# Note: This file is NOT the official one from dnsdb, as it has a python3 cherry-picked pull-request applied for python3 compatibility
|
||||||
|
# See https://github.com/dnsdb/dnsdb-query/pull/30
|
||||||
|
#
|
||||||
|
# Copyright (c) 2013 by Farsight Security, Inc.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
from __future__ import print_function
|
||||||
|
|
||||||
|
import calendar
|
||||||
|
import errno
|
||||||
|
import locale
|
||||||
|
import optparse
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
import json
|
||||||
|
from io import StringIO
|
||||||
|
|
||||||
|
try:
|
||||||
|
from urllib2 import build_opener, Request, ProxyHandler, HTTPError, URLError
|
||||||
|
from urllib import quote as urllib_quote, urlencode
|
||||||
|
except ImportError:
|
||||||
|
from urllib.request import build_opener, Request, ProxyHandler, HTTPError, URLError
|
||||||
|
from urllib.parse import quote as urllib_quote, urlencode
|
||||||
|
|
||||||
|
|
||||||
|
DEFAULT_CONFIG_FILES = filter(os.path.isfile, ('/etc/dnsdb-query.conf', os.path.expanduser('~/.dnsdb-query.conf')))
|
||||||
|
DEFAULT_DNSDB_SERVER = 'https://api.dnsdb.info'
|
||||||
|
DEFAULT_HTTP_PROXY = ''
|
||||||
|
DEFAULT_HTTPS_PROXY = ''
|
||||||
|
|
||||||
|
cfg = None
|
||||||
|
options = None
|
||||||
|
|
||||||
|
locale.setlocale(locale.LC_ALL, '')
|
||||||
|
|
||||||
|
class QueryError(Exception):
|
||||||
|
pass
|
||||||
|
|
||||||
|
class DnsdbClient(object):
|
||||||
|
def __init__(self, server, apikey, limit=None, http_proxy=None, https_proxy=None):
|
||||||
|
self.server = server
|
||||||
|
self.apikey = apikey
|
||||||
|
self.limit = limit
|
||||||
|
self.http_proxy = http_proxy
|
||||||
|
self.https_proxy = https_proxy
|
||||||
|
|
||||||
|
def query_rrset(self, oname, rrtype=None, bailiwick=None, before=None, after=None):
|
||||||
|
if bailiwick:
|
||||||
|
if not rrtype:
|
||||||
|
rrtype = 'ANY'
|
||||||
|
path = 'rrset/name/%s/%s/%s' % (quote(oname), rrtype, quote(bailiwick))
|
||||||
|
elif rrtype:
|
||||||
|
path = 'rrset/name/%s/%s' % (quote(oname), rrtype)
|
||||||
|
else:
|
||||||
|
path = 'rrset/name/%s' % quote(oname)
|
||||||
|
return self._query(path, before, after)
|
||||||
|
|
||||||
|
def query_rdata_name(self, rdata_name, rrtype=None, before=None, after=None):
|
||||||
|
if rrtype:
|
||||||
|
path = 'rdata/name/%s/%s' % (quote(rdata_name), rrtype)
|
||||||
|
else:
|
||||||
|
path = 'rdata/name/%s' % quote(rdata_name)
|
||||||
|
return self._query(path, before, after)
|
||||||
|
|
||||||
|
def query_rdata_ip(self, rdata_ip, before=None, after=None):
|
||||||
|
path = 'rdata/ip/%s' % rdata_ip.replace('/', ',')
|
||||||
|
return self._query(path, before, after)
|
||||||
|
|
||||||
|
def _query(self, path, before=None, after=None):
|
||||||
|
res = []
|
||||||
|
url = '%s/lookup/%s' % (self.server, path)
|
||||||
|
|
||||||
|
params = {}
|
||||||
|
if self.limit:
|
||||||
|
params['limit'] = self.limit
|
||||||
|
if before and after:
|
||||||
|
params['time_first_after'] = after
|
||||||
|
params['time_last_before'] = before
|
||||||
|
else:
|
||||||
|
if before:
|
||||||
|
params['time_first_before'] = before
|
||||||
|
if after:
|
||||||
|
params['time_last_after'] = after
|
||||||
|
if params:
|
||||||
|
url += '?{0}'.format(urlencode(params))
|
||||||
|
|
||||||
|
req = Request(url)
|
||||||
|
req.add_header('Accept', 'application/json')
|
||||||
|
req.add_header('X-Api-Key', self.apikey)
|
||||||
|
|
||||||
|
proxy_args = {}
|
||||||
|
if self.http_proxy:
|
||||||
|
proxy_args['http'] = self.http_proxy
|
||||||
|
if self.https_proxy:
|
||||||
|
proxy_args['https'] = self.https_proxy
|
||||||
|
proxy_handler = ProxyHandler(proxy_args)
|
||||||
|
opener = build_opener(proxy_handler)
|
||||||
|
|
||||||
|
try:
|
||||||
|
http = opener.open(req)
|
||||||
|
while True:
|
||||||
|
line = http.readline()
|
||||||
|
if not line:
|
||||||
|
break
|
||||||
|
yield json.loads(line.decode('ascii'))
|
||||||
|
except (HTTPError, URLError) as e:
|
||||||
|
raise QueryError(str(e), sys.exc_traceback)
|
||||||
|
|
||||||
|
def quote(path):
|
||||||
|
return urllib_quote(path, safe='')
|
||||||
|
|
||||||
|
def sec_to_text(ts):
|
||||||
|
return time.strftime('%Y-%m-%d %H:%M:%S -0000', time.gmtime(ts))
|
||||||
|
|
||||||
|
def rrset_to_text(m):
|
||||||
|
s = StringIO()
|
||||||
|
|
||||||
|
try:
|
||||||
|
if 'bailiwick' in m:
|
||||||
|
s.write(';; bailiwick: %s\n' % m['bailiwick'])
|
||||||
|
|
||||||
|
if 'count' in m:
|
||||||
|
s.write(';; count: %s\n' % locale.format('%d', m['count'], True))
|
||||||
|
|
||||||
|
if 'time_first' in m:
|
||||||
|
s.write(';; first seen: %s\n' % sec_to_text(m['time_first']))
|
||||||
|
if 'time_last' in m:
|
||||||
|
s.write(';; last seen: %s\n' % sec_to_text(m['time_last']))
|
||||||
|
|
||||||
|
if 'zone_time_first' in m:
|
||||||
|
s.write(';; first seen in zone file: %s\n' % sec_to_text(m['zone_time_first']))
|
||||||
|
if 'zone_time_last' in m:
|
||||||
|
s.write(';; last seen in zone file: %s\n' % sec_to_text(m['zone_time_last']))
|
||||||
|
|
||||||
|
if 'rdata' in m:
|
||||||
|
for rdata in m['rdata']:
|
||||||
|
s.write('%s IN %s %s\n' % (m['rrname'], m['rrtype'], rdata))
|
||||||
|
|
||||||
|
s.seek(0)
|
||||||
|
return s.read()
|
||||||
|
finally:
|
||||||
|
s.close()
|
||||||
|
|
||||||
|
def rdata_to_text(m):
|
||||||
|
return '%s IN %s %s' % (m['rrname'], m['rrtype'], m['rdata'])
|
||||||
|
|
||||||
|
def parse_config(cfg_files):
|
||||||
|
config = {}
|
||||||
|
|
||||||
|
if not cfg_files:
|
||||||
|
raise IOError(errno.ENOENT, 'dnsdb_query: No config files found')
|
||||||
|
|
||||||
|
for fname in cfg_files:
|
||||||
|
for line in open(fname):
|
||||||
|
key, eq, val = line.strip().partition('=')
|
||||||
|
val = val.strip('"')
|
||||||
|
config[key] = val
|
||||||
|
|
||||||
|
return config
|
||||||
|
|
||||||
|
def time_parse(s):
|
||||||
|
try:
|
||||||
|
epoch = int(s)
|
||||||
|
return epoch
|
||||||
|
except ValueError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
try:
|
||||||
|
epoch = int(calendar.timegm(time.strptime(s, '%Y-%m-%d')))
|
||||||
|
return epoch
|
||||||
|
except ValueError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
try:
|
||||||
|
epoch = int(calendar.timegm(time.strptime(s, '%Y-%m-%d %H:%M:%S')))
|
||||||
|
return epoch
|
||||||
|
except ValueError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
m = re.match(r'^(?=\d)(?:(\d+)w)?(?:(\d+)d)?(?:(\d+)h)?(?:(\d+)m)?(?:(\d+)s?)?$', s, re.I)
|
||||||
|
if m:
|
||||||
|
return -1*(int(m.group(1) or 0)*604800 +
|
||||||
|
int(m.group(2) or 0)*86400+
|
||||||
|
int(m.group(3) or 0)*3600+
|
||||||
|
int(m.group(4) or 0)*60+
|
||||||
|
int(m.group(5) or 0))
|
||||||
|
|
||||||
|
raise ValueError('Invalid time: "%s"' % s)
|
||||||
|
|
||||||
|
def epipe_wrapper(func):
|
||||||
|
def f(*args, **kwargs):
|
||||||
|
try:
|
||||||
|
return func(*args, **kwargs)
|
||||||
|
except IOError as e:
|
||||||
|
if e.errno == errno.EPIPE:
|
||||||
|
sys.exit(e.errno)
|
||||||
|
raise
|
||||||
|
return f
|
||||||
|
|
||||||
|
@epipe_wrapper
|
||||||
|
def main():
|
||||||
|
global cfg
|
||||||
|
global options
|
||||||
|
|
||||||
|
parser = optparse.OptionParser(epilog='Time formats are: "%Y-%m-%d", "%Y-%m-%d %H:%M:%S", "%d" (UNIX timestamp), "-%d" (Relative time in seconds), BIND format (e.g. 1w1h, (w)eek, (d)ay, (h)our, (m)inute, (s)econd)')
|
||||||
|
parser.add_option('-c', '--config', dest='config',
|
||||||
|
help='config file', action='append')
|
||||||
|
parser.add_option('-r', '--rrset', dest='rrset', type='string',
|
||||||
|
help='rrset <ONAME>[/<RRTYPE>[/BAILIWICK]]')
|
||||||
|
parser.add_option('-n', '--rdataname', dest='rdata_name', type='string',
|
||||||
|
help='rdata name <NAME>[/<RRTYPE>]')
|
||||||
|
parser.add_option('-i', '--rdataip', dest='rdata_ip', type='string',
|
||||||
|
help='rdata ip <IPADDRESS|IPRANGE|IPNETWORK>')
|
||||||
|
parser.add_option('-t', '--rrtype', dest='rrtype', type='string',
|
||||||
|
help='rrset or rdata rrtype')
|
||||||
|
parser.add_option('-b', '--bailiwick', dest='bailiwick', type='string',
|
||||||
|
help='rrset bailiwick')
|
||||||
|
parser.add_option('-s', '--sort', dest='sort', type='string', help='sort key')
|
||||||
|
parser.add_option('-R', '--reverse', dest='reverse', action='store_true', default=False,
|
||||||
|
help='reverse sort')
|
||||||
|
parser.add_option('-j', '--json', dest='json', action='store_true', default=False,
|
||||||
|
help='output in JSON format')
|
||||||
|
parser.add_option('-l', '--limit', dest='limit', type='int', default=0,
|
||||||
|
help='limit number of results')
|
||||||
|
|
||||||
|
parser.add_option('', '--before', dest='before', type='string', help='only output results seen before this time')
|
||||||
|
parser.add_option('', '--after', dest='after', type='string', help='only output results seen after this time')
|
||||||
|
|
||||||
|
options, args = parser.parse_args()
|
||||||
|
if args:
|
||||||
|
parser.print_help()
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
try:
|
||||||
|
if options.before:
|
||||||
|
options.before = time_parse(options.before)
|
||||||
|
except ValueError:
|
||||||
|
print('Could not parse before: {}'.format(options.before))
|
||||||
|
|
||||||
|
try:
|
||||||
|
if options.after:
|
||||||
|
options.after = time_parse(options.after)
|
||||||
|
except ValueError:
|
||||||
|
print('Could not parse after: {}'.format(options.after))
|
||||||
|
|
||||||
|
try:
|
||||||
|
cfg = parse_config(options.config or DEFAULT_CONFIG_FILES)
|
||||||
|
except IOError as e:
|
||||||
|
print(str(e), file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
if not 'DNSDB_SERVER' in cfg:
|
||||||
|
cfg['DNSDB_SERVER'] = DEFAULT_DNSDB_SERVER
|
||||||
|
if not 'HTTP_PROXY' in cfg:
|
||||||
|
cfg['HTTP_PROXY'] = DEFAULT_HTTP_PROXY
|
||||||
|
if not 'HTTPS_PROXY' in cfg:
|
||||||
|
cfg['HTTPS_PROXY'] = DEFAULT_HTTPS_PROXY
|
||||||
|
if not 'APIKEY' in cfg:
|
||||||
|
sys.stderr.write('dnsdb_query: APIKEY not defined in config file\n')
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
client = DnsdbClient(cfg['DNSDB_SERVER'], cfg['APIKEY'],
|
||||||
|
limit=options.limit,
|
||||||
|
http_proxy=cfg['HTTP_PROXY'],
|
||||||
|
https_proxy=cfg['HTTPS_PROXY'])
|
||||||
|
if options.rrset:
|
||||||
|
if options.rrtype or options.bailiwick:
|
||||||
|
qargs = (options.rrset, options.rrtype, options.bailiwick)
|
||||||
|
else:
|
||||||
|
qargs = (options.rrset.split('/', 2))
|
||||||
|
|
||||||
|
results = client.query_rrset(*qargs, before=options.before, after=options.after)
|
||||||
|
fmt_func = rrset_to_text
|
||||||
|
elif options.rdata_name:
|
||||||
|
if options.rrtype:
|
||||||
|
qargs = (options.rdata_name, options.rrtype)
|
||||||
|
else:
|
||||||
|
qargs = (options.rdata_name.split('/', 1))
|
||||||
|
|
||||||
|
results = client.query_rdata_name(*qargs, before=options.before, after=options.after)
|
||||||
|
fmt_func = rdata_to_text
|
||||||
|
elif options.rdata_ip:
|
||||||
|
results = client.query_rdata_ip(options.rdata_ip, before=options.before, after=options.after)
|
||||||
|
fmt_func = rdata_to_text
|
||||||
|
else:
|
||||||
|
parser.print_help()
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
if options.json:
|
||||||
|
fmt_func = json.dumps
|
||||||
|
|
||||||
|
try:
|
||||||
|
if options.sort:
|
||||||
|
results = list(results)
|
||||||
|
if len(results) > 0:
|
||||||
|
if not options.sort in results[0]:
|
||||||
|
sort_keys = results[0].keys()
|
||||||
|
sort_keys.sort()
|
||||||
|
sys.stderr.write('dnsdb_query: invalid sort key "%s". valid sort keys are %s\n' % (options.sort, ', '.join(sort_keys)))
|
||||||
|
sys.exit(1)
|
||||||
|
results.sort(key=lambda r: r[options.sort], reverse=options.reverse)
|
||||||
|
for res in results:
|
||||||
|
sys.stdout.write('%s\n' % fmt_func(res))
|
||||||
|
except QueryError as e:
|
||||||
|
print(e.message, file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
|
|
@ -0,0 +1,72 @@
|
||||||
|
import json
|
||||||
|
from ._dnsdb_query.dnsdb_query import DnsdbClient
|
||||||
|
|
||||||
|
|
||||||
|
misperrors = {'error': 'Error'}
|
||||||
|
mispattributes = {'input': ['hostname', 'domain', 'ip-src', 'ip-dst'], 'output': ['freetext']}
|
||||||
|
moduleinfo = {'version': '0.1', 'author': 'Christophe Vandeplas', 'description': 'Module to access Farsight DNSDB Passive DNS', 'module-type': ['expansion', 'hover']}
|
||||||
|
moduleconfig = ['apikey']
|
||||||
|
|
||||||
|
server = 'https://api.dnsdb.info'
|
||||||
|
|
||||||
|
# TODO return a MISP object with the different attributes
|
||||||
|
|
||||||
|
|
||||||
|
def handler(q=False):
|
||||||
|
if q is False:
|
||||||
|
return False
|
||||||
|
request = json.loads(q)
|
||||||
|
if (request.get('config')):
|
||||||
|
if (request['config'].get('apikey') is None):
|
||||||
|
misperrors['error'] = 'Farsight DNSDB apikey is missing'
|
||||||
|
return misperrors
|
||||||
|
client = DnsdbClient(server, request['config']['apikey'])
|
||||||
|
if request.get('hostname'):
|
||||||
|
res = lookup_name(client, request['hostname'])
|
||||||
|
elif request.get('domain'):
|
||||||
|
res = lookup_name(client, request['domain'])
|
||||||
|
elif request.get('ip-src'):
|
||||||
|
res = lookup_ip(client, request['ip-src'])
|
||||||
|
elif request.get('ip-dst'):
|
||||||
|
res = lookup_ip(client, request['ip-dst'])
|
||||||
|
else:
|
||||||
|
misperrors['error'] = "Unsupported attributes type"
|
||||||
|
return misperrors
|
||||||
|
|
||||||
|
out = ''
|
||||||
|
for v in set(res): # uniquify entries
|
||||||
|
out = out + "{} ".format(v)
|
||||||
|
r = {'results': [{'types': mispattributes['output'], 'values': out}]}
|
||||||
|
return r
|
||||||
|
|
||||||
|
|
||||||
|
def lookup_name(client, name):
|
||||||
|
res = client.query_rrset(name) # RRSET = entries in the left-hand side of the domain name related labels
|
||||||
|
for item in res:
|
||||||
|
if item.get('rrtype') in ['A', 'AAAA', 'CNAME']:
|
||||||
|
for i in item.get('rdata'):
|
||||||
|
yield(i.rstrip('.'))
|
||||||
|
if item.get('rrtype') in ['SOA']:
|
||||||
|
for i in item.get('rdata'):
|
||||||
|
# grab email field and replace first dot by @ to convert to an email address
|
||||||
|
yield(i.split(' ')[1].rstrip('.').replace('.', '@', 1))
|
||||||
|
res = client.query_rdata_name(name) # RDATA = entries on the right-hand side of the domain name related labels
|
||||||
|
for item in res:
|
||||||
|
if item.get('rrtype') in ['A', 'AAAA', 'CNAME']:
|
||||||
|
yield(item.get('rrname').rstrip('.'))
|
||||||
|
|
||||||
|
|
||||||
|
def lookup_ip(client, ip):
|
||||||
|
res = client.query_rdata_ip(ip)
|
||||||
|
for item in res:
|
||||||
|
print(item)
|
||||||
|
yield(item['rrname'].rstrip('.'))
|
||||||
|
|
||||||
|
|
||||||
|
def introspection():
|
||||||
|
return mispattributes
|
||||||
|
|
||||||
|
|
||||||
|
def version():
|
||||||
|
moduleinfo['config'] = moduleconfig
|
||||||
|
return moduleinfo
|
||||||
Loading…
Reference in New Issue