Chg generator-publicdns: work with new CSV format
1. The CSV format has changed with the update on 2020-07-14. 2. The script also generates IPv4, IPv6, and the hostname lists at once. 3. Downloaded file added to .gitignorepull/154/head
Kevin Holvoet
parent
cb52a472e6
commit
d32eb23a58
|
|
@ -15,4 +15,5 @@ ocsp_ocsp-ipv6.txt.txt
|
|||
PublicAllIntermediateCertsWithPEMCSV.csv
|
||||
top500.domains.csv
|
||||
top500.pages.csv
|
||||
top-1m.csv.zip
|
||||
top-1m.csv.zip
|
||||
public-dns-nameservers.csv
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
|
|
@ -1,279 +1,31 @@
|
|||
{
|
||||
"description": "Event contains one or more public IPv6 DNS resolvers as attribute with an IDS flag set",
|
||||
"list": [
|
||||
"2001:1488:800:400::130",
|
||||
"2001:14b8:100:350::2",
|
||||
"2001:14b8:100:8350::1",
|
||||
"2001:14b8:100:8350::8",
|
||||
"2001:1608:10:167:342::eb52",
|
||||
"2001:1608:10:195:3:dead:beef:cafe",
|
||||
"2001:1608:10:25::1c04:b12f",
|
||||
"2001:1608:10:25::9249:d69b",
|
||||
"2001:1620:2777:1::10",
|
||||
"2001:1620:2777:1::11",
|
||||
"2001:1620:2777::2",
|
||||
"2001:19f0:5001:133:5400:ff:fe30:d565",
|
||||
"2001:19f0:5801:11:5400:ff:fe2d:7724",
|
||||
"2001:19f0:7001:929:5400:ff:fe30:50af",
|
||||
"2001:19f0:8001:5e:5400:ff:fe35:c3ae",
|
||||
"2001:1a68::d911:2244",
|
||||
"2001:1bc0::ffff:aaaa:2",
|
||||
"2001:1bc0::ffff:bbbb:2",
|
||||
"2001:2040:39::5",
|
||||
"2001:418:3ff::1:53",
|
||||
"2001:418:3ff::53",
|
||||
"2001:41d0:52:cff::1325",
|
||||
"2001:41d0:52:f00::413",
|
||||
"2001:41d0:8:be92::1",
|
||||
"2001:41d0:a:1011::1",
|
||||
"2001:41d0:a:28::1",
|
||||
"2001:41d0:203:4b1f:2:2:2:2",
|
||||
"2001:428:101:100:205:171:2:65",
|
||||
"2001:428:101:100:205:171:3:65",
|
||||
"2001:428::1",
|
||||
"2001:450:2005:1::4",
|
||||
"2001:450:2005:2::4",
|
||||
"2001:450:2005:2::5",
|
||||
"2001:450:2005:3::5",
|
||||
"2001:468:c80:2101:0:100:0:22",
|
||||
"2001:468:c80:4101:0:100:0:42",
|
||||
"2001:470:0:45::2",
|
||||
"2001:470:0:69::2",
|
||||
"2001:470:0:6e::2",
|
||||
"2001:470:0:78::2",
|
||||
"2001:470:0:7d::2",
|
||||
"2001:470:0:8c::2",
|
||||
"2001:470:0:90::2",
|
||||
"2001:470:0:9d::2",
|
||||
"2001:470:0:c0::2",
|
||||
"2001:470:1f14:fd8::2",
|
||||
"2001:470:20::2",
|
||||
"2001:470:26:1c8::1",
|
||||
"2001:470:6d:80:224:1dff:fe84:797",
|
||||
"2001:470:6d:80:a5f6:5a97:a53:71cb",
|
||||
"2001:470:6d:80:c4f1:32a:4521:c34",
|
||||
"2001:470:6d:f1e:1337:360:dead:beef",
|
||||
"2001:470:8e08::",
|
||||
"2001:470:d:bb7::8888",
|
||||
"2001:470:f032:1::254",
|
||||
"2001:4860:4860::8844",
|
||||
"2001:4860:4860::8888",
|
||||
"2001:4870:6082:3::100",
|
||||
"2001:4870:6082:3::5",
|
||||
"2001:4870:8000:3::100",
|
||||
"2001:4870:8000:3::5",
|
||||
"2001:4ce8::53",
|
||||
"2001:4dd0:fd5e::53",
|
||||
"2001:4f8:0:2::14",
|
||||
"2001:550:1:1::d",
|
||||
"2001:5b8:1::5",
|
||||
"2001:610:1108:5010::130",
|
||||
"2001:638:902:1::10",
|
||||
"2001:648:2ffc:100::211",
|
||||
"2001:678:1::206",
|
||||
"2001:67c:15e8:d1::18",
|
||||
"2001:67c:15e8:d1::19",
|
||||
"2001:67c:240c:214::4",
|
||||
"2001:67c:240c:214::5",
|
||||
"2001:470:1f1a:78e::2",
|
||||
"2001:470:2351::1",
|
||||
"2001:4b8:2:101::602",
|
||||
"2001:4b8:3:201::902",
|
||||
"2001:67c:28a4::",
|
||||
"2001:67c:2b0::1",
|
||||
"2001:67c:2b0::2",
|
||||
"2001:67c:2b24:1000::10",
|
||||
"2001:67c:2b24:1000::11",
|
||||
"2001:6b0:3f::a",
|
||||
"2001:7b8:1509::1",
|
||||
"2001:840:0:200::1",
|
||||
"2001:840:200::",
|
||||
"2001:840:2010:413::100",
|
||||
"2001:910:800::12",
|
||||
"2001:910:800::40",
|
||||
"2001:978:1:1::d",
|
||||
"2001:978:1:2::d",
|
||||
"2001:b000:168::1",
|
||||
"2001:b08:2:280::4:1",
|
||||
"2001:bf0::2",
|
||||
"2001:ec0:1::1",
|
||||
"2001:ec0:3::3",
|
||||
"2400:6180:0:d0::38:d001",
|
||||
"2400:8900::f03c:91ff:fe70:c452",
|
||||
"2402:2f80:5::",
|
||||
"2402:9e80:1::1:e554",
|
||||
"2403:5680::1:200f",
|
||||
"2407:9000:0:4::2",
|
||||
"2600:3c00::20:b1ff",
|
||||
"2600:3c02::f03c:91ff:fe84:cb54",
|
||||
"2600:3c02::f03c:91ff:fee0:5e5",
|
||||
"2600::1",
|
||||
"2600::2",
|
||||
"2602:3f:e75c:1bff::1",
|
||||
"2602:ffb6:2:0:f816:3eff:fe23:ae28",
|
||||
"2602:ffc5:30::1:d69b",
|
||||
"2604:a880:1:20::c5b:1001",
|
||||
"2604:a880:400:d0::6d6:2001",
|
||||
"2605:f700:c0:1::1089:53ef",
|
||||
"2606:4700:4700::1001",
|
||||
"2606:4700:4700::1111",
|
||||
"2607:fa88:1::2",
|
||||
"2610:130:100:3::200",
|
||||
"2610:a1:1018::22",
|
||||
"2610:a1:1018::23",
|
||||
"2610:a1:1018::24",
|
||||
"2610:a1:1018::25",
|
||||
"2610:a1:1018::26",
|
||||
"2610:a1:1018::27",
|
||||
"2610:a1:1018::28",
|
||||
"2610:a1:1018::29",
|
||||
"2610:a1:1018::30",
|
||||
"2610:a1:1018::31",
|
||||
"2610:a1:1018::32",
|
||||
"2610:a1:1018::33",
|
||||
"2001:de4::102",
|
||||
"2606:ed00:2:babe::10",
|
||||
"2607:5300:203:1797::53",
|
||||
"2607:f130:0:d7::d41",
|
||||
"2610:a1:1018::34",
|
||||
"2610:a1:1018::35",
|
||||
"2610:a1:1018::5",
|
||||
"2610:a1:1019::22",
|
||||
"2610:a1:1019::23",
|
||||
"2610:a1:1019::24",
|
||||
"2610:a1:1019::25",
|
||||
"2610:a1:1019::26",
|
||||
"2610:a1:1019::27",
|
||||
"2610:a1:1019::28",
|
||||
"2610:a1:1019::29",
|
||||
"2610:a1:1019::30",
|
||||
"2610:a1:1019::31",
|
||||
"2610:a1:1019::32",
|
||||
"2610:a1:1019::33",
|
||||
"2610:a1:1019::34",
|
||||
"2610:a1:1019::35",
|
||||
"2610:a1:1019::5",
|
||||
"2620:0:ccc::2",
|
||||
"2620:0:ccd::2",
|
||||
"2620:74:1b::1:1",
|
||||
"2620:74:1c::2:2",
|
||||
"2a00-1508-0-4--9.puntcat.ip6.guifi.net.",
|
||||
"2a00-1dc0-cafe--ad86-fa7e.static.host.",
|
||||
"2a00-1dc0-cafe--c6af-c19d.static.host.",
|
||||
"2a00:12d8:7002::2",
|
||||
"2a00:1508:0:4::9",
|
||||
"2a00:1ca8:a7::1e9",
|
||||
"2a00:1dc0:cafe::ad86:fa7e",
|
||||
"2a00:1dc0:cafe::c6af:c19d",
|
||||
"2a00:5881:8100:1000::3",
|
||||
"2a00:5884:8218::1",
|
||||
"2a00:dcc0:eda:88:245:71:858e:a15",
|
||||
"2a00:dcc0:eda:98:183:193:d85a:389b",
|
||||
"2a00:dcc7:2202:11::7b28",
|
||||
"2a00:dcc7:2202:14::2",
|
||||
"2a00:f48:100c:7b::2",
|
||||
"2a00:f48:100c:7e::2",
|
||||
"2a01:4f8:131:1278::2",
|
||||
"2a01:4f8:141:4281::3000",
|
||||
"2a01:4f8:151:90e9::2",
|
||||
"2a01:4f8:151:90e9::b",
|
||||
"2a01:4f8:161:4109::6",
|
||||
"2a01:4f8:191:306c::2",
|
||||
"2a02:2178:1:2::2",
|
||||
"2a02:2ca0:64:22::2",
|
||||
"2a02:6b8::feed:ff",
|
||||
"2a02:7aa0:1201::f60e:2719",
|
||||
"2a02:7aa0:1619::4f50:a69",
|
||||
"2a02:940:0:4293::100",
|
||||
"2a02:e00:fffd:139::9",
|
||||
"2a03:b0c0:0:1010::62:f001",
|
||||
"2a03:b0c0:3:d0::7c:5001",
|
||||
"2a04:92c7:7:7::14ae:460a",
|
||||
"2a04:9dc0:c1:7::cb9:f785",
|
||||
"2a05:b0c6:5e4::53",
|
||||
"2a05:dfc7:5::53",
|
||||
"2a05:dfc7:5::5353",
|
||||
"2c0f:fda8:5::2ed1:d2ec",
|
||||
"::ffff:9538:1aed",
|
||||
"::ffff:9e45:abfe",
|
||||
"ClemenTroniQ89-1-pt.tunnel.tserv11.ams1.ipv6.he.net.",
|
||||
"anyone.dnsrec.meo.ws.",
|
||||
"anytwo.dnsrec.meo.ws.",
|
||||
"b-root.cesidian.info.",
|
||||
"canopus.ne2000.nl.",
|
||||
"cl-849.hel-01.fi.sixxs.net.",
|
||||
"crt-public-dns-a.cesidianroot.eu.",
|
||||
"cznic-public-dns-1.nic.cz.",
|
||||
"dlfw-rdns-01.dlfw.twtelecom.net.",
|
||||
"dns.yandex.ru.",
|
||||
"dns01.jordbruksverket.se.",
|
||||
"dns02.jordbruksverket.se.",
|
||||
"dns1.host.net.",
|
||||
"dns1.lon.gblx.net.",
|
||||
"dns1.phx.gblx.net.",
|
||||
"dns1.totbb.net.",
|
||||
"dns2.phx.gblx.net.",
|
||||
"dns2.roc.gblx.net.",
|
||||
"dns2.totbb.net.",
|
||||
"dnsdist.mysrvr.net.",
|
||||
"dnsres1.nic.cz.",
|
||||
"eu-res1.dns.cogentco.com.",
|
||||
"eu-res2.dns.cogentco.com.",
|
||||
"freya.stelas.de.",
|
||||
"google-public-dns-b.google.com.",
|
||||
"hntp1.hinet.net.",
|
||||
"homens.b-hs.de.",
|
||||
"host19-65-static.59-88-b.business.telecomitalia.it.",
|
||||
"jeru.cns.ipv6.vt.edu.",
|
||||
"log.bzh.",
|
||||
"lpc1.stu.neva.ru.",
|
||||
"lucy.s.imvry.pw.",
|
||||
"mail2.cesidianroot.eu.",
|
||||
"na-res1.dns.cogentco.com.",
|
||||
"ns-3.iastate.edu.",
|
||||
"ns.ipv6.uni-leipzig.de.",
|
||||
"ns0.fdn.org.",
|
||||
"ns1.ams.dns.lchi.mp.",
|
||||
"ns1.ata.dns.lchi.mp.",
|
||||
"ns1.fdn.fr.",
|
||||
"ns1.hnd.dns.lchi.mp.",
|
||||
"ns1.init7.net.",
|
||||
"ns1.nl.dns.d0wn.biz.",
|
||||
"ns1.probe-networks.de.",
|
||||
"ns1.sea.dns.lchi.mp.",
|
||||
"ns1.sg.dns.d0wn.biz.",
|
||||
"ns1.shodan.io.",
|
||||
"ns1.syd.dns.lchi.mp.",
|
||||
"ns1.twtelecom.net.",
|
||||
"ns10.init7.net.",
|
||||
"ns11.init7.net.",
|
||||
"ns2.all.de.",
|
||||
"ns2.itandtel.at.",
|
||||
"ns2.powertech.no.",
|
||||
"ns2.probe-networks.de.",
|
||||
"ns2.shodan.io.",
|
||||
"ns2.sprintlink.net.",
|
||||
"ns2.twtelecom.net.",
|
||||
"ns532549.ip-149-56-26.net.",
|
||||
"open-root.cesidian.info.",
|
||||
"or.isc.org.",
|
||||
"ordns.he.net.",
|
||||
"plfgr.eu.org.",
|
||||
"primary.server.edv-froehlich.de.",
|
||||
"proxyvm.stejau.de.",
|
||||
"public-dns-a.primawebtools.de.",
|
||||
"recursif.arn-fai.net.",
|
||||
"resolver.qwest.net.",
|
||||
"resolver1.dns.trex.fi.",
|
||||
"resolver1.ipv6-sandbox.opendns.com.",
|
||||
"resolver2.dns.trex.fi.",
|
||||
"resolver2.ipv6-sandbox.opendns.com.",
|
||||
"rickhunter.ns.ielo.net.",
|
||||
"rns1.grnet.gr.",
|
||||
"secondary.server.edv-froehlich.de.",
|
||||
"services.donotuse.de.",
|
||||
"spcr-2.machadosbsmarketing.com.br.",
|
||||
"test.cesidian.info.",
|
||||
"tserv1.ams1.he.net.",
|
||||
"tserv1.dal1.he.net.",
|
||||
"tserv1.fmt2.he.net.",
|
||||
"tserv1.fra1.he.net.",
|
||||
"tserv1.mia1.he.net.",
|
||||
"tserv1.tor1.he.net.",
|
||||
"x.ns.gin.ntt.net.",
|
||||
"y.ns.gin.ntt.net.",
|
||||
"yardbird.cns.ipv6.vt.edu.",
|
||||
"zen.stack.nl."
|
||||
"2a00:aa40:0:225::2",
|
||||
"2a01:238:42f6:ac00:2a29:4f7f:b6d:ef46",
|
||||
"2a01:3a0:53:53::",
|
||||
"2a01:4f8:141:316d::117",
|
||||
"2a01:4f8:151:34aa::198",
|
||||
"2a01:4f8:c0c:1e44::1",
|
||||
"2a01:4f8:c17:739a::2",
|
||||
"2a02:2970:1002::18",
|
||||
"2a02:c205:3001:4558::1",
|
||||
"2a03:8600:1001::2"
|
||||
],
|
||||
"matching_attributes": [
|
||||
"ip-src",
|
||||
|
|
@ -282,5 +34,5 @@
|
|||
],
|
||||
"name": "List of known IPv6 public DNS resolvers",
|
||||
"type": "string",
|
||||
"version": 20181114
|
||||
"version": 20200720
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,69 +1,77 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import csv
|
||||
import datetime
|
||||
import logging
|
||||
import ipaddress
|
||||
import json
|
||||
import os
|
||||
import requests
|
||||
import logging
|
||||
|
||||
servers_url = 'http://public-dns.info/nameservers.csv'
|
||||
csv_path = 'nameservers.csv'
|
||||
dns4_path = 'list4.json'
|
||||
dns6_path = 'list6.json'
|
||||
|
||||
if os.path.isfile(csv_path):
|
||||
logging.warning('Not erasing local csv file')
|
||||
else:
|
||||
req = requests.get(servers_url)
|
||||
with open(csv_path, 'wb') as fd:
|
||||
for chunk in req.iter_content(4096):
|
||||
fd.write(chunk)
|
||||
|
||||
ip4_list = []
|
||||
ip6_list = []
|
||||
with open(csv_path) as csv_file:
|
||||
servers_list = csv.reader(csv_file, delimiter=',', quotechar='"')
|
||||
|
||||
for row in servers_list:
|
||||
if row[5] == '':
|
||||
try:
|
||||
ip = ipaddress.ip_address(row[0])
|
||||
|
||||
if ip.version == 4:
|
||||
list = ip4_list
|
||||
else:
|
||||
list = ip6_list
|
||||
|
||||
list.append(ip.compressed)
|
||||
if len(row[1]) > 0 and row[1] != '.':
|
||||
list.append(row[1])
|
||||
|
||||
except ValueError as exc:
|
||||
logging.warning(str(exc))
|
||||
|
||||
version = int(datetime.date.today().strftime('%Y%m%d'))
|
||||
|
||||
out4_list = {}
|
||||
out4_list['name'] = 'List of known IPv4 public DNS resolvers'
|
||||
out4_list['version'] = version
|
||||
out4_list['description'] = 'Event contains one or more public IPv4 DNS resolvers as attribute with an IDS flag set'
|
||||
out4_list['matching_attributes'] = ['ip-src', 'ip-dst', 'domain|ip']
|
||||
out4_list['list'] = sorted(set(ip4_list))
|
||||
from generator import download_to_file, get_abspath_list_file, get_version
|
||||
|
||||
|
||||
out6_list = {}
|
||||
out6_list['name'] = 'List of known IPv6 public DNS resolvers'
|
||||
out6_list['version'] = version
|
||||
out6_list['description'] = 'Event contains one or more public IPv6 DNS resolvers as attribute with an IDS flag set'
|
||||
out6_list['matching_attributes'] = ['ip-src', 'ip-dst', 'domain|ip']
|
||||
out6_list['list'] = sorted(set(ip6_list))
|
||||
def process(file, warninglist, dst, type='v4'):
|
||||
|
||||
with open(file) as csv_file:
|
||||
servers_list = csv.reader(csv_file, delimiter=',', quotechar='"')
|
||||
|
||||
data_list = []
|
||||
for row in servers_list:
|
||||
if row[7] in (None, ""):
|
||||
try:
|
||||
ip = ipaddress.ip_address(row[0])
|
||||
|
||||
if type == 'v4' and ip.version == 4:
|
||||
data_list.append(ip.compressed)
|
||||
elif type == 'v6' and ip.version == 6:
|
||||
data_list.append(ip.compressed)
|
||||
elif type == 'hostname' and row[1] not in (None, "", '.'):
|
||||
data_list.append(row[1])
|
||||
except ValueError as exc:
|
||||
logging.warning(str(exc))
|
||||
|
||||
warninglist['version'] = get_version()
|
||||
warninglist['list'] = sorted(set(data_list))
|
||||
|
||||
with open(get_abspath_list_file(dst), 'w') as data_file:
|
||||
json.dump(warninglist, data_file, indent=2, sort_keys=True)
|
||||
data_file.write("\n")
|
||||
|
||||
|
||||
# print(json.dumps(out4_list, indent=True))
|
||||
with open(dns4_path, 'w') as dns4_file:
|
||||
dns4_file.write(json.dumps(out4_list, indent=4, sort_keys=True))
|
||||
if __name__ == '__main__':
|
||||
publicdns_url = 'https://public-dns.info/nameservers.csv'
|
||||
publicdns_file = 'public-dns-nameservers.csv'
|
||||
|
||||
with open(dns6_path, 'w') as dns6_file:
|
||||
dns6_file.write(json.dumps(out6_list, indent=4, sort_keys=True))
|
||||
download_to_file(publicdns_url, publicdns_file)
|
||||
|
||||
# Public DNS Domains
|
||||
publicdns_hostname_dst = 'public-dns-hostname'
|
||||
publicdns_hostname_warninglist = {
|
||||
'description': 'Event contains one or more public DNS resolvers (expressed as hostname) as attribute with an IDS flag set',
|
||||
'name': 'List of known public DNS resolvers expressed as hostname',
|
||||
'type': 'hostname',
|
||||
'matching_attributes': ['hostname', 'domain', 'url', 'domain|ip']
|
||||
}
|
||||
process(publicdns_file, publicdns_hostname_warninglist,
|
||||
publicdns_hostname_dst, type='hostname')
|
||||
|
||||
# Public DNS IPv4
|
||||
publicdns_ipv4_dst = 'public-dns-v4'
|
||||
publicdns_ipv4_warninglist = {
|
||||
'description': 'Event contains one or more public IPv4 DNS resolvers as attribute with an IDS flag set',
|
||||
'name': 'List of known IPv4 public DNS resolvers',
|
||||
'type': 'string',
|
||||
'matching_attributes': ['ip-src', 'ip-dst', 'domain|ip']
|
||||
}
|
||||
process(publicdns_file, publicdns_ipv4_warninglist,
|
||||
publicdns_ipv4_dst, type='v4')
|
||||
|
||||
# Public DNS IPv4
|
||||
publicdns_ipv6_dst = 'public-dns-v6'
|
||||
publicdns_ipv6_warninglist = {
|
||||
'description': 'Event contains one or more public IPv6 DNS resolvers as attribute with an IDS flag set',
|
||||
'name': 'List of known IPv6 public DNS resolvers',
|
||||
'type': 'string',
|
||||
'matching_attributes': ['ip-src', 'ip-dst', 'domain|ip']
|
||||
}
|
||||
process(publicdns_file, publicdns_ipv6_warninglist,
|
||||
publicdns_ipv6_dst, type='v6')
|
||||
|
|
|
|||
Loading…
Reference in New Issue