Update lists, add schema.
parent
9214f207c3
commit
cdef6f192e
|
@ -9,4 +9,4 @@ install:
|
|||
- sudo apt-get install -y -qq jq
|
||||
|
||||
script:
|
||||
- cat */*/*.json | jq . >/dev/null
|
||||
- for dir in lists/*/list.json; do echo -n "${dir}: ";jsonschema -i ${dir} schema.json; echo ''; done
|
||||
|
|
|
@ -0,0 +1,9 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
set -x
|
||||
|
||||
for dir in lists/*/list.json
|
||||
do
|
||||
cat ${dir} | jq . | tee ${dir}
|
||||
done
|
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
|
@ -2,17 +2,31 @@
|
|||
"description": "Event contains one or more public IPv6 DNS resolvers as attribute with an IDS flag set",
|
||||
"list": [
|
||||
"2001:1488:800:400::130",
|
||||
"2001:14b8:100:350::2",
|
||||
"2001:14b8:100:8350::1",
|
||||
"2001:14b8:100:8350::8",
|
||||
"2001:1608:10:167:342::eb52",
|
||||
"2001:1608:10:195:3:dead:beef:cafe",
|
||||
"2001:1608:10:25::1c04:b12f",
|
||||
"2001:1608:10:25::9249:d69b",
|
||||
"2001:1620:2777:1::10",
|
||||
"2001:1620:2777:1::11",
|
||||
"2001:1620:2777::2",
|
||||
"2001:19f0:5001:133:5400:ff:fe30:d565",
|
||||
"2001:19f0:5801:11:5400:ff:fe2d:7724",
|
||||
"2001:19f0:7001:929:5400:ff:fe30:50af",
|
||||
"2001:19f0:8001:5e:5400:ff:fe35:c3ae",
|
||||
"2001:1a68::d911:2244",
|
||||
"2001:1bc0::ffff:aaaa:2",
|
||||
"2001:1bc0::ffff:bbbb:2",
|
||||
"2001:2040:39::5",
|
||||
"2001:418:3ff::1:53",
|
||||
"2001:418:3ff::53",
|
||||
"2001:41d0:52:cff::1325",
|
||||
"2001:41d0:52:f00::413",
|
||||
"2001:41d0:8:be92::1",
|
||||
"2001:41d0:a:1011::1",
|
||||
"2001:41d0:a:28::1",
|
||||
"2001:428:101:100:205:171:2:65",
|
||||
"2001:428:101:100:205:171:3:65",
|
||||
"2001:428::1",
|
||||
|
@ -21,19 +35,25 @@
|
|||
"2001:450:2005:2::5",
|
||||
"2001:450:2005:3::5",
|
||||
"2001:468:c80:2101:0:100:0:22",
|
||||
"2001:468:c80:2101:0:100:0:22",
|
||||
"2001:468:c80:4101:0:100:0:42",
|
||||
"2001:468:c80:4101:0:100:0:42",
|
||||
"2001:470:0:45::2",
|
||||
"2001:470:0:69::2",
|
||||
"2001:470:0:6e::2",
|
||||
"2001:470:0:78::2",
|
||||
"2001:470:0:7d::2",
|
||||
"2001:470:0:8c::2",
|
||||
"2001:470:0:90::2",
|
||||
"2001:470:0:9d::2",
|
||||
"2001:470:0:c0::2",
|
||||
"2001:470:1f14:fd8::2",
|
||||
"2001:470:20::2",
|
||||
"2001:470:520a::1",
|
||||
"2001:470:6c:521::2",
|
||||
"2001:470:6d:521::1",
|
||||
"2001:470:26:1c8::1",
|
||||
"2001:470:6d:80:224:1dff:fe84:797",
|
||||
"2001:470:6d:80:a5f6:5a97:a53:71cb",
|
||||
"2001:470:6d:80:c4f1:32a:4521:c34",
|
||||
"2001:470:6d:f1e:1337:360:dead:beef",
|
||||
"2001:470:8e08::",
|
||||
"2001:470:d:bb7::8888",
|
||||
"2001:470:f032:1::254",
|
||||
"2001:4860:4860::8844",
|
||||
"2001:4860:4860::8888",
|
||||
|
@ -42,7 +62,6 @@
|
|||
"2001:4870:8000:3::100",
|
||||
"2001:4870:8000:3::5",
|
||||
"2001:4ce8::53",
|
||||
"2001:4dd0:fb32:3::d",
|
||||
"2001:4dd0:fd5e::53",
|
||||
"2001:4f8:0:2::14",
|
||||
"2001:550:1:1::d",
|
||||
|
@ -53,58 +72,130 @@
|
|||
"2001:678:1::206",
|
||||
"2001:67c:15e8:d1::18",
|
||||
"2001:67c:15e8:d1::19",
|
||||
"2001:67c:240c:214::4",
|
||||
"2001:67c:240c:214::5",
|
||||
"2001:67c:2b0::1",
|
||||
"2001:67c:2b0::2",
|
||||
"2001:67c:2b24:1000::10",
|
||||
"2001:67c:2b24:1000::11",
|
||||
"2001:6b0:3f::a",
|
||||
"2001:750:2:3::51",
|
||||
"2001:750:2:3::52",
|
||||
"2001:7b8:1509::1",
|
||||
"2001:840:0:200::1",
|
||||
"2001:840:200::",
|
||||
"2001:840:2010:413::100",
|
||||
"2001:910:800::12",
|
||||
"2001:910:800::40",
|
||||
"2001:913::8",
|
||||
"2001:978:1:1::d",
|
||||
"2001:978:1:2::d",
|
||||
"2001:b000:168::1",
|
||||
"2001:b08:2:280::4:1",
|
||||
"2001:bf0::2",
|
||||
"2001:da8:202:10::37",
|
||||
"2001:ec0:1::1",
|
||||
"2001:ec0:3::3",
|
||||
"2400:6180:0:d0::38:d001",
|
||||
"2400:8900::f03c:91ff:fe70:c452",
|
||||
"2402:2f80:5::",
|
||||
"2402:9e80:1::1:e554",
|
||||
"2403:5680::1:200f",
|
||||
"2407:9000:0:4::2",
|
||||
"2600:3c00::20:b1ff",
|
||||
"2600:3c02::f03c:91ff:fe84:cb54",
|
||||
"2600:3c02::f03c:91ff:fee0:5e5",
|
||||
"2600::1",
|
||||
"2600::2",
|
||||
"2602:3f:e75c:1bff::1",
|
||||
"2602:ffb6:2:0:f816:3eff:fe23:ae28",
|
||||
"2602:ffc5:30::1:d69b",
|
||||
"2604:a880:1:20::c5b:1001",
|
||||
"2604:a880:400:d0::6d6:2001",
|
||||
"2605:f700:c0:1::1089:53ef",
|
||||
"2607:fa88:1::2",
|
||||
"2610:130:100:3::200",
|
||||
"2610:a1:1018::22",
|
||||
"2610:a1:1018::23",
|
||||
"2610:a1:1018::24",
|
||||
"2610:a1:1018::25",
|
||||
"2610:a1:1018::26",
|
||||
"2610:a1:1018::27",
|
||||
"2610:a1:1018::28",
|
||||
"2610:a1:1018::29",
|
||||
"2610:a1:1018::30",
|
||||
"2610:a1:1018::31",
|
||||
"2610:a1:1018::32",
|
||||
"2610:a1:1018::33",
|
||||
"2610:a1:1018::34",
|
||||
"2610:a1:1018::35",
|
||||
"2610:a1:1018::5",
|
||||
"2610:a1:1019::22",
|
||||
"2610:a1:1019::23",
|
||||
"2610:a1:1019::24",
|
||||
"2610:a1:1019::25",
|
||||
"2610:a1:1019::26",
|
||||
"2610:a1:1019::27",
|
||||
"2610:a1:1019::28",
|
||||
"2610:a1:1019::29",
|
||||
"2610:a1:1019::30",
|
||||
"2610:a1:1019::31",
|
||||
"2610:a1:1019::32",
|
||||
"2610:a1:1019::33",
|
||||
"2610:a1:1019::34",
|
||||
"2610:a1:1019::35",
|
||||
"2610:a1:1019::5",
|
||||
"2620:0:ccc::2",
|
||||
"2620:0:ccd::2",
|
||||
"2800:960:0:12:201:217:1:231",
|
||||
"2620:74:1b::1:1",
|
||||
"2620:74:1c::2:2",
|
||||
"2a00-1508-0-4--9.puntcat.ip6.guifi.net.",
|
||||
"2a00-1dc0-cafe--ad86-fa7e.static.host.",
|
||||
"2a00-1dc0-cafe--c6af-c19d.static.host.",
|
||||
"2a00:12d8:7002::2",
|
||||
"2a00:1508:0:4::9",
|
||||
"2a00:1ca8:a7::1e9",
|
||||
"2a00:1dc0:cafe::ad86:fa7e",
|
||||
"2a00:1dc0:cafe::c6af:c19d",
|
||||
"2a00:5881:8100:1000::3",
|
||||
"2a00:5884:8218::1",
|
||||
"2a00:dcc0:eda:88:245:71:858e:a15",
|
||||
"2a00:dcc0:eda:98:183:193:d85a:389b",
|
||||
"2a00:dcc7:2202:11::7b28",
|
||||
"2a00:dcc7:2202:14::2",
|
||||
"2a00:f48:100c:7b::2",
|
||||
"2a00:f48:100c:7e::2",
|
||||
"2a01:4f8:131:1278::2",
|
||||
"2a01:4f8:141:4281::3000",
|
||||
"2a01:4f8:151:90e9::2",
|
||||
"2a01:4f8:151:90e9::b",
|
||||
"2a01:4f8:161:4109::6",
|
||||
"2a01:4f8:191:306c::2",
|
||||
"2a02:180:1:1::517:1045",
|
||||
"2a02:2178:1:2::2",
|
||||
"2a02:2ca0:64:22::2",
|
||||
"2a02:6b8::feed:ff",
|
||||
"2a02:7aa0:1201::f60e:2719",
|
||||
"2a02:7aa0:1619::4f50:a69",
|
||||
"2a02:940:0:4293::100",
|
||||
"2a03:4000:6:510b::1",
|
||||
"2a02:e00:fffd:139::9",
|
||||
"2a03:b0c0:0:1010::62:f001",
|
||||
"2a03:b0c0:3:d0::7c:5001",
|
||||
"McRip-5-pt.tunnel.tserv26.ber1.ipv6.he.net.",
|
||||
"2a04:92c7:7:7::14ae:460a",
|
||||
"2a04:9dc0:c1:7::cb9:f785",
|
||||
"2a05:b0c6:5e4::53",
|
||||
"2a05:dfc7:5::53",
|
||||
"2a05:dfc7:5::5353",
|
||||
"2c0f:fda8:5::2ed1:d2ec",
|
||||
"::ffff:9538:1aed",
|
||||
"::ffff:9e45:abfe",
|
||||
"ClemenTroniQ89-1-pt.tunnel.tserv11.ams1.ipv6.he.net.",
|
||||
"anyone.dnsrec.meo.ws.",
|
||||
"anytwo.dnsrec.meo.ws.",
|
||||
"b-root.cesidian.info.",
|
||||
"canopus.ne2000.nl.",
|
||||
"copaco-public-resolver-ipv6-b.copaco.com.py.",
|
||||
"cl-849.hel-01.fi.sixxs.net.",
|
||||
"crt-public-dns-a.cesidianroot.eu.",
|
||||
"cznic-public-dns-1.nic.cz.",
|
||||
"dlfw-rdns-01.dlfw.twtelecom.net.",
|
||||
"dns.cesidianroot.eu.",
|
||||
"dns.yandex.ru.",
|
||||
"dns01.jordbruksverket.se.",
|
||||
"dns02.jordbruksverket.se.",
|
||||
"dns1.host.net.",
|
||||
"dns1.lon.gblx.net.",
|
||||
"dns1.phx.gblx.net.",
|
||||
|
@ -112,44 +203,53 @@
|
|||
"dns2.phx.gblx.net.",
|
||||
"dns2.roc.gblx.net.",
|
||||
"dns2.totbb.net.",
|
||||
"dnsdist.mysrvr.net.",
|
||||
"dnsres1.nic.cz.",
|
||||
"dnvr-rdns-01.dnvr.twtelecom.net.",
|
||||
"emma.robingroppe.de.",
|
||||
"eu-res1.dns.cogentco.com.",
|
||||
"eu-res2.dns.cogentco.com.",
|
||||
"freya.stelas.de.",
|
||||
"google-public-dns-a.google.com.",
|
||||
"google-public-dns-b.google.com.",
|
||||
"hntp1.hinet.net.",
|
||||
"homens.b-hs.de.",
|
||||
"host19-65-static.59-88-b.business.telecomitalia.it.",
|
||||
"jeru.cns.ipv6.vt.edu.",
|
||||
"jeru.cns.ipv6.vt.edu.",
|
||||
"leia.fdn.org.",
|
||||
"log.bzh.",
|
||||
"lpc1.stu.neva.ru.",
|
||||
"mnt1.eutelia.it.",
|
||||
"mnt2.eutelia.it.",
|
||||
"lucy.s.imvry.pw.",
|
||||
"mail2.cesidianroot.eu.",
|
||||
"na-res1.dns.cogentco.com.",
|
||||
"ns-3.iastate.edu.",
|
||||
"ns.ipv6.uni-leipzig.de.",
|
||||
"ns0.ldn-fai.net.",
|
||||
"ns1.fdn.org.",
|
||||
"ns0.fdn.org.",
|
||||
"ns1.ams.dns.lchi.mp.",
|
||||
"ns1.ata.dns.lchi.mp.",
|
||||
"ns1.fdn.fr.",
|
||||
"ns1.hnd.dns.lchi.mp.",
|
||||
"ns1.init7.net.",
|
||||
"ns1.nl.dns.d0wn.biz.",
|
||||
"ns1.probe-networks.de.",
|
||||
"ns1.sprintlink.net.",
|
||||
"ns1.sea.dns.lchi.mp.",
|
||||
"ns1.sg.dns.d0wn.biz.",
|
||||
"ns1.shodan.io.",
|
||||
"ns1.syd.dns.lchi.mp.",
|
||||
"ns1.twtelecom.net.",
|
||||
"ns10.init7.net.",
|
||||
"ns11.init7.net.",
|
||||
"ns2.all.de.",
|
||||
"ns2.itandtel.at.",
|
||||
"ns2.powertech.no.",
|
||||
"ns2.powertech.no.",
|
||||
"ns2.probe-networks.de.",
|
||||
"ns2.shodan.io.",
|
||||
"ns2.sprintlink.net.",
|
||||
"ns2.twtelecom.net.",
|
||||
"ns532549.ip-149-56-26.net.",
|
||||
"open-root.cesidian.info.",
|
||||
"or.isc.org.",
|
||||
"ordns.he.net.",
|
||||
"plfgr.eu.org.",
|
||||
"primary.server.edv-froehlich.de.",
|
||||
"proxyvm.stejau.de.",
|
||||
"public-dns-a.primawebtools.de.",
|
||||
"public-dns-c.ipv6.primawebtools.de.",
|
||||
"recursif.arn-fai.net.",
|
||||
"resolver.qwest.net.",
|
||||
"resolver1.dns.trex.fi.",
|
||||
|
@ -158,20 +258,19 @@
|
|||
"resolver2.ipv6-sandbox.opendns.com.",
|
||||
"rickhunter.ns.ielo.net.",
|
||||
"rns1.grnet.gr.",
|
||||
"secondary.server.edv-froehlich.de.",
|
||||
"services.donotuse.de.",
|
||||
"spcr-2.machadosbsmarketing.com.br.",
|
||||
"test.cesidian.info.",
|
||||
"tserv1.ams1.he.net.",
|
||||
"tserv1.dal1.he.net.",
|
||||
"tserv1.fmt2.he.net.",
|
||||
"tserv1.fra1.he.net.",
|
||||
"tserv1.mia1.he.net.",
|
||||
"tserv1.tor1.he.net.",
|
||||
"tungsten.gparent.org.",
|
||||
"voip.zee.li.",
|
||||
"www.cesidianroot.eu.",
|
||||
"x.ns.gin.ntt.net.",
|
||||
"y.ns.gin.ntt.net.",
|
||||
"yardbird.cns.ipv6.vt.edu.",
|
||||
"yardbird.cns.ipv6.vt.edu.",
|
||||
"zen.stack.nl."
|
||||
],
|
||||
"matching_attributes": [
|
||||
|
@ -180,5 +279,5 @@
|
|||
"domain|ip"
|
||||
],
|
||||
"name": "List of known IPv6 public DNS resolvers",
|
||||
"version": "20160803"
|
||||
}
|
||||
"version": 20170212
|
||||
}
|
|
@ -0,0 +1,45 @@
|
|||
{
|
||||
"$schema": "http://json-schema.org/schema#",
|
||||
"title": "Validator for misp-warninglists",
|
||||
"id": "https://www.github.com/MISP/misp-warninglists/schema.json",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"description": {
|
||||
"type": "string"
|
||||
},
|
||||
"name": {
|
||||
"type": "string"
|
||||
},
|
||||
"version": {
|
||||
"type": "integer"
|
||||
},
|
||||
"list": {
|
||||
"type": "array",
|
||||
"uniqueItems": true,
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"type": {
|
||||
"type": "array",
|
||||
"uniqueItems": true,
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"matching_attributes": {
|
||||
"type": "array",
|
||||
"uniqueItems": true,
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"list",
|
||||
"matching_attributes",
|
||||
"description",
|
||||
"version",
|
||||
"name"
|
||||
]
|
||||
}
|
|
@ -8,7 +8,7 @@ import json
|
|||
|
||||
alexa_url = "http://s3.amazonaws.com/alexa-static/top-1m.csv.zip"
|
||||
alexa_file = "top-1m.csv.zip"
|
||||
user_agent = {"User-agent":"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:46.0) Gecko/20100101 Firefox/46.0"}
|
||||
user_agent = {"User-agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:46.0) Gecko/20100101 Firefox/46.0"}
|
||||
r = requests.get(alexa_url, headers=user_agent)
|
||||
with open(alexa_file, 'wb') as fd:
|
||||
for chunk in r.iter_content(4096):
|
||||
|
@ -22,15 +22,17 @@ with zipfile.ZipFile(alexa_file, 'r') as alexa_lists:
|
|||
continue
|
||||
|
||||
alexa_warninglist = {}
|
||||
version = int(datetime.date.today().strftime('%Y%m%d'))
|
||||
|
||||
alexa_warninglist['description'] = "Event contains one or more entries from the top 1000 of the most used website (Alexa)."
|
||||
d = datetime.datetime.now()
|
||||
alexa_warninglist['version'] = "{0}{1:02d}{2:02d}".format(d.year,d.month,d.day)
|
||||
alexa_warninglist['version'] = version
|
||||
alexa_warninglist['name'] = "Top 1000 website from Alexa"
|
||||
alexa_warninglist['list'] = []
|
||||
alexa_warninglist['matching_attributes'] = ['hostname','domain']
|
||||
alexa_warninglist['matching_attributes'] = ['hostname', 'domain']
|
||||
|
||||
for site in top1000:
|
||||
v = str(site).split(',')[1]
|
||||
alexa_warninglist['list'].append(v[:-3])
|
||||
print (json.dumps(alexa_warninglist))
|
||||
alexa_warninglist['list'] = sorted(set(alexa_warninglist['list']))
|
||||
print(json.dumps(alexa_warninglist))
|
||||
|
|
|
@ -15,10 +15,10 @@ for address in office365.iter('address'):
|
|||
|
||||
warninglist = {}
|
||||
warninglist['name'] = 'List of known Office 365 URLs and IP address ranges'
|
||||
d = datetime.datetime.now()
|
||||
warninglist['version'] = "{0}{1:02d}{2:02d}".format(d.year, d.month, d.day)
|
||||
warninglist['version'] = int(datetime.date.today().strftime('%Y%m%d'))
|
||||
warninglist['description'] = 'Office 365 URLs and IP address ranges'
|
||||
warninglist['list'] = l
|
||||
warninglist['list'] = sorted(set(l))
|
||||
warninglist['matching_attributes'] = ["ip-src", "ip-dst", "domain|ip", "hostname"]
|
||||
|
||||
|
||||
print (json.dumps(warninglist))
|
||||
print(json.dumps(warninglist))
|
||||
|
|
|
@ -28,7 +28,7 @@ with open(csv_path) as csv_file:
|
|||
|
||||
for row in servers_list:
|
||||
if row[5] == '':
|
||||
try:
|
||||
try:
|
||||
ip = ipaddress.ip_address(row[0])
|
||||
|
||||
if ip.version == 4:
|
||||
|
@ -43,25 +43,25 @@ with open(csv_path) as csv_file:
|
|||
except ValueError as exc:
|
||||
logging.warning(str(exc))
|
||||
|
||||
version = datetime.datetime.now().strftime('%Y%m%d')
|
||||
version = int(datetime.date.today().strftime('%Y%m%d'))
|
||||
|
||||
out4_list = {}
|
||||
out4_list['name'] = 'List of known IPv4 public DNS resolvers'
|
||||
out4_list['version'] = version
|
||||
out4_list['description'] = 'Event contains one or more public IPv4 DNS resolvers as attribute with an IDS flag set'
|
||||
out4_list['matching_attributes'] = [ 'ip-src', 'ip-dst', 'domain|ip' ]
|
||||
out4_list['list'] = sorted(ip4_list)
|
||||
out4_list['matching_attributes'] = ['ip-src', 'ip-dst', 'domain|ip']
|
||||
out4_list['list'] = sorted(set(ip4_list))
|
||||
|
||||
|
||||
out6_list = {}
|
||||
out6_list['name'] = 'List of known IPv6 public DNS resolvers'
|
||||
out6_list['version'] = version
|
||||
out6_list['description'] = 'Event contains one or more public IPv6 DNS resolvers as attribute with an IDS flag set'
|
||||
out6_list['matching_attributes'] = [ 'ip-src', 'ip-dst', 'domain|ip' ]
|
||||
out6_list['list'] = sorted(ip6_list)
|
||||
out6_list['matching_attributes'] = ['ip-src', 'ip-dst', 'domain|ip']
|
||||
out6_list['list'] = sorted(set(ip6_list))
|
||||
|
||||
|
||||
#print(json.dumps(out4_list, indent=True))
|
||||
# print(json.dumps(out4_list, indent=True))
|
||||
with open(dns4_path, 'w') as dns4_file:
|
||||
dns4_file.write(json.dumps(out4_list, indent=4, sort_keys=True))
|
||||
|
||||
|
|
Loading…
Reference in New Issue