MISP
/
misp-warninglists
mirror of https://github.com/MISP/misp-warninglists
2
0
Fork 0
Code Issues Releases Wiki Activity

Update lists, add schema.

pull/27/head
Raphaël Vinot 2017-02-12 21:01:36 +01:00
parent 9214f207c3
commit cdef6f192e
10 changed files with 13497 additions and 43257 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.travis.yml
View File

@ -9,4 +9,4 @@ install:
- sudo apt-get install -y -qq jq
script:
- cat */*/*.json | jq . >/dev/null
- for dir in lists/*/list.json; do echo -n "${dir}: ";jsonschema -i ${dir} schema.json; echo ''; done

9
jq_all_the_things.sh Normal file
View File

@ -0,0 +1,9 @@
#!/bin/bash
set -e
set -x
for dir in lists/*/list.json
do
cat ${dir} | jq . | tee ${dir}
done

1978
lists/alexa/list.json
View File

File diff suppressed because it is too large Load Diff

2417
lists/microsoft-office365/list.json
View File

File diff suppressed because it is too large Load Diff

52104
lists/public-dns-v4/list.json
View File

File diff suppressed because it is too large Load Diff

167
lists/public-dns-v6/list.json
View File

@ -2,17 +2,31 @@
"description": "Event contains one or more public IPv6 DNS resolvers as attribute with an IDS flag set",
"list": [
"2001:1488:800:400::130",
"2001:14b8:100:350::2",
"2001:14b8:100:8350::1",
"2001:14b8:100:8350::8",
"2001:1608:10:167:342::eb52",
"2001:1608:10:195:3:dead:beef:cafe",
"2001:1608:10:25::1c04:b12f",
"2001:1608:10:25::9249:d69b",
"2001:1620:2777:1::10",
"2001:1620:2777:1::11",
"2001:1620:2777::2",
"2001:19f0:5001:133:5400:ff:fe30:d565",
"2001:19f0:5801:11:5400:ff:fe2d:7724",
"2001:19f0:7001:929:5400:ff:fe30:50af",
"2001:19f0:8001:5e:5400:ff:fe35:c3ae",
"2001:1a68::d911:2244",
"2001:1bc0::ffff:aaaa:2",
"2001:1bc0::ffff:bbbb:2",
"2001:2040:39::5",
"2001:418:3ff::1:53",
"2001:418:3ff::53",
"2001:41d0:52:cff::1325",
"2001:41d0:52:f00::413",
"2001:41d0:8:be92::1",
"2001:41d0:a:1011::1",
"2001:41d0:a:28::1",
"2001:428:101:100:205:171:2:65",
"2001:428:101:100:205:171:3:65",
"2001:428::1",
@ -21,19 +35,25 @@
"2001:450:2005:2::5",
"2001:450:2005:3::5",
"2001:468:c80:2101:0:100:0:22",
"2001:468:c80:2101:0:100:0:22",
"2001:468:c80:4101:0:100:0:42",
"2001:468:c80:4101:0:100:0:42",
"2001:470:0:45::2",
"2001:470:0:69::2",
"2001:470:0:6e::2",
"2001:470:0:78::2",
"2001:470:0:7d::2",
"2001:470:0:8c::2",
"2001:470:0:90::2",
"2001:470:0:9d::2",
"2001:470:0:c0::2",
"2001:470:1f14:fd8::2",
"2001:470:20::2",
"2001:470:520a::1",
"2001:470:6c:521::2",
"2001:470:6d:521::1",
"2001:470:26:1c8::1",
"2001:470:6d:80:224:1dff:fe84:797",
"2001:470:6d:80:a5f6:5a97:a53:71cb",
"2001:470:6d:80:c4f1:32a:4521:c34",
"2001:470:6d:f1e:1337:360:dead:beef",
"2001:470:8e08::",
"2001:470:d:bb7::8888",
"2001:470:f032:1::254",
"2001:4860:4860::8844",
"2001:4860:4860::8888",
@ -42,7 +62,6 @@
"2001:4870:8000:3::100",
"2001:4870:8000:3::5",
"2001:4ce8::53",
"2001:4dd0:fb32:3::d",
"2001:4dd0:fd5e::53",
"2001:4f8:0:2::14",
"2001:550:1:1::d",
@ -53,58 +72,130 @@
"2001:678:1::206",
"2001:67c:15e8:d1::18",
"2001:67c:15e8:d1::19",
"2001:67c:240c:214::4",
"2001:67c:240c:214::5",
"2001:67c:2b0::1",
"2001:67c:2b0::2",
"2001:67c:2b24:1000::10",
"2001:67c:2b24:1000::11",
"2001:6b0:3f::a",
"2001:750:2:3::51",
"2001:750:2:3::52",
"2001:7b8:1509::1",
"2001:840:0:200::1",
"2001:840:200::",
"2001:840:2010:413::100",
"2001:910:800::12",
"2001:910:800::40",
"2001:913::8",
"2001:978:1:1::d",
"2001:978:1:2::d",
"2001:b000:168::1",
"2001:b08:2:280::4:1",
"2001:bf0::2",
"2001:da8:202:10::37",
"2001:ec0:1::1",
"2001:ec0:3::3",
"2400:6180:0:d0::38:d001",
"2400:8900::f03c:91ff:fe70:c452",
"2402:2f80:5::",
"2402:9e80:1::1:e554",
"2403:5680::1:200f",
"2407:9000:0:4::2",
"2600:3c00::20:b1ff",
"2600:3c02::f03c:91ff:fe84:cb54",
"2600:3c02::f03c:91ff:fee0:5e5",
"2600::1",
"2600::2",
"2602:3f:e75c:1bff::1",
"2602:ffb6:2:0:f816:3eff:fe23:ae28",
"2602:ffc5:30::1:d69b",
"2604:a880:1:20::c5b:1001",
"2604:a880:400:d0::6d6:2001",
"2605:f700:c0:1::1089:53ef",
"2607:fa88:1::2",
"2610:130:100:3::200",
"2610:a1:1018::22",
"2610:a1:1018::23",
"2610:a1:1018::24",
"2610:a1:1018::25",
"2610:a1:1018::26",
"2610:a1:1018::27",
"2610:a1:1018::28",
"2610:a1:1018::29",
"2610:a1:1018::30",
"2610:a1:1018::31",
"2610:a1:1018::32",
"2610:a1:1018::33",
"2610:a1:1018::34",
"2610:a1:1018::35",
"2610:a1:1018::5",
"2610:a1:1019::22",
"2610:a1:1019::23",
"2610:a1:1019::24",
"2610:a1:1019::25",
"2610:a1:1019::26",
"2610:a1:1019::27",
"2610:a1:1019::28",
"2610:a1:1019::29",
"2610:a1:1019::30",
"2610:a1:1019::31",
"2610:a1:1019::32",
"2610:a1:1019::33",
"2610:a1:1019::34",
"2610:a1:1019::35",
"2610:a1:1019::5",
"2620:0:ccc::2",
"2620:0:ccd::2",
"2800:960:0:12:201:217:1:231",
"2620:74:1b::1:1",
"2620:74:1c::2:2",
"2a00-1508-0-4--9.puntcat.ip6.guifi.net.",
"2a00-1dc0-cafe--ad86-fa7e.static.host.",
"2a00-1dc0-cafe--c6af-c19d.static.host.",
"2a00:12d8:7002::2",
"2a00:1508:0:4::9",
"2a00:1ca8:a7::1e9",
"2a00:1dc0:cafe::ad86:fa7e",
"2a00:1dc0:cafe::c6af:c19d",
"2a00:5881:8100:1000::3",
"2a00:5884:8218::1",
"2a00:dcc0:eda:88:245:71:858e:a15",
"2a00:dcc0:eda:98:183:193:d85a:389b",
"2a00:dcc7:2202:11::7b28",
"2a00:dcc7:2202:14::2",
"2a00:f48:100c:7b::2",
"2a00:f48:100c:7e::2",
"2a01:4f8:131:1278::2",
"2a01:4f8:141:4281::3000",
"2a01:4f8:151:90e9::2",
"2a01:4f8:151:90e9::b",
"2a01:4f8:161:4109::6",
"2a01:4f8:191:306c::2",
"2a02:180:1:1::517:1045",
"2a02:2178:1:2::2",
"2a02:2ca0:64:22::2",
"2a02:6b8::feed:ff",
"2a02:7aa0:1201::f60e:2719",
"2a02:7aa0:1619::4f50:a69",
"2a02:940:0:4293::100",
"2a03:4000:6:510b::1",
"2a02:e00:fffd:139::9",
"2a03:b0c0:0:1010::62:f001",
"2a03:b0c0:3:d0::7c:5001",
"McRip-5-pt.tunnel.tserv26.ber1.ipv6.he.net.",
"2a04:92c7:7:7::14ae:460a",
"2a04:9dc0:c1:7::cb9:f785",
"2a05:b0c6:5e4::53",
"2a05:dfc7:5::53",
"2a05:dfc7:5::5353",
"2c0f:fda8:5::2ed1:d2ec",
"::ffff:9538:1aed",
"::ffff:9e45:abfe",
"ClemenTroniQ89-1-pt.tunnel.tserv11.ams1.ipv6.he.net.",
"anyone.dnsrec.meo.ws.",
"anytwo.dnsrec.meo.ws.",
"b-root.cesidian.info.",
"canopus.ne2000.nl.",
"copaco-public-resolver-ipv6-b.copaco.com.py.",
"cl-849.hel-01.fi.sixxs.net.",
"crt-public-dns-a.cesidianroot.eu.",
"cznic-public-dns-1.nic.cz.",
"dlfw-rdns-01.dlfw.twtelecom.net.",
"dns.cesidianroot.eu.",
"dns.yandex.ru.",
"dns01.jordbruksverket.se.",
"dns02.jordbruksverket.se.",
"dns1.host.net.",
"dns1.lon.gblx.net.",
"dns1.phx.gblx.net.",
@ -112,44 +203,53 @@
"dns2.phx.gblx.net.",
"dns2.roc.gblx.net.",
"dns2.totbb.net.",
"dnsdist.mysrvr.net.",
"dnsres1.nic.cz.",
"dnvr-rdns-01.dnvr.twtelecom.net.",
"emma.robingroppe.de.",
"eu-res1.dns.cogentco.com.",
"eu-res2.dns.cogentco.com.",
"freya.stelas.de.",
"google-public-dns-a.google.com.",
"google-public-dns-b.google.com.",
"hntp1.hinet.net.",
"homens.b-hs.de.",
"host19-65-static.59-88-b.business.telecomitalia.it.",
"jeru.cns.ipv6.vt.edu.",
"jeru.cns.ipv6.vt.edu.",
"leia.fdn.org.",
"log.bzh.",
"lpc1.stu.neva.ru.",
"mnt1.eutelia.it.",
"mnt2.eutelia.it.",
"lucy.s.imvry.pw.",
"mail2.cesidianroot.eu.",
"na-res1.dns.cogentco.com.",
"ns-3.iastate.edu.",
"ns.ipv6.uni-leipzig.de.",
"ns0.ldn-fai.net.",
"ns1.fdn.org.",
"ns0.fdn.org.",
"ns1.ams.dns.lchi.mp.",
"ns1.ata.dns.lchi.mp.",
"ns1.fdn.fr.",
"ns1.hnd.dns.lchi.mp.",
"ns1.init7.net.",
"ns1.nl.dns.d0wn.biz.",
"ns1.probe-networks.de.",
"ns1.sprintlink.net.",
"ns1.sea.dns.lchi.mp.",
"ns1.sg.dns.d0wn.biz.",
"ns1.shodan.io.",
"ns1.syd.dns.lchi.mp.",
"ns1.twtelecom.net.",
"ns10.init7.net.",
"ns11.init7.net.",
"ns2.all.de.",
"ns2.itandtel.at.",
"ns2.powertech.no.",
"ns2.powertech.no.",
"ns2.probe-networks.de.",
"ns2.shodan.io.",
"ns2.sprintlink.net.",
"ns2.twtelecom.net.",
"ns532549.ip-149-56-26.net.",
"open-root.cesidian.info.",
"or.isc.org.",
"ordns.he.net.",
"plfgr.eu.org.",
"primary.server.edv-froehlich.de.",
"proxyvm.stejau.de.",
"public-dns-a.primawebtools.de.",
"public-dns-c.ipv6.primawebtools.de.",
"recursif.arn-fai.net.",
"resolver.qwest.net.",
"resolver1.dns.trex.fi.",
@ -158,20 +258,19 @@
"resolver2.ipv6-sandbox.opendns.com.",
"rickhunter.ns.ielo.net.",
"rns1.grnet.gr.",
"secondary.server.edv-froehlich.de.",
"services.donotuse.de.",
"spcr-2.machadosbsmarketing.com.br.",
"test.cesidian.info.",
"tserv1.ams1.he.net.",
"tserv1.dal1.he.net.",
"tserv1.fmt2.he.net.",
"tserv1.fra1.he.net.",
"tserv1.mia1.he.net.",
"tserv1.tor1.he.net.",
"tungsten.gparent.org.",
"voip.zee.li.",
"www.cesidianroot.eu.",
"x.ns.gin.ntt.net.",
"y.ns.gin.ntt.net.",
"yardbird.cns.ipv6.vt.edu.",
"yardbird.cns.ipv6.vt.edu.",
"zen.stack.nl."
],
"matching_attributes": [
@ -180,5 +279,5 @@
"domain|ip"
],
"name": "List of known IPv6 public DNS resolvers",
"version": "20160803"
}
"version": 20170212
}

45
schema.json Normal file
View File

@ -0,0 +1,45 @@
{
"$schema": "http://json-schema.org/schema#",
"title": "Validator for misp-warninglists",
"id": "https://www.github.com/MISP/misp-warninglists/schema.json",
"type": "object",
"properties": {
"description": {
"type": "string"
},
"name": {
"type": "string"
},
"version": {
"type": "integer"
},
"list": {
"type": "array",
"uniqueItems": true,
"items": {
"type": "string"
}
},
"type": {
"type": "array",
"uniqueItems": true,
"items": {
"type": "string"
}
},
"matching_attributes": {
"type": "array",
"uniqueItems": true,
"items": {
"type": "string"
}
}
},
"required": [
"list",
"matching_attributes",
"description",
"version",
"name"
]
}

10
tools/generate-alexa.py Normal file → Executable file
View File

@ -8,7 +8,7 @@ import json
alexa_url = "http://s3.amazonaws.com/alexa-static/top-1m.csv.zip"
alexa_file = "top-1m.csv.zip"
user_agent = {"User-agent":"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:46.0) Gecko/20100101 Firefox/46.0"}
user_agent = {"User-agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:46.0) Gecko/20100101 Firefox/46.0"}
r = requests.get(alexa_url, headers=user_agent)
with open(alexa_file, 'wb') as fd:
for chunk in r.iter_content(4096):
@ -22,15 +22,17 @@ with zipfile.ZipFile(alexa_file, 'r') as alexa_lists:
continue
alexa_warninglist = {}
version = int(datetime.date.today().strftime('%Y%m%d'))
alexa_warninglist['description'] = "Event contains one or more entries from the top 1000 of the most used website (Alexa)."
d = datetime.datetime.now()
alexa_warninglist['version'] = "{0}{1:02d}{2:02d}".format(d.year,d.month,d.day)
alexa_warninglist['version'] = version
alexa_warninglist['name'] = "Top 1000 website from Alexa"
alexa_warninglist['list'] = []
alexa_warninglist['matching_attributes'] = ['hostname','domain']
alexa_warninglist['matching_attributes'] = ['hostname', 'domain']
for site in top1000:
v = str(site).split(',')[1]
alexa_warninglist['list'].append(v[:-3])
print (json.dumps(alexa_warninglist))
alexa_warninglist['list'] = sorted(set(alexa_warninglist['list']))
print(json.dumps(alexa_warninglist))

8
tools/generate-office365.py Normal file → Executable file
View File

@ -15,10 +15,10 @@ for address in office365.iter('address'):
warninglist = {}
warninglist['name'] = 'List of known Office 365 URLs and IP address ranges'
d = datetime.datetime.now()
warninglist['version'] = "{0}{1:02d}{2:02d}".format(d.year, d.month, d.day)
warninglist['version'] = int(datetime.date.today().strftime('%Y%m%d'))
warninglist['description'] = 'Office 365 URLs and IP address ranges'
warninglist['list'] = l
warninglist['list'] = sorted(set(l))
warninglist['matching_attributes'] = ["ip-src", "ip-dst", "domain|ip", "hostname"]
print (json.dumps(warninglist))
print(json.dumps(warninglist))

14
tools/generate-publicdns.py
View File

@ -28,7 +28,7 @@ with open(csv_path) as csv_file:
for row in servers_list:
if row[5] == '':
try:
try:
ip = ipaddress.ip_address(row[0])
if ip.version == 4:
@ -43,25 +43,25 @@ with open(csv_path) as csv_file:
except ValueError as exc:
logging.warning(str(exc))
version = datetime.datetime.now().strftime('%Y%m%d')
version = int(datetime.date.today().strftime('%Y%m%d'))
out4_list = {}
out4_list['name'] = 'List of known IPv4 public DNS resolvers'
out4_list['version'] = version
out4_list['description'] = 'Event contains one or more public IPv4 DNS resolvers as attribute with an IDS flag set'
out4_list['matching_attributes'] = [ 'ip-src', 'ip-dst', 'domain|ip' ]
out4_list['list'] = sorted(ip4_list)
out4_list['matching_attributes'] = ['ip-src', 'ip-dst', 'domain|ip']
out4_list['list'] = sorted(set(ip4_list))
out6_list = {}
out6_list['name'] = 'List of known IPv6 public DNS resolvers'
out6_list['version'] = version
out6_list['description'] = 'Event contains one or more public IPv6 DNS resolvers as attribute with an IDS flag set'
out6_list['matching_attributes'] = [ 'ip-src', 'ip-dst', 'domain|ip' ]
out6_list['list'] = sorted(ip6_list)
out6_list['matching_attributes'] = ['ip-src', 'ip-dst', 'domain|ip']
out6_list['list'] = sorted(set(ip6_list))
#print(json.dumps(out4_list, indent=True))
# print(json.dumps(out4_list, indent=True))
with open(dns4_path, 'w') as dns4_file:
dns4_file.write(json.dumps(out4_list, indent=4, sort_keys=True))