From 93b6a5c1204ce7594836297159006e0626a6bbbe Mon Sep 17 00:00:00 2001 From: rmarsollier Date: Mon, 19 Jun 2017 09:08:58 +0200 Subject: [PATCH 1/3] importing google domains from wikipedia --- lists/google/list.json | 775 ++++++++++++++++++++++++++++------------- 1 file changed, 527 insertions(+), 248 deletions(-) diff --git a/lists/google/list.json b/lists/google/list.json index 95e23cc..9722004 100644 --- a/lists/google/list.json +++ b/lists/google/list.json @@ -1,6 +1,6 @@ { "name": "List of known google domains", - "version": 2, + "version": 3, "description": "Event contains one or more entries of known google domains", "matching_attributes": [ "domain", @@ -8,391 +8,670 @@ "domain|ip" ], "list": [ - ".google.com", + "1e100.net", + "466453.com", + "abc.xyz", + "admob.com", + "adsense.com", + "advertisercommunity.com", + "adwords.com", + "ai.google", + "android.com", + "blogger.com", + "blog.google", + "blogspot.com", + "capitalg.com", + "chromebook.com", + "chromecast.com", + "chrome.com", + "chromium.org", + "cobrasearch.com", + "com.google", + "domains.google", + "doubleclickbygoogle.com", + "doubleclick.com", + "duck.com", + "elgoog.im", + "feedburner.com", + "foofle.com", + "froogle.com", + "g.co", + "ggpht.com", + "gmail.com", + "gmodules.com", + "gogle.com", + "gogole.com", + "googel.com", + "googil.com", + "goo.gl", + "googl.com", + "google.ac", ".google.ad", + "google.ad", + "googleadservices.com", ".google.ae", + "google.ae", + "google.af", + "google.ag", + "google.ai", + ".google.al", + "google.al", + ".google.am", + "google.am", + "google-analytics.com", + "google.ao", + "googleapis.com", + "googleapps.com", + "google.ar", + "googlearth.com", + ".google.as", + "google.as", + ".google.at", + "google.at", + "google.au", + ".google.az", + "google.az", + ".google.ba", + "google.ba", + "google.bd", + ".google.be", + "google.be", + ".google.bf", + "google.bf", + ".google.bg", + "google.bg", + "google.bh", + ".google.bi", + "google.bi", + ".google.bj", + "google.bj", + "google.bn", + "google.bo", + "googlebot.com", + "google.br", + ".google.bs", + "google.bs", + ".google.bt", + "google.bt", + "google.bw", + ".google.by", + "google.by", + "google.bz", + "google.ca", + ".google.cat", + "google.cat", + "google.cc", + ".google.cd", + "google.cd", + ".google.cf", + "google.cf", + ".google.cg", + "google.cg", + ".google.ch", + "google.ch", + ".google.ci", + "google.ci", + "google.ck", + ".google.cl", + "google.cl", + ".google.cm", + "google.cm", + ".google.cn", + "google.cn", + "google.co", + ".google.co.ao", + ".google.co.bw", + ".google.co.ck", + ".google.co.cr", + "googlecode.com", + ".google.co.id", + ".google.co.il", + ".google.co.in", + ".google.co.jp", + ".google.co.ke", + ".google.co.kr", + ".google.co.ls", + ".google.com", + "google.com", + ".google.co.ma", ".google.com.af", ".google.com.ag", ".google.com.ai", - ".google.al", - ".google.am", - ".google.co.ao", ".google.com.ar", - ".google.as", - ".google.at", ".google.com.au", - ".google.az", - ".google.ba", ".google.com.bd", - ".google.be", - ".google.bf", - ".google.bg", ".google.com.bh", - ".google.bi", - ".google.bj", ".google.com.bn", ".google.com.bo", ".google.com.br", - ".google.bs", - ".google.bt", - ".google.co.bw", - ".google.by", ".google.com.bz", - ".google.ca", - ".google.cd", - ".google.cf", - ".google.cg", - ".google.ch", - ".google.ci", - ".google.co.ck", - ".google.cl", - ".google.cm", - ".google.cn", ".google.com.co", - ".google.co.cr", ".google.com.cu", - ".google.cv", ".google.com.cy", - ".google.cz", - ".google.de", - ".google.dj", - ".google.dk", - ".google.dm", ".google.com.do", - ".google.dz", ".google.com.ec", - ".google.ee", ".google.com.eg", - ".google.es", ".google.com.et", - ".google.fi", ".google.com.fj", - ".google.fm", - ".google.fr", - ".google.ga", - ".google.ge", - ".google.gg", ".google.com.gh", ".google.com.gi", - ".google.gl", - ".google.gm", - ".google.gp", - ".google.gr", ".google.com.gt", - ".google.gy", ".google.com.hk", - ".google.hn", - ".google.hr", - ".google.ht", - ".google.hu", - ".google.co.id", - ".google.ie", - ".google.co.il", - ".google.im", - ".google.co.in", - ".google.iq", - ".google.is", - ".google.it", - ".google.je", ".google.com.jm", - ".google.jo", - ".google.co.jp", - ".google.co.ke", ".google.com.kh", - ".google.ki", - ".google.kg", - ".google.co.kr", ".google.com.kw", - ".google.kz", - ".google.la", ".google.com.lb", - ".google.li", - ".google.lk", - ".google.co.ls", - ".google.lt", - ".google.lu", - ".google.lv", ".google.com.ly", - ".google.co.ma", - ".google.md", - ".google.me", - ".google.mg", - ".google.mk", - ".google.ml", + "googlecommerce.com", ".google.com.mm", - ".google.mn", - ".google.ms", ".google.com.mt", - ".google.mu", - ".google.mv", - ".google.mw", ".google.com.mx", ".google.com.my", - ".google.co.mz", ".google.com.na", ".google.com.nf", ".google.com.ng", ".google.com.ni", - ".google.ne", - ".google.nl", - ".google.no", ".google.com.np", - ".google.nr", - ".google.nu", - ".google.co.nz", ".google.com.om", ".google.com.pa", ".google.com.pe", ".google.com.pg", ".google.com.ph", ".google.com.pk", - ".google.pl", - ".google.pn", ".google.com.pr", - ".google.ps", - ".google.pt", ".google.com.py", ".google.com.qa", - ".google.ro", - ".google.ru", - ".google.rw", ".google.com.sa", ".google.com.sb", - ".google.sc", - ".google.se", ".google.com.sg", - ".google.sh", - ".google.si", - ".google.sk", ".google.com.sl", - ".google.sn", - ".google.so", - ".google.sm", - ".google.sr", - ".google.st", ".google.com.sv", - ".google.td", - ".google.tg", - ".google.co.th", ".google.com.tj", - ".google.tk", - ".google.tl", - ".google.tm", - ".google.tn", - ".google.to", ".google.com.tr", - ".google.tt", ".google.com.tw", - ".google.co.tz", ".google.com.ua", + ".google.com.uy", + ".google.com.vc", + ".google.com.vn", + ".google.co.mz", + ".google.co.nz", + ".google.co.th", + ".google.co.tz", ".google.co.ug", ".google.co.uk", - ".google.com.uy", ".google.co.uz", - ".google.com.vc", ".google.co.ve", - ".google.vg", ".google.co.vi", - ".google.com.vn", - ".google.vu", - ".google.ws", - ".google.rs", ".google.co.za", ".google.co.zm", ".google.co.zw", - ".google.cat", - "www.google.com", + "google.cr", + "google.cu", + ".google.cv", + "google.cv", + "google.cx", + "google.cy", + ".google.cz", + "google.cz", + ".google.de", + "google.de", + ".google.dj", + "google.dj", + ".google.dk", + "google.dk", + ".google.dm", + "google.dm", + "google.do", + "googledrive.com", + ".google.dz", + "google.dz", + "googleearth.com", + "google.ec", + "googlee.com", + ".google.ee", + "google.ee", + "google.eg", + ".google.es", + "google.es", + "google.et", + "google.eu", + ".google.fi", + "google.fi", + "google.fj", + ".google.fm", + "google.fm", + ".google.fr", + "google.fr", + ".google.ga", + "google.ga", + ".google.ge", + "google.ge", + "google.gf", + ".google.gg", + "google.gg", + "google.gh", + "google.gi", + ".google.gl", + "google.gl", + ".google.gm", + "google.gm", + ".google.gp", + "google.gp", + ".google.gr", + "google.gr", + "google.gt", + ".google.gy", + "google.gy", + "google.hk", + ".google.hn", + "google.hn", + ".google.hr", + "google.hr", + ".google.ht", + "google.ht", + ".google.hu", + "google.hu", + "google.id", + ".google.ie", + "google.ie", + "google.il", + ".google.im", + "google.im", + "google.in", + "google.io", + ".google.iq", + "google.iq", + ".google.is", + "google.is", + ".google.it", + "google.it", + ".google.je", + "google.je", + "google.jm", + ".google.jo", + "google.jo", + "google.jp", + "google.ke", + ".google.kg", + "google.kg", + "google.kh", + ".google.ki", + "google.ki", + "google.kr", + "google.kw", + ".google.kz", + "google.kz", + ".google.la", + "google.la", + "google.lb", + "google.lc", + ".google.li", + "google.li", + ".google.lk", + "google.lk", + "google.ls", + ".google.lt", + "google.lt", + ".google.lu", + "google.lu", + ".google.lv", + "google.lv", + "google.ly", + "google.ma", + "googlemail.com", + "googlemaps.com", + ".google.md", + "google.md", + ".google.me", + "google.me", + ".google.mg", + "google.mg", + ".google.mk", + "google.mk", + ".google.ml", + "google.ml", + "google.mm", + ".google.mn", + "google.mn", + ".google.ms", + "google.ms", + "google.mt", + ".google.mu", + "google.mu", + ".google.mv", + "google.mv", + ".google.mw", + "google.mw", + "google.mx", + "google.my", + "google.mz", + "google.na", + ".google.ne", + "google.ne", + "google.net", + "google.nf", + "google.ng", + "google.ni", + ".google.nl", + "google.nl", + ".google.no", + "google.no", + "google.np", + ".google.nr", + "google.nr", + ".google.nu", + "google.nu", + "google.nz", + "google.om", + "google.org", + "google.pa", + "googlepagecreator.com", + "google.pe", + "google.pg", + "google.ph", + "google.pk", + ".google.pl", + "google.pl", + ".google.pn", + "google.pn", + "google.pr", + ".google.ps", + "google.ps", + ".google.pt", + "google.pt", + "google.py", + "google.qa", + ".google.ro", + "google.ro", + ".google.rs", + "google.rs", + ".google.ru", + "google.ru", + ".google.rw", + "google.rw", + "google.sa", + "google.sb", + ".google.sc", + "google.sc", + "googlescholar.com", + ".google.se", + "google.se", + "google.sg", + ".google.sh", + "google.sh", + ".google.si", + "google.si", + ".google.sk", + "google.sk", + "google.sl", + ".google.sm", + "google.sm", + ".google.sn", + "google.sn", + ".google.so", + "google.so", + "googlesource.com", + ".google.sr", + "google.sr", + ".google.st", + "google.st", + "google.sv", + "googlesyndication.com", + "googletagmanager.com", + ".google.td", + "google.td", + ".google.tg", + "google.tg", + "google.th", + "google.tj", + ".google.tk", + "google.tk", + ".google.tl", + "google.tl", + ".google.tm", + "google.tm", + ".google.tn", + "google.tn", + ".google.to", + "google.to", + "google.tr", + ".google.tt", + "google.tt", + "google.tw", + "google.tz", + "google.ua", + "google.ug", + "google.uk", + "google.us", + "googleusercontent.com", + "google.uy", + "google.uz", + "google.vc", + "google.ve", + ".google.vg", + "google.vg", + "google.vi", + "google.vn", + ".google.vu", + "google.vu", + ".google.ws", + "google.ws", + "google.za", + "google.zm", + "google.zw", + "googlr.com", + "goolge.com", + "gooogle.com", + "gv.com", + "keyhole.com", + "like.com", + "localguidesconnect.com", + "madewithcode.com", + "panoramio.com", + "picasa.com", + "urchin.com", + "waze.com", + "whatbrowser.org", + "withgoogle.com", "www.google.ad", "www.google.ae", - "www.google.com.af", - "www.google.com.ag", - "www.google.com.ai", "www.google.al", "www.google.am", - "www.google.co.ao", - "www.google.com.ar", "www.google.as", "www.google.at", - "www.google.com.au", "www.google.az", "www.google.ba", - "www.google.com.bd", "www.google.be", "www.google.bf", "www.google.bg", - "www.google.com.bh", "www.google.bi", "www.google.bj", - "www.google.com.bn", - "www.google.com.bo", - "www.google.com.br", "www.google.bs", "www.google.bt", - "www.google.co.bw", "www.google.by", - "www.google.com.bz", + "www.google.ca", "www.google.ca", "www.google.cd", "www.google.cf", "www.google.cg", "www.google.ch", "www.google.ci", - "www.google.co.ck", "www.google.cl", "www.google.cm", "www.google.cn", - "www.google.com.co", + "www.google.co.ao", + "www.google.co.bw", + "www.google.co.ck", "www.google.co.cr", - "www.google.com.cu", - "www.google.cv", - "www.google.com.cy", - "www.google.cz", - "www.google.de", - "www.google.dj", - "www.google.dk", - "www.google.dm", - "www.google.com.do", - "www.google.dz", - "www.google.com.ec", - "www.google.ee", - "www.google.com.eg", - "www.google.es", - "www.google.com.et", - "www.google.fi", - "www.google.com.fj", - "www.google.fm", - "www.google.fr", - "www.google.ga", - "www.google.ge", - "www.google.gg", - "www.google.com.gh", - "www.google.com.gi", - "www.google.gl", - "www.google.gm", - "www.google.gp", - "www.google.gr", - "www.google.com.gt", - "www.google.gy", - "www.google.com.hk", - "www.google.hn", - "www.google.hr", - "www.google.ht", - "www.google.hu", "www.google.co.id", - "www.google.ie", "www.google.co.il", - "www.google.im", "www.google.co.in", - "www.google.iq", - "www.google.is", - "www.google.it", - "www.google.je", - "www.google.com.jm", - "www.google.jo", "www.google.co.jp", "www.google.co.ke", - "www.google.com.kh", - "www.google.ki", - "www.google.kg", "www.google.co.kr", - "www.google.com.kw", - "www.google.kz", - "www.google.la", - "www.google.com.lb", - "www.google.li", - "www.google.lk", "www.google.co.ls", - "www.google.lt", - "www.google.lu", - "www.google.lv", - "www.google.com.ly", + "www.google.com", "www.google.co.ma", - "www.google.md", - "www.google.me", - "www.google.mg", - "www.google.mk", - "www.google.ml", + "www.google.com.af", + "www.google.com.ag", + "www.google.com.ai", + "www.google.com.ar", + "www.google.com.au", + "www.google.com.bd", + "www.google.com.bh", + "www.google.com.bn", + "www.google.com.bo", + "www.google.com.br", + "www.google.com.bz", + "www.google.com.co", + "www.google.com.cu", + "www.google.com.cy", + "www.google.com.do", + "www.google.com.ec", + "www.google.com.eg", + "www.google.com.et", + "www.google.com.fj", + "www.google.com.gh", + "www.google.com.gi", + "www.google.com.gt", + "www.google.com.hk", + "www.google.com.jm", + "www.google.com.kh", + "www.google.com.kw", + "www.google.com.lb", + "www.google.com.ly", "www.google.com.mm", - "www.google.mn", - "www.google.ms", "www.google.com.mt", - "www.google.mu", - "www.google.mv", - "www.google.mw", "www.google.com.mx", "www.google.com.my", - "www.google.co.mz", "www.google.com.na", "www.google.com.nf", "www.google.com.ng", "www.google.com.ni", - "www.google.ne", - "www.google.nl", - "www.google.no", "www.google.com.np", - "www.google.nr", - "www.google.nu", - "www.google.co.nz", "www.google.com.om", "www.google.com.pa", "www.google.com.pe", "www.google.com.pg", "www.google.com.ph", "www.google.com.pk", - "www.google.pl", - "www.google.pn", "www.google.com.pr", - "www.google.ps", - "www.google.pt", "www.google.com.py", "www.google.com.qa", - "www.google.ro", - "www.google.ru", - "www.google.rw", "www.google.com.sa", "www.google.com.sb", + "www.google.com.sg", + "www.google.com.sl", + "www.google.com.sv", + "www.google.com.tj", + "www.google.com.tr", + "www.google.com.tw", + "www.google.com.ua", + "www.google.com.uy", + "www.google.com.vc", + "www.google.com.vn", + "www.google.co.mz", + "www.google.co.nz", + "www.google.co.th", + "www.google.co.tz", + "www.google.co.ug", + "www.google.co.uk", + "www.google.co.uz", + "www.google.co.ve", + "www.google.co.vi", + "www.google.co.za", + "www.google.co.zm", + "www.google.co.zw", + "www.google.cv", + "www.google.cz", + "www.google.de", + "www.google.dj", + "www.google.dk", + "www.google.dm", + "www.google.dz", + "www.google.ee", + "www.google.es", + "www.google.fi", + "www.google.fm", + "www.google.fr", + "www.google.ga", + "www.google.ge", + "www.google.gg", + "www.google.gl", + "www.google.gm", + "www.google.gp", + "www.google.gr", + "www.google.gy", + "www.google.hn", + "www.google.hr", + "www.google.ht", + "www.google.hu", + "www.google.ie", + "www.google.im", + "www.google.iq", + "www.google.is", + "www.google.it", + "www.google.je", + "www.google.jo", + "www.google.kg", + "www.google.ki", + "www.google.kz", + "www.google.la", + "www.google.li", + "www.google.lk", + "www.google.lt", + "www.google.lu", + "www.google.lv", + "www.google.md", + "www.google.me", + "www.google.mg", + "www.google.mk", + "www.google.ml", + "www.google.mn", + "www.google.ms", + "www.google.mu", + "www.google.mv", + "www.google.mw", + "www.google.ne", + "www.google.nl", + "www.google.no", + "www.google.nr", + "www.google.nu", + "www.google.pl", + "www.google.pn", + "www.google.ps", + "www.google.pt", + "www.google.ro", + "www.google.rs", + "www.google.ru", + "www.google.rw", "www.google.sc", "www.google.se", - "www.google.com.sg", "www.google.sh", "www.google.si", "www.google.sk", - "www.google.com.sl", + "www.google.sm", "www.google.sn", "www.google.so", - "www.google.sm", "www.google.sr", "www.google.st", - "www.google.com.sv", "www.google.td", "www.google.tg", - "www.google.co.th", - "www.google.com.tj", "www.google.tk", "www.google.tl", "www.google.tm", "www.google.tn", "www.google.to", - "www.google.com.tr", "www.google.tt", - "www.google.com.tw", - "www.google.co.tz", - "www.google.com.ua", - "www.google.co.ug", - "www.google.co.uk", - "www.google.com.uy", - "www.google.co.uz", - "www.google.com.vc", - "www.google.co.ve", "www.google.vg", - "www.google.co.vi", - "www.google.com.vn", "www.google.vu", "www.google.ws", - "www.google.rs", - "www.google.co.za", - "www.google.co.zm", - "www.google.co.zw", - "www.google.cat" + "youtu.be", + "youtube.com", + "youtubeeducation.com", + "youtubegaming.com", + "youtube-nocookie.com", + "yt.be", + "ytimg.com" ] } From 02b4bf8a52c2b37c24bd25ed548df59a626d3954 Mon Sep 17 00:00:00 2001 From: rmarsollier Date: Mon, 19 Jun 2017 09:09:33 +0200 Subject: [PATCH 2/3] adding wikipedia scrapper for google domains --- tools/generate-google.py | 53 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 tools/generate-google.py diff --git a/tools/generate-google.py b/tools/generate-google.py new file mode 100644 index 0000000..e5d9299 --- /dev/null +++ b/tools/generate-google.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 + +from lxml import etree +from bs4 import BeautifulSoup +import datetime +import urllib.request, urllib.parse, urllib.error +import json + +#webpage = urllib.request.urlopen("https://en.wikipedia.org/w/index.php?title=List_of_Google_domains&printable=yes") +webpage = urllib.request.urlopen("https://en.wikipedia.org/w/index.php?title=List_of_Google_domains") +soup = BeautifulSoup(webpage,'html.parser') + +tables = soup.find_all("table", { "class" : "wikitable sortable" }) +#print(tables) + +gdomains = [] +for tabl in tables : + + for row in tabl.findAll("tr"): + cells = row.findAll('td') + if len(cells) == 4: + domain = cells[2].find_all(text=True) + + if len(domain) is 1 : + domain = "{}".format(domain[0]) + elif len(domain) is 2 : + domain = "{}{}".format(domain[0], domain[1]) + elif len(domain) is 3 and not "[" in domain[2]: + domain = "{}{}".format(domain[0], domain[2]) + else: + domain = "{}{}".format(domain[0], domain[1]) + + print(domain) + + gdomains.append(domain) +#print(gdomains) +gdomains = sorted(set(gdomains)) + +google_warninglist = {} +version = int(datetime.date.today().strftime('%Y%m%d')) + +google_warninglist['description'] = "Event contains one or more entries from the google owned domains." +d = datetime.datetime.now() +google_warninglist['version'] = version +google_warninglist['name'] = "Known Google domains" +google_warninglist['list'] = [] +google_warninglist['matching_attributes'] = ['hostname', 'domain'] + +for site in gdomains: + #v = str(site).split(',')[1] + google_warninglist['list'].append(site) +google_warninglist['list'] = sorted(set(google_warninglist['list'])) +#print(json.dumps(google_warninglist)) From 70b0742c45699ebfd72c14147e1b838d97e71b6d Mon Sep 17 00:00:00 2001 From: rmarsollier Date: Mon, 19 Jun 2017 09:19:21 +0200 Subject: [PATCH 3/3] solving last problem with google domain list --- lists/google/list.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lists/google/list.json b/lists/google/list.json index 9722004..b728b7d 100644 --- a/lists/google/list.json +++ b/lists/google/list.json @@ -97,6 +97,7 @@ ".google.by", "google.by", "google.bz", + ".google.ca", "google.ca", ".google.cat", "google.cat", @@ -490,7 +491,6 @@ "www.google.bt", "www.google.by", "www.google.ca", - "www.google.ca", "www.google.cd", "www.google.cf", "www.google.cg",