From bc05ddcdd030188279aa709cec4cc6702591071e Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Thu, 16 Mar 2017 16:13:18 +0100 Subject: [PATCH 01/10] Fixed #25 adding more URL shorteners --- lists/url-shortener/list.json | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lists/url-shortener/list.json b/lists/url-shortener/list.json index 9c0d163..937f447 100644 --- a/lists/url-shortener/list.json +++ b/lists/url-shortener/list.json @@ -1,6 +1,6 @@ { "name": "List of known URL Shorteners domains", - "version": 2, + "version": 3, "description": "Event contains one or more entries of known Shorteners domains", "matching_attributes": [ "domain", @@ -16,6 +16,7 @@ "adflav.com", "adf.ly", "aka.gr", + "amzn.to", "bc.vc", "bee4.biz", "bit.do", @@ -25,6 +26,7 @@ "buzurl.com", "cektkp.com", "cur.lv", + "cut.by", "cutt.us", "db.tt", "dft.ba", @@ -35,6 +37,7 @@ "golinks.co", "goo.gl", "hit.my", + "hyperurl.co", "id.tl", "is.gd", "ity.im", @@ -56,8 +59,11 @@ "scrnch.me", "shortquik.com", "sk.gy", + "smarturl.it", + "snip.ly", "su.pr", "t.co", + "tiny.cc", "tinyarrows.com", "tinyurl.com", "tota2.com", @@ -68,10 +74,12 @@ "u.to", "v.gd", "vzturl.com", + "wp.me", "x.co", "xlinkz.info", "xtu.me", "yourls.org", + "youtu.be", "yu2.it", "zpag.es" ] From c66018c9251c986b51dffc0cde400d8dba04817e Mon Sep 17 00:00:00 2001 From: RbN Date: Mon, 15 May 2017 20:40:24 +0200 Subject: [PATCH 02/10] Adding sha224 d14a028c2a3a2bc9476102bb288234c415a2b01f828ea62ac5b3e42f is a sha224, let's use it. --- lists/empty-hashes/list.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lists/empty-hashes/list.json b/lists/empty-hashes/list.json index d8530e2..0b050df 100644 --- a/lists/empty-hashes/list.json +++ b/lists/empty-hashes/list.json @@ -5,10 +5,12 @@ "matching_attributes": [ "md5", "sha1", + "sha224", "sha256", "sha512", "filename|md5", "filename|sha1", + "filename|sha224", "filename|sha256", "filename|sha512" ], From 40fa1df9b6b4bc32021ad4c2116948c5b8479c0a Mon Sep 17 00:00:00 2001 From: RbN Date: Mon, 15 May 2017 21:29:33 +0200 Subject: [PATCH 03/10] Adding domains of #23 --- lists/whats-my-ip/list.json | 184 +++++++++++++++++++++++++++++++++++- 1 file changed, 182 insertions(+), 2 deletions(-) diff --git a/lists/whats-my-ip/list.json b/lists/whats-my-ip/list.json index e9b2b57..a4eff39 100644 --- a/lists/whats-my-ip/list.json +++ b/lists/whats-my-ip/list.json @@ -10,36 +10,216 @@ "url" ], "list": [ + "2ip.ru", + "2ip.tools", + "aboutmyip.com", "api.ipify.org", + "api.wipmania.com", + "bearsmyip.com", + "checkip.dns.he.net", "checkip.dyndns.com", + "checkip.dyndns.es", + "checkip.dyndns.info", + "checkip.dyndns.org", + "checkip.narak.com", + "checkmyip.com", + "check-my-ip.net", + "cmyip.com", + "cmyip.net", + "crymyip.com", + "curlmyip.com", + "curlmyip.net", + "dawhois.com", + "dnswatch.info", + "dpool.sina.com.cn", + "findmyipaddress.com", + "findmyip.org", + "formyip.com", + "freegeoip.net", + "geoip.co.uk", + "geoiptool.com", + "geoip.vmn.net", + "get-myip.com", + "getmyip.co.uk", + "getmyip.org", "icanhazip.com", "ifcfg.me", "ifconfig.co", + "ifconfig.me", + "inet-ip.info", + "ip138.com", + "ip-1.com", + "ip2location.com", + "ip2nation.com", "ip4.me", + "ip-addr.es", + "ip-address.cc", + "ipaddresscheck.com", + "ipaddress.com", + "ipaddress.org", + "ip-address.ru", + "ip-adress.com", "ip-adress.eu", + "ip.amulex.com", "ip.anysrc.net", + "ip-api.com", + "ip.cctv.pk", + "ipchecker.info", + "ip-check.info", "ipchicken.com", + "ip.chinaz.com", + "ip.cn", + "ip-detect.net", "ipecho.net", + "ipify.org", + "ipinfodb.com", "ipinfo.info", + "ipinfo.io", + "ip-info.org", + "ip-info.xyz", + "ipleak.net", + "iplocation.net", + "iplogger.ru", + "ipmonkey.com", + "ip.my-proxy.com", + "ip-ping.ru", + "ip-score.com", "ip-secrets.com", + "ip.taobao.com", + "ip.telize.com", + "ip.tool.la", + "iptrackeronline.com", + "ip.tyk.nu", + "ip.webmasterhome.cn", + "ip-who-is.com", + "ip-whois.net", + "j.maxmind.com", + "l2.io", + "meip.eu", + "meuip.net.br", + "mon-ip.com", + "mycamip.com", "myexternalip.com", + "myglobalip.com", "myipaddress.com", "my-ip-address.net", + "myip.am", + "myip.by", + "myip.cc", + "myip.cf", + "myip.ch", + "myip.cn", + "myip.co.il", + "myip.com.br", + "myip.com.tw", + "myip.com.ua", + "myip.co.nz", + "myip.cz", + "myip.dk", + "myip.dnsdynamic.org", + "myip.dnsomatic.com", + "myip.dramor.net", + "myip.easylife.tw", + "myip.es", + "myip.eu", + "myip.fi", + "myip.gr", + "myip.gratis", + "myip.heltech.se", + "myip.ht", + "myip.info", + "myipinfo.net", + "myip.io", + "myip.is", + "myip.israel.net", + "myip.jacware.com", + "myip.knet.ca", + "myip.kz", + "myip.ma", + "myip.ms", + "myip.mudfish.net", + "myip.mx", + "myip.mysau.com.au", + "myip.net", + "myip.nl", + "myip.nmonitoring.com", + "myip.northstate.net", + "myipnow.com", + "myip.nu", "myipnumber.com", + "myiponline.com", + "myip.opendns.com", + "myip.ozymo.com", + "myip.report", + "myip.rs.sr", + "myip.ru", + "myip.sdu.dk", + "myip.se", + "myip.shorty.org", + "myip.si", + "myip.surfeasy.com", + "myip.telespex.com", + "myip.tk", + "myip.tw", + "myip.ua.edu", + "myip.uconn.edu", + "myip.v6shell.org", + "myip.zone", + "mylocation.org", + "qualmeuip.com.br", + "readip.info", + "shmyip.com", + "show-ip.com", + "showipinfo.net", + "showip.net", + "showmemyip.com", + "showmyipaddress.com", + "showmyipaddress.eu", + "showmyip.com", + "showmyip.com.ar", + "showmyip.co.uk", + "show-my-ip.de", + "showmyip.gr", + "showmyipnow.com", + "smart-ip.net", + "tell-my-ip.com", + "tracemyip.com", + "tracemyip.org", + "trackip.net", + "ultratools.com", + "utrace.de", "vermiip.es", + "vinflag.com", "whatismybrowser.com", "whatismyipaddress.com", + "whatismyip.akamai.com", + "whatismyip.ca", "whatismyip.com", "whatismyip.com.br", + "whatismyip.everdot.org", "whatismyip.li", "whatismyip.net", "whatismyip.org", "whatismypublicip.com", + "whatmyip.us", + "whatsmyipaddress.com", + "whatsmyipaddress.net", + "whats-my-ip-address.org", + "w.hatsmyip.com", "whatsmyip.ie", "whatsmyip.net", "whatsmyip.org", + "whatsmyip.us", + "whereisip.net", + "whoer.me", + "whoer.net", "wtfismyip.com", - "ip-score.com", - "xmyip.com" + "www.ip[.]cn", + "xmyip.com", + "yougetsignal.com", + "youip.net", + "your-ip-address.com", + "your-ip-fast.com", + "yourip.us" ] } From b15b1de53ed8e0ccbc4548db4737e23cb16a3713 Mon Sep 17 00:00:00 2001 From: RbN Date: Mon, 15 May 2017 21:32:20 +0200 Subject: [PATCH 04/10] Fix typo --- lists/whats-my-ip/list.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lists/whats-my-ip/list.json b/lists/whats-my-ip/list.json index a4eff39..45c79b7 100644 --- a/lists/whats-my-ip/list.json +++ b/lists/whats-my-ip/list.json @@ -214,7 +214,7 @@ "whoer.me", "whoer.net", "wtfismyip.com", - "www.ip[.]cn", + "www.ip.cn", "xmyip.com", "yougetsignal.com", "youip.net", From 93b6a5c1204ce7594836297159006e0626a6bbbe Mon Sep 17 00:00:00 2001 From: rmarsollier Date: Mon, 19 Jun 2017 09:08:58 +0200 Subject: [PATCH 05/10] importing google domains from wikipedia --- lists/google/list.json | 775 ++++++++++++++++++++++++++++------------- 1 file changed, 527 insertions(+), 248 deletions(-) diff --git a/lists/google/list.json b/lists/google/list.json index 95e23cc..9722004 100644 --- a/lists/google/list.json +++ b/lists/google/list.json @@ -1,6 +1,6 @@ { "name": "List of known google domains", - "version": 2, + "version": 3, "description": "Event contains one or more entries of known google domains", "matching_attributes": [ "domain", @@ -8,391 +8,670 @@ "domain|ip" ], "list": [ - ".google.com", + "1e100.net", + "466453.com", + "abc.xyz", + "admob.com", + "adsense.com", + "advertisercommunity.com", + "adwords.com", + "ai.google", + "android.com", + "blogger.com", + "blog.google", + "blogspot.com", + "capitalg.com", + "chromebook.com", + "chromecast.com", + "chrome.com", + "chromium.org", + "cobrasearch.com", + "com.google", + "domains.google", + "doubleclickbygoogle.com", + "doubleclick.com", + "duck.com", + "elgoog.im", + "feedburner.com", + "foofle.com", + "froogle.com", + "g.co", + "ggpht.com", + "gmail.com", + "gmodules.com", + "gogle.com", + "gogole.com", + "googel.com", + "googil.com", + "goo.gl", + "googl.com", + "google.ac", ".google.ad", + "google.ad", + "googleadservices.com", ".google.ae", + "google.ae", + "google.af", + "google.ag", + "google.ai", + ".google.al", + "google.al", + ".google.am", + "google.am", + "google-analytics.com", + "google.ao", + "googleapis.com", + "googleapps.com", + "google.ar", + "googlearth.com", + ".google.as", + "google.as", + ".google.at", + "google.at", + "google.au", + ".google.az", + "google.az", + ".google.ba", + "google.ba", + "google.bd", + ".google.be", + "google.be", + ".google.bf", + "google.bf", + ".google.bg", + "google.bg", + "google.bh", + ".google.bi", + "google.bi", + ".google.bj", + "google.bj", + "google.bn", + "google.bo", + "googlebot.com", + "google.br", + ".google.bs", + "google.bs", + ".google.bt", + "google.bt", + "google.bw", + ".google.by", + "google.by", + "google.bz", + "google.ca", + ".google.cat", + "google.cat", + "google.cc", + ".google.cd", + "google.cd", + ".google.cf", + "google.cf", + ".google.cg", + "google.cg", + ".google.ch", + "google.ch", + ".google.ci", + "google.ci", + "google.ck", + ".google.cl", + "google.cl", + ".google.cm", + "google.cm", + ".google.cn", + "google.cn", + "google.co", + ".google.co.ao", + ".google.co.bw", + ".google.co.ck", + ".google.co.cr", + "googlecode.com", + ".google.co.id", + ".google.co.il", + ".google.co.in", + ".google.co.jp", + ".google.co.ke", + ".google.co.kr", + ".google.co.ls", + ".google.com", + "google.com", + ".google.co.ma", ".google.com.af", ".google.com.ag", ".google.com.ai", - ".google.al", - ".google.am", - ".google.co.ao", ".google.com.ar", - ".google.as", - ".google.at", ".google.com.au", - ".google.az", - ".google.ba", ".google.com.bd", - ".google.be", - ".google.bf", - ".google.bg", ".google.com.bh", - ".google.bi", - ".google.bj", ".google.com.bn", ".google.com.bo", ".google.com.br", - ".google.bs", - ".google.bt", - ".google.co.bw", - ".google.by", ".google.com.bz", - ".google.ca", - ".google.cd", - ".google.cf", - ".google.cg", - ".google.ch", - ".google.ci", - ".google.co.ck", - ".google.cl", - ".google.cm", - ".google.cn", ".google.com.co", - ".google.co.cr", ".google.com.cu", - ".google.cv", ".google.com.cy", - ".google.cz", - ".google.de", - ".google.dj", - ".google.dk", - ".google.dm", ".google.com.do", - ".google.dz", ".google.com.ec", - ".google.ee", ".google.com.eg", - ".google.es", ".google.com.et", - ".google.fi", ".google.com.fj", - ".google.fm", - ".google.fr", - ".google.ga", - ".google.ge", - ".google.gg", ".google.com.gh", ".google.com.gi", - ".google.gl", - ".google.gm", - ".google.gp", - ".google.gr", ".google.com.gt", - ".google.gy", ".google.com.hk", - ".google.hn", - ".google.hr", - ".google.ht", - ".google.hu", - ".google.co.id", - ".google.ie", - ".google.co.il", - ".google.im", - ".google.co.in", - ".google.iq", - ".google.is", - ".google.it", - ".google.je", ".google.com.jm", - ".google.jo", - ".google.co.jp", - ".google.co.ke", ".google.com.kh", - ".google.ki", - ".google.kg", - ".google.co.kr", ".google.com.kw", - ".google.kz", - ".google.la", ".google.com.lb", - ".google.li", - ".google.lk", - ".google.co.ls", - ".google.lt", - ".google.lu", - ".google.lv", ".google.com.ly", - ".google.co.ma", - ".google.md", - ".google.me", - ".google.mg", - ".google.mk", - ".google.ml", + "googlecommerce.com", ".google.com.mm", - ".google.mn", - ".google.ms", ".google.com.mt", - ".google.mu", - ".google.mv", - ".google.mw", ".google.com.mx", ".google.com.my", - ".google.co.mz", ".google.com.na", ".google.com.nf", ".google.com.ng", ".google.com.ni", - ".google.ne", - ".google.nl", - ".google.no", ".google.com.np", - ".google.nr", - ".google.nu", - ".google.co.nz", ".google.com.om", ".google.com.pa", ".google.com.pe", ".google.com.pg", ".google.com.ph", ".google.com.pk", - ".google.pl", - ".google.pn", ".google.com.pr", - ".google.ps", - ".google.pt", ".google.com.py", ".google.com.qa", - ".google.ro", - ".google.ru", - ".google.rw", ".google.com.sa", ".google.com.sb", - ".google.sc", - ".google.se", ".google.com.sg", - ".google.sh", - ".google.si", - ".google.sk", ".google.com.sl", - ".google.sn", - ".google.so", - ".google.sm", - ".google.sr", - ".google.st", ".google.com.sv", - ".google.td", - ".google.tg", - ".google.co.th", ".google.com.tj", - ".google.tk", - ".google.tl", - ".google.tm", - ".google.tn", - ".google.to", ".google.com.tr", - ".google.tt", ".google.com.tw", - ".google.co.tz", ".google.com.ua", + ".google.com.uy", + ".google.com.vc", + ".google.com.vn", + ".google.co.mz", + ".google.co.nz", + ".google.co.th", + ".google.co.tz", ".google.co.ug", ".google.co.uk", - ".google.com.uy", ".google.co.uz", - ".google.com.vc", ".google.co.ve", - ".google.vg", ".google.co.vi", - ".google.com.vn", - ".google.vu", - ".google.ws", - ".google.rs", ".google.co.za", ".google.co.zm", ".google.co.zw", - ".google.cat", - "www.google.com", + "google.cr", + "google.cu", + ".google.cv", + "google.cv", + "google.cx", + "google.cy", + ".google.cz", + "google.cz", + ".google.de", + "google.de", + ".google.dj", + "google.dj", + ".google.dk", + "google.dk", + ".google.dm", + "google.dm", + "google.do", + "googledrive.com", + ".google.dz", + "google.dz", + "googleearth.com", + "google.ec", + "googlee.com", + ".google.ee", + "google.ee", + "google.eg", + ".google.es", + "google.es", + "google.et", + "google.eu", + ".google.fi", + "google.fi", + "google.fj", + ".google.fm", + "google.fm", + ".google.fr", + "google.fr", + ".google.ga", + "google.ga", + ".google.ge", + "google.ge", + "google.gf", + ".google.gg", + "google.gg", + "google.gh", + "google.gi", + ".google.gl", + "google.gl", + ".google.gm", + "google.gm", + ".google.gp", + "google.gp", + ".google.gr", + "google.gr", + "google.gt", + ".google.gy", + "google.gy", + "google.hk", + ".google.hn", + "google.hn", + ".google.hr", + "google.hr", + ".google.ht", + "google.ht", + ".google.hu", + "google.hu", + "google.id", + ".google.ie", + "google.ie", + "google.il", + ".google.im", + "google.im", + "google.in", + "google.io", + ".google.iq", + "google.iq", + ".google.is", + "google.is", + ".google.it", + "google.it", + ".google.je", + "google.je", + "google.jm", + ".google.jo", + "google.jo", + "google.jp", + "google.ke", + ".google.kg", + "google.kg", + "google.kh", + ".google.ki", + "google.ki", + "google.kr", + "google.kw", + ".google.kz", + "google.kz", + ".google.la", + "google.la", + "google.lb", + "google.lc", + ".google.li", + "google.li", + ".google.lk", + "google.lk", + "google.ls", + ".google.lt", + "google.lt", + ".google.lu", + "google.lu", + ".google.lv", + "google.lv", + "google.ly", + "google.ma", + "googlemail.com", + "googlemaps.com", + ".google.md", + "google.md", + ".google.me", + "google.me", + ".google.mg", + "google.mg", + ".google.mk", + "google.mk", + ".google.ml", + "google.ml", + "google.mm", + ".google.mn", + "google.mn", + ".google.ms", + "google.ms", + "google.mt", + ".google.mu", + "google.mu", + ".google.mv", + "google.mv", + ".google.mw", + "google.mw", + "google.mx", + "google.my", + "google.mz", + "google.na", + ".google.ne", + "google.ne", + "google.net", + "google.nf", + "google.ng", + "google.ni", + ".google.nl", + "google.nl", + ".google.no", + "google.no", + "google.np", + ".google.nr", + "google.nr", + ".google.nu", + "google.nu", + "google.nz", + "google.om", + "google.org", + "google.pa", + "googlepagecreator.com", + "google.pe", + "google.pg", + "google.ph", + "google.pk", + ".google.pl", + "google.pl", + ".google.pn", + "google.pn", + "google.pr", + ".google.ps", + "google.ps", + ".google.pt", + "google.pt", + "google.py", + "google.qa", + ".google.ro", + "google.ro", + ".google.rs", + "google.rs", + ".google.ru", + "google.ru", + ".google.rw", + "google.rw", + "google.sa", + "google.sb", + ".google.sc", + "google.sc", + "googlescholar.com", + ".google.se", + "google.se", + "google.sg", + ".google.sh", + "google.sh", + ".google.si", + "google.si", + ".google.sk", + "google.sk", + "google.sl", + ".google.sm", + "google.sm", + ".google.sn", + "google.sn", + ".google.so", + "google.so", + "googlesource.com", + ".google.sr", + "google.sr", + ".google.st", + "google.st", + "google.sv", + "googlesyndication.com", + "googletagmanager.com", + ".google.td", + "google.td", + ".google.tg", + "google.tg", + "google.th", + "google.tj", + ".google.tk", + "google.tk", + ".google.tl", + "google.tl", + ".google.tm", + "google.tm", + ".google.tn", + "google.tn", + ".google.to", + "google.to", + "google.tr", + ".google.tt", + "google.tt", + "google.tw", + "google.tz", + "google.ua", + "google.ug", + "google.uk", + "google.us", + "googleusercontent.com", + "google.uy", + "google.uz", + "google.vc", + "google.ve", + ".google.vg", + "google.vg", + "google.vi", + "google.vn", + ".google.vu", + "google.vu", + ".google.ws", + "google.ws", + "google.za", + "google.zm", + "google.zw", + "googlr.com", + "goolge.com", + "gooogle.com", + "gv.com", + "keyhole.com", + "like.com", + "localguidesconnect.com", + "madewithcode.com", + "panoramio.com", + "picasa.com", + "urchin.com", + "waze.com", + "whatbrowser.org", + "withgoogle.com", "www.google.ad", "www.google.ae", - "www.google.com.af", - "www.google.com.ag", - "www.google.com.ai", "www.google.al", "www.google.am", - "www.google.co.ao", - "www.google.com.ar", "www.google.as", "www.google.at", - "www.google.com.au", "www.google.az", "www.google.ba", - "www.google.com.bd", "www.google.be", "www.google.bf", "www.google.bg", - "www.google.com.bh", "www.google.bi", "www.google.bj", - "www.google.com.bn", - "www.google.com.bo", - "www.google.com.br", "www.google.bs", "www.google.bt", - "www.google.co.bw", "www.google.by", - "www.google.com.bz", + "www.google.ca", "www.google.ca", "www.google.cd", "www.google.cf", "www.google.cg", "www.google.ch", "www.google.ci", - "www.google.co.ck", "www.google.cl", "www.google.cm", "www.google.cn", - "www.google.com.co", + "www.google.co.ao", + "www.google.co.bw", + "www.google.co.ck", "www.google.co.cr", - "www.google.com.cu", - "www.google.cv", - "www.google.com.cy", - "www.google.cz", - "www.google.de", - "www.google.dj", - "www.google.dk", - "www.google.dm", - "www.google.com.do", - "www.google.dz", - "www.google.com.ec", - "www.google.ee", - "www.google.com.eg", - "www.google.es", - "www.google.com.et", - "www.google.fi", - "www.google.com.fj", - "www.google.fm", - "www.google.fr", - "www.google.ga", - "www.google.ge", - "www.google.gg", - "www.google.com.gh", - "www.google.com.gi", - "www.google.gl", - "www.google.gm", - "www.google.gp", - "www.google.gr", - "www.google.com.gt", - "www.google.gy", - "www.google.com.hk", - "www.google.hn", - "www.google.hr", - "www.google.ht", - "www.google.hu", "www.google.co.id", - "www.google.ie", "www.google.co.il", - "www.google.im", "www.google.co.in", - "www.google.iq", - "www.google.is", - "www.google.it", - "www.google.je", - "www.google.com.jm", - "www.google.jo", "www.google.co.jp", "www.google.co.ke", - "www.google.com.kh", - "www.google.ki", - "www.google.kg", "www.google.co.kr", - "www.google.com.kw", - "www.google.kz", - "www.google.la", - "www.google.com.lb", - "www.google.li", - "www.google.lk", "www.google.co.ls", - "www.google.lt", - "www.google.lu", - "www.google.lv", - "www.google.com.ly", + "www.google.com", "www.google.co.ma", - "www.google.md", - "www.google.me", - "www.google.mg", - "www.google.mk", - "www.google.ml", + "www.google.com.af", + "www.google.com.ag", + "www.google.com.ai", + "www.google.com.ar", + "www.google.com.au", + "www.google.com.bd", + "www.google.com.bh", + "www.google.com.bn", + "www.google.com.bo", + "www.google.com.br", + "www.google.com.bz", + "www.google.com.co", + "www.google.com.cu", + "www.google.com.cy", + "www.google.com.do", + "www.google.com.ec", + "www.google.com.eg", + "www.google.com.et", + "www.google.com.fj", + "www.google.com.gh", + "www.google.com.gi", + "www.google.com.gt", + "www.google.com.hk", + "www.google.com.jm", + "www.google.com.kh", + "www.google.com.kw", + "www.google.com.lb", + "www.google.com.ly", "www.google.com.mm", - "www.google.mn", - "www.google.ms", "www.google.com.mt", - "www.google.mu", - "www.google.mv", - "www.google.mw", "www.google.com.mx", "www.google.com.my", - "www.google.co.mz", "www.google.com.na", "www.google.com.nf", "www.google.com.ng", "www.google.com.ni", - "www.google.ne", - "www.google.nl", - "www.google.no", "www.google.com.np", - "www.google.nr", - "www.google.nu", - "www.google.co.nz", "www.google.com.om", "www.google.com.pa", "www.google.com.pe", "www.google.com.pg", "www.google.com.ph", "www.google.com.pk", - "www.google.pl", - "www.google.pn", "www.google.com.pr", - "www.google.ps", - "www.google.pt", "www.google.com.py", "www.google.com.qa", - "www.google.ro", - "www.google.ru", - "www.google.rw", "www.google.com.sa", "www.google.com.sb", + "www.google.com.sg", + "www.google.com.sl", + "www.google.com.sv", + "www.google.com.tj", + "www.google.com.tr", + "www.google.com.tw", + "www.google.com.ua", + "www.google.com.uy", + "www.google.com.vc", + "www.google.com.vn", + "www.google.co.mz", + "www.google.co.nz", + "www.google.co.th", + "www.google.co.tz", + "www.google.co.ug", + "www.google.co.uk", + "www.google.co.uz", + "www.google.co.ve", + "www.google.co.vi", + "www.google.co.za", + "www.google.co.zm", + "www.google.co.zw", + "www.google.cv", + "www.google.cz", + "www.google.de", + "www.google.dj", + "www.google.dk", + "www.google.dm", + "www.google.dz", + "www.google.ee", + "www.google.es", + "www.google.fi", + "www.google.fm", + "www.google.fr", + "www.google.ga", + "www.google.ge", + "www.google.gg", + "www.google.gl", + "www.google.gm", + "www.google.gp", + "www.google.gr", + "www.google.gy", + "www.google.hn", + "www.google.hr", + "www.google.ht", + "www.google.hu", + "www.google.ie", + "www.google.im", + "www.google.iq", + "www.google.is", + "www.google.it", + "www.google.je", + "www.google.jo", + "www.google.kg", + "www.google.ki", + "www.google.kz", + "www.google.la", + "www.google.li", + "www.google.lk", + "www.google.lt", + "www.google.lu", + "www.google.lv", + "www.google.md", + "www.google.me", + "www.google.mg", + "www.google.mk", + "www.google.ml", + "www.google.mn", + "www.google.ms", + "www.google.mu", + "www.google.mv", + "www.google.mw", + "www.google.ne", + "www.google.nl", + "www.google.no", + "www.google.nr", + "www.google.nu", + "www.google.pl", + "www.google.pn", + "www.google.ps", + "www.google.pt", + "www.google.ro", + "www.google.rs", + "www.google.ru", + "www.google.rw", "www.google.sc", "www.google.se", - "www.google.com.sg", "www.google.sh", "www.google.si", "www.google.sk", - "www.google.com.sl", + "www.google.sm", "www.google.sn", "www.google.so", - "www.google.sm", "www.google.sr", "www.google.st", - "www.google.com.sv", "www.google.td", "www.google.tg", - "www.google.co.th", - "www.google.com.tj", "www.google.tk", "www.google.tl", "www.google.tm", "www.google.tn", "www.google.to", - "www.google.com.tr", "www.google.tt", - "www.google.com.tw", - "www.google.co.tz", - "www.google.com.ua", - "www.google.co.ug", - "www.google.co.uk", - "www.google.com.uy", - "www.google.co.uz", - "www.google.com.vc", - "www.google.co.ve", "www.google.vg", - "www.google.co.vi", - "www.google.com.vn", "www.google.vu", "www.google.ws", - "www.google.rs", - "www.google.co.za", - "www.google.co.zm", - "www.google.co.zw", - "www.google.cat" + "youtu.be", + "youtube.com", + "youtubeeducation.com", + "youtubegaming.com", + "youtube-nocookie.com", + "yt.be", + "ytimg.com" ] } From 02b4bf8a52c2b37c24bd25ed548df59a626d3954 Mon Sep 17 00:00:00 2001 From: rmarsollier Date: Mon, 19 Jun 2017 09:09:33 +0200 Subject: [PATCH 06/10] adding wikipedia scrapper for google domains --- tools/generate-google.py | 53 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 tools/generate-google.py diff --git a/tools/generate-google.py b/tools/generate-google.py new file mode 100644 index 0000000..e5d9299 --- /dev/null +++ b/tools/generate-google.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 + +from lxml import etree +from bs4 import BeautifulSoup +import datetime +import urllib.request, urllib.parse, urllib.error +import json + +#webpage = urllib.request.urlopen("https://en.wikipedia.org/w/index.php?title=List_of_Google_domains&printable=yes") +webpage = urllib.request.urlopen("https://en.wikipedia.org/w/index.php?title=List_of_Google_domains") +soup = BeautifulSoup(webpage,'html.parser') + +tables = soup.find_all("table", { "class" : "wikitable sortable" }) +#print(tables) + +gdomains = [] +for tabl in tables : + + for row in tabl.findAll("tr"): + cells = row.findAll('td') + if len(cells) == 4: + domain = cells[2].find_all(text=True) + + if len(domain) is 1 : + domain = "{}".format(domain[0]) + elif len(domain) is 2 : + domain = "{}{}".format(domain[0], domain[1]) + elif len(domain) is 3 and not "[" in domain[2]: + domain = "{}{}".format(domain[0], domain[2]) + else: + domain = "{}{}".format(domain[0], domain[1]) + + print(domain) + + gdomains.append(domain) +#print(gdomains) +gdomains = sorted(set(gdomains)) + +google_warninglist = {} +version = int(datetime.date.today().strftime('%Y%m%d')) + +google_warninglist['description'] = "Event contains one or more entries from the google owned domains." +d = datetime.datetime.now() +google_warninglist['version'] = version +google_warninglist['name'] = "Known Google domains" +google_warninglist['list'] = [] +google_warninglist['matching_attributes'] = ['hostname', 'domain'] + +for site in gdomains: + #v = str(site).split(',')[1] + google_warninglist['list'].append(site) +google_warninglist['list'] = sorted(set(google_warninglist['list'])) +#print(json.dumps(google_warninglist)) From 70b0742c45699ebfd72c14147e1b838d97e71b6d Mon Sep 17 00:00:00 2001 From: rmarsollier Date: Mon, 19 Jun 2017 09:19:21 +0200 Subject: [PATCH 07/10] solving last problem with google domain list --- lists/google/list.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lists/google/list.json b/lists/google/list.json index 9722004..b728b7d 100644 --- a/lists/google/list.json +++ b/lists/google/list.json @@ -97,6 +97,7 @@ ".google.by", "google.by", "google.bz", + ".google.ca", "google.ca", ".google.cat", "google.cat", @@ -490,7 +491,6 @@ "www.google.bt", "www.google.by", "www.google.ca", - "www.google.ca", "www.google.cd", "www.google.cf", "www.google.cg", From 7c5a09a735a469dc6606e1ba8880c36a9554171e Mon Sep 17 00:00:00 2001 From: Andras Iklody Date: Fri, 4 Aug 2017 15:51:07 +0200 Subject: [PATCH 08/10] Changed warninglist from sting matches to hostname type --- lists/url-shortener/list.json | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lists/url-shortener/list.json b/lists/url-shortener/list.json index 937f447..2d07293 100644 --- a/lists/url-shortener/list.json +++ b/lists/url-shortener/list.json @@ -1,6 +1,6 @@ { "name": "List of known URL Shorteners domains", - "version": 3, + "version": 4, "description": "Event contains one or more entries of known Shorteners domains", "matching_attributes": [ "domain", @@ -9,6 +9,9 @@ "url", "uri" ], + "type": [ + "hostname" + ], "list": [ "1url.com", "adcraft.co", From 3d88354268b9105e35f5de00407d6e520028b658 Mon Sep 17 00:00:00 2001 From: ater49 <31342330+ater49@users.noreply.github.com> Date: Wed, 27 Sep 2017 21:41:36 +0200 Subject: [PATCH 09/10] Update list.json Just to add checkip.amazonaws.com into WarningList --- lists/whats-my-ip/list.json | 1 + 1 file changed, 1 insertion(+) diff --git a/lists/whats-my-ip/list.json b/lists/whats-my-ip/list.json index 45c79b7..ba338cf 100644 --- a/lists/whats-my-ip/list.json +++ b/lists/whats-my-ip/list.json @@ -16,6 +16,7 @@ "api.ipify.org", "api.wipmania.com", "bearsmyip.com", + "checkip.amazonaws.com" "checkip.dns.he.net", "checkip.dyndns.com", "checkip.dyndns.es", From fa3e724fdc447ecc7754ae32f0ded630a99b76c5 Mon Sep 17 00:00:00 2001 From: ater49 <31342330+ater49@users.noreply.github.com> Date: Thu, 28 Sep 2017 11:14:45 +0200 Subject: [PATCH 10/10] Update list.json Comma added to the line --- lists/whats-my-ip/list.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lists/whats-my-ip/list.json b/lists/whats-my-ip/list.json index ba338cf..0a95df7 100644 --- a/lists/whats-my-ip/list.json +++ b/lists/whats-my-ip/list.json @@ -16,7 +16,7 @@ "api.ipify.org", "api.wipmania.com", "bearsmyip.com", - "checkip.amazonaws.com" + "checkip.amazonaws.com", "checkip.dns.he.net", "checkip.dyndns.com", "checkip.dyndns.es",