MISP
/
misp-warninglists
mirror of https://github.com/MISP/misp-warninglists
2
0
Fork 0
Code Issues Releases Wiki Activity

Refactor last scripts, logging, central directory for downloads 

* Refactored generate_moz-top50.py
* Download all file to new /tmp file to centralize all downloads
* Add central logging to generators.log file
* Create Bash script that generates all warninglists
* Add /tmp folder and extra files to .gitignore
* Start adding exception handling in download_to_file and write_to_file
pull/162/head
Kevin Holvoet 2020-07-27 10:44:30 +02:00
parent 610292e90f
commit a9c163ef20
28 changed files with 24766 additions and 25065 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -1,4 +1,5 @@
# files download by generators # files download by generators
tmp/*
alexa_top-1m.csv.zip alexa_top-1m.csv.zip
amazon_ip-ranges.json amazon_ip-ranges.json
cisco_top-1m.csv.zip cisco_top-1m.csv.zip
@ -18,3 +19,5 @@ top500.pages.csv
top-1m.csv.zip top-1m.csv.zip
public-dns-nameservers.csv public-dns-nameservers.csv
ms-azure.json ms-azure.json
moz-top500.pages.csv
moz-top500.domains.csv

32
generate_all.sh Executable file
View File

@ -0,0 +1,32 @@
#!/bin/bash
set -e
set -x
pushd tools
python3 generate_alexa.py
python3 generate-amazon-aws.py
python3 generate-cisco.py
python3 generate-cloudflare.py
python3 generate-covid.py
python3 generate-crl-ip-list.py
python3 generate-disposal.py
# TODO: Google page on Wikipedia does not exist anymore
# Suggestion came to use a passivetotal whois search for org:Google LLC
#python3 generate-google.py > lists/google/list.json
python3 generate_majestic-million.py
python3 generate-microsoft-azure.py
python3 generate_mozilla_certificates.py
python3 generate_moz-top500.py
# Deprecate?
#python3 generate-office365-cn.py > lists/microsoft-office365-cn/list.json
python3 generate-office365.py
python3 generate-publicdns.py
python3 generate-tlds.py
python3 generate_tranco.py
python3 generate-university-domain-list.py
python3 generate-vpn.py
python3 generate-wikimedia.py
popd
./jq_all_the_things.sh

78
lists/amazon-aws/list.json
View File

@ -134,6 +134,10 @@
"13.34.23.32/27", "13.34.23.32/27",
"13.34.23.64/27", "13.34.23.64/27",
"13.34.23.96/27", "13.34.23.96/27",
"13.34.24.128/27",
"13.34.24.160/27",
"13.34.24.192/27",
"13.34.24.96/27",
"13.35.0.0/16", "13.35.0.0/16",
"13.48.0.0/15", "13.48.0.0/15",
"13.48.186.128/27", "13.48.186.128/27",
@ -164,6 +168,7 @@
"140.179.0.0/16", "140.179.0.0/16",
"143.204.0.0/16", "143.204.0.0/16",
"144.220.0.0/16", "144.220.0.0/16",
"15.152.0.0/16",
"15.161.0.0/16", "15.161.0.0/16",
"15.161.135.164/30", "15.161.135.164/30",
"15.164.0.0/15", "15.164.0.0/15",
@ -226,6 +231,9 @@
"15.221.20.0/22", "15.221.20.0/22",
"15.221.24.0/21", "15.221.24.0/21",
"15.221.3.0/24", "15.221.3.0/24",
"15.221.33.0/24",
"15.221.34.0/24",
"15.221.35.0/24",
"15.221.4.0/23", "15.221.4.0/23",
"15.221.8.0/21", "15.221.8.0/21",
"15.222.0.0/15", "15.222.0.0/15",
@ -382,6 +390,7 @@
"15.230.40.0/24", "15.230.40.0/24",
"15.230.41.0/24", "15.230.41.0/24",
"15.230.42.0/24", "15.230.42.0/24",
"15.230.43.0/24",
"15.230.5.0/24", "15.230.5.0/24",
"15.230.6.0/24", "15.230.6.0/24",
"15.230.64.0/26", "15.230.64.0/26",
@ -414,6 +423,22 @@
"150.222.118.0/24", "150.222.118.0/24",
"150.222.119.0/24", "150.222.119.0/24",
"150.222.12.0/24", "150.222.12.0/24",
"150.222.120.20/31",
"150.222.120.224/31",
"150.222.120.226/31",
"150.222.120.228/31",
"150.222.120.230/31",
"150.222.120.232/31",
"150.222.120.234/31",
"150.222.120.240/31",
"150.222.120.242/31",
"150.222.120.244/31",
"150.222.120.246/31",
"150.222.120.248/31",
"150.222.120.250/31",
"150.222.120.252/32",
"150.222.120.255/32",
"150.222.120.62/31",
"150.222.121.0/24", "150.222.121.0/24",
"150.222.122.100/31", "150.222.122.100/31",
"150.222.122.102/31", "150.222.122.102/31",
@ -494,6 +519,7 @@
"150.222.222.0/24", "150.222.222.0/24",
"150.222.223.0/24", "150.222.223.0/24",
"150.222.224.0/24", "150.222.224.0/24",
"150.222.226.0/24",
"150.222.227.0/24", "150.222.227.0/24",
"150.222.228.0/24", "150.222.228.0/24",
"150.222.229.0/24", "150.222.229.0/24",
@ -545,6 +571,10 @@
"150.222.244.35/32", "150.222.244.35/32",
"150.222.244.37/32", "150.222.244.37/32",
"150.222.245.122/31", "150.222.245.122/31",
"150.222.3.176/32",
"150.222.3.177/32",
"150.222.3.178/32",
"150.222.3.179/32",
"150.222.5.0/24", "150.222.5.0/24",
"150.222.6.0/24", "150.222.6.0/24",
"150.222.66.0/24", "150.222.66.0/24",
@ -623,7 +653,6 @@
"176.34.64.0/18", "176.34.64.0/18",
"177.71.128.0/17", "177.71.128.0/17",
"177.71.207.128/26", "177.71.207.128/26",
"177.71.207.16/29",
"177.72.240.0/21", "177.72.240.0/21",
"178.236.0.0/20", "178.236.0.0/20",
"18.130.0.0/16", "18.130.0.0/16",
@ -1009,11 +1038,42 @@
"2600:1ffc:8000::/40", "2600:1ffc:8000::/40",
"2600:1ffc:c000::/40", "2600:1ffc:c000::/40",
"2600:1ffc:e000::/40", "2600:1ffc:e000::/40",
"2600:1ffd:807b::/48",
"2600:1ffd:807f::/48", "2600:1ffd:807f::/48",
"2600:1ffd:80c8::/48",
"2600:1ffd:80cb::/48", "2600:1ffd:80cb::/48",
"2600:1ffd:80d0::/48",
"2600:1ffd:80e1::/48", "2600:1ffd:80e1::/48",
"2600:1ffd:80e8::/48",
"2600:1ffd:80f0::/48",
"2600:1ffd:8143::/48",
"2600:1ffd:8149::/48",
"2600:1ffd:814b::/48",
"2600:1ffd:8152::/48",
"2600:1ffd:8165::/48",
"2600:1ffd:8166::/48",
"2600:1ffd:816c::/48",
"2600:1ffd:8188::/48",
"2600:1ffd:818b::/48",
"2600:1ffd:818f::/48",
"2600:1ffd:8190::/48", "2600:1ffd:8190::/48",
"2600:1ffd:857f::/48", "2600:1ffd:8195::/48",
"2600:1ffd:819f::/48",
"2600:1ffd:81a0::/48",
"2600:1ffd:81a3::/48",
"2600:1ffd:81a5::/48",
"2600:1ffd:81a7::/48",
"2600:1ffd:81ab::/48",
"2600:1ffd:81ad::/48",
"2600:1ffd:81bb::/48",
"2600:1ffd:81be::/48",
"2600:1ffd:84a0::/48",
"2600:1ffd:84a4::/48",
"2600:1ffd:84ab::/48",
"2600:1ffd:84ae::/48",
"2600:1ffd:84af::/48",
"2600:1ffd:84b4::/48",
"2600:1ffd:84bd::/48",
"2600:1ffe:1000::/40", "2600:1ffe:1000::/40",
"2600:1ffe:2000::/40", "2600:1ffe:2000::/40",
"2600:1ffe:4000::/40", "2600:1ffe:4000::/40",
@ -1216,7 +1276,14 @@
"3.216.144.0/23", "3.216.144.0/23",
"3.216.148.0/22", "3.216.148.0/22",
"3.217.228.0/22", "3.217.228.0/22",
"3.218.180.0/22", "3.218.180.0/25",
"3.218.180.128/25",
"3.218.181.0/25",
"3.218.181.128/25",
"3.218.182.0/25",
"3.218.182.128/25",
"3.218.183.0/25",
"3.218.183.128/25",
"3.224.0.0/12", "3.224.0.0/12",
"3.227.250.128/25", "3.227.250.128/25",
"3.231.2.0/25", "3.231.2.0/25",
@ -2423,7 +2490,8 @@
"99.83.64.0/21", "99.83.64.0/21",
"99.83.98.0/24", "99.83.98.0/24",
"99.84.0.0/16", "99.84.0.0/16",
"99.86.0.0/16" "99.86.0.0/16",
"99.87.0.0/22"
], ],
"matching_attributes": [ "matching_attributes": [
"ip-src", "ip-src",
@ -2432,5 +2500,5 @@
], ],
"name": "List of known Amazon AWS IP address ranges", "name": "List of known Amazon AWS IP address ranges",
"type": "cidr", "type": "cidr",
"version": 20200721 "version": 20200727
} }

330
lists/cisco_top1000/list.json
View File

File diff suppressed because it is too large Load Diff

3436
lists/cisco_top10k/list.json
View File

File diff suppressed because it is too large Load Diff

5972
lists/cisco_top20k/list.json
View File

File diff suppressed because it is too large Load Diff

1840
lists/cisco_top5k/list.json
View File

File diff suppressed because it is too large Load Diff

4
lists/crl-ip-hostname/list.json
View File

@ -387,6 +387,6 @@
"domain|ip" "domain|ip"
], ],
"name": "CRL Warninglist", "name": "CRL Warninglist",
"type": "substring", "type": "string",
"version": 20190301 "version": 20200727
} }

348
lists/majestic_million/list.json
View File

File diff suppressed because it is too large Load Diff

907
lists/moz-top500/list.json
View File

File diff suppressed because it is too large Load Diff

38
lists/mozilla-IntermediateCA/list.json
View File

@ -55,6 +55,7 @@
"01ed4f29eccef3110717c21d3e4498f409472c39", "01ed4f29eccef3110717c21d3e4498f409472c39",
"02007a05ced36899aa8a03a2cf307f1c0449fc31", "02007a05ced36899aa8a03a2cf307f1c0449fc31",
"0207056d172c80bdfb6dc45be9e5808846078d1e6eef1b6ed70259ab332a64c1", "0207056d172c80bdfb6dc45be9e5808846078d1e6eef1b6ed70259ab332a64c1",
"0208ba5266fb27f3de0d2e9ac509f2c550c2159a",
"022886ecf3bce37b2e114a7a34823c81", "022886ecf3bce37b2e114a7a34823c81",
"0230feebdb5087ea5b75b046e25c8542738efe86", "0230feebdb5087ea5b75b046e25c8542738efe86",
"0238e4e67c1a837b5dc674afc154a4ad", "0238e4e67c1a837b5dc674afc154a4ad",
@ -151,6 +152,7 @@
"04e8f12522427c5e23aa79221cf6b328", "04e8f12522427c5e23aa79221cf6b328",
"04e9aabd14bf0d39c472c788a4b4c642", "04e9aabd14bf0d39c472c788a4b4c642",
"04ecba8f92bff7458c4a5e7c69261fc7e2ef52d5af54fbdd92b17141bbe0651f", "04ecba8f92bff7458c4a5e7c69261fc7e2ef52d5af54fbdd92b17141bbe0651f",
"04eeea8e50b4775b3c24797262917ee50002ec4c75b56cdf3ee1c18cfca5ba52",
"04f7a1c112607e593928fec786cc6021bd6ebd95001f1c693b8c4b384eb19f6a", "04f7a1c112607e593928fec786cc6021bd6ebd95001f1c693b8c4b384eb19f6a",
"0503e0da86c78af88dc720362432f202b51ab1a8", "0503e0da86c78af88dc720362432f202b51ab1a8",
"05177d018c5cf4ea0dfb630a0c846f50e8d245df", "05177d018c5cf4ea0dfb630a0c846f50e8d245df",
@ -177,6 +179,7 @@
"05c7bcf83f02a84ff931441c640dc308", "05c7bcf83f02a84ff931441c640dc308",
"05cb0545b3d382ae1bc23e2a1eb0cd590c20a862", "05cb0545b3d382ae1bc23e2a1eb0cd590c20a862",
"05cf90f70d1e94a4c919f575ba9144d8", "05cf90f70d1e94a4c919f575ba9144d8",
"05e4005db0c382f3bd66b47729e9011577601bf6f7b287e9a52ced710d258346",
"05e69942d6af1baed73f67c731872195", "05e69942d6af1baed73f67c731872195",
"05e8dea8556df54e4f709376573ec990", "05e8dea8556df54e4f709376573ec990",
"05ee9c2ac66f75d964ac5f1a3c7de75d", "05ee9c2ac66f75d964ac5f1a3c7de75d",
@ -252,6 +255,7 @@
"084c318595e957ec218866984b9f2246", "084c318595e957ec218866984b9f2246",
"0855414af5f5fd7e264f8b002a39cced67e5952e89b61b680cc847baa34944de", "0855414af5f5fd7e264f8b002a39cced67e5952e89b61b680cc847baa34944de",
"085a0714524df7ae9919a6f597884439e12a533e61552d29d6a18c35096fc548", "085a0714524df7ae9919a6f597884439e12a533e61552d29d6a18c35096fc548",
"085a74d4683d0fccde44677cfa579b732c9ce2da6921ad3fe70f814ee7c0bf52",
"086d6e110dd75c37ea9f0530a93b67453733e4ef38dc53fc2c8077db75267971", "086d6e110dd75c37ea9f0530a93b67453733e4ef38dc53fc2c8077db75267971",
"08745487e891c19e3078c1f2a07e452950ef36f6", "08745487e891c19e3078c1f2a07e452950ef36f6",
"087feac17c75ea0ab5952f0017814ace3c7107ec", "087feac17c75ea0ab5952f0017814ace3c7107ec",
@ -328,7 +332,6 @@
"0ad741d98a84c3b9c4e7552e15bff1e369b6806839455dbafcac638124d5f100", "0ad741d98a84c3b9c4e7552e15bff1e369b6806839455dbafcac638124d5f100",
"0af61aa1400809d1f612b53ce61961098b787767", "0af61aa1400809d1f612b53ce61961098b787767",
"0af912694434515de92ae4c4337b56fd", "0af912694434515de92ae4c4337b56fd",
"0afa0f7bbfb8eeb76f451139138f8806cc56a6285aad7cc11957f4bd09ae48c1",
"0afc8ca591cd34e06b30b23ffbcb5ec44939fc86", "0afc8ca591cd34e06b30b23ffbcb5ec44939fc86",
"0b1d4bac054b0dcb5866cf73f1856959b3ffdee50456001ffc6df0e35748ab92", "0b1d4bac054b0dcb5866cf73f1856959b3ffdee50456001ffc6df0e35748ab92",
"0b2280ef840adf325520b70d8785512f5148a21f", "0b2280ef840adf325520b70d8785512f5148a21f",
@ -582,6 +585,7 @@
"13bc6504fbf77b74048ec5d46aff63b0", "13bc6504fbf77b74048ec5d46aff63b0",
"13c6b7395bc64fc06cef2ce53f66d4b9800c52cd8816ee37532f3ab24b8165cc", "13c6b7395bc64fc06cef2ce53f66d4b9800c52cd8816ee37532f3ab24b8165cc",
"13c8bf84a1196bd07d16c41829199210", "13c8bf84a1196bd07d16c41829199210",
"13cbf05d4dca3e6bcce1f0403a0bc9f467d108b3",
"13cbf88815ef41e879bedcb8b68e75e8051aa62fce6799eea34bd33d343bb32c", "13cbf88815ef41e879bedcb8b68e75e8051aa62fce6799eea34bd33d343bb32c",
"13db41e68e0b6126302f4789b9373177", "13db41e68e0b6126302f4789b9373177",
"13dd43e3c9357d31dbef2b6f4ff99cde", "13dd43e3c9357d31dbef2b6f4ff99cde",
@ -725,6 +729,7 @@
"18176016b41268270cb34175d67422c421a0e9b5", "18176016b41268270cb34175d67422c421a0e9b5",
"1817d47b242f12389f140e6d10810a3f4fdd394f", "1817d47b242f12389f140e6d10810a3f4fdd394f",
"181dc8be7e91edf163e66cc3fad14d979a6dae53b556daee8f7dca0e114c14f6", "181dc8be7e91edf163e66cc3fad14d979a6dae53b556daee8f7dca0e114c14f6",
"1820404404c65eff36dfd3a5210c81aa6c294bee",
"183244b8712297be207bf17d76bd7055", "183244b8712297be207bf17d76bd7055",
"1835b0e482ea65536fc010e4bc13c060f65668165fba97e2f542ce96ca6dfefc", "1835b0e482ea65536fc010e4bc13c060f65668165fba97e2f542ce96ca6dfefc",
"18370b91824ea99e398f7086c76b0cd5", "18370b91824ea99e398f7086c76b0cd5",
@ -740,6 +745,7 @@
"187de1f9e61a2ec4f90c16bbed97e95ac8fbaa31", "187de1f9e61a2ec4f90c16bbed97e95ac8fbaa31",
"188b39ceeecb6e612e01b1f70bfd30035f703e28", "188b39ceeecb6e612e01b1f70bfd30035f703e28",
"188e7931b6d781af22b645f9f695239fe44da8b759cba662aecccdecd924b7e1", "188e7931b6d781af22b645f9f695239fe44da8b759cba662aecccdecd924b7e1",
"189286494b786a5ab8dc7dbc58905aca",
"1892884bb13fbff75ca6c982e46545021025c5b2", "1892884bb13fbff75ca6c982e46545021025c5b2",
"1893205276c4448e00ecbd613e5fc75cb672acc9", "1893205276c4448e00ecbd613e5fc75cb672acc9",
"18958d03afb409687a1bc263860d0d735a25a004ab60e0f0e45d6333587437ae", "18958d03afb409687a1bc263860d0d735a25a004ab60e0f0e45d6333587437ae",
@ -1156,6 +1162,7 @@
"27efe682c7479c38133a0943666fbc9103fcbb1a919634e439711cbea7e3cfc7", "27efe682c7479c38133a0943666fbc9103fcbb1a919634e439711cbea7e3cfc7",
"27fa1be4f4b56a142a47a56a7e94b72502059b4022e4623e748ca1ef151cf222", "27fa1be4f4b56a142a47a56a7e94b72502059b4022e4623e748ca1ef151cf222",
"280bbc34a98cc79606a89690e52ab72fdc5fff275dc366c4827b1656313e9790", "280bbc34a98cc79606a89690e52ab72fdc5fff275dc366c4827b1656313e9790",
"280c44a7fe1e31eba23e0fc6a7eac611fa5be1e8",
"281bd27daa72be30316ba5b52ca0fd66", "281bd27daa72be30316ba5b52ca0fd66",
"2822861d1e3d1a3d4e28cef50c362aab", "2822861d1e3d1a3d4e28cef50c362aab",
"282f2953d96ea3194d14bad6a1046018", "282f2953d96ea3194d14bad6a1046018",
@ -1536,6 +1543,7 @@
"345eff15b7a49add451b65a7f4bdc6ae", "345eff15b7a49add451b65a7f4bdc6ae",
"346071ce2ee312d1551e9bd9e88a38cb", "346071ce2ee312d1551e9bd9e88a38cb",
"3473aecd1d6248fd45fee88a2d1758397f6c4a45aefdc8ce97c764500d214560", "3473aecd1d6248fd45fee88a2d1758397f6c4a45aefdc8ce97c764500d214560",
"347d18dccc2efc51a920e7a7fbb07bfbda35613681f82dca5c4c72bb0c83c035",
"3486ed23622155459e9b25ff3f21ad7627987387", "3486ed23622155459e9b25ff3f21ad7627987387",
"348fda2b99cd5ae0b92ffa52fd6b3d72", "348fda2b99cd5ae0b92ffa52fd6b3d72",
"34b6d26a5f8f1c1c064cc24ebcd0f8cf", "34b6d26a5f8f1c1c064cc24ebcd0f8cf",
@ -1971,6 +1979,7 @@
"43db8b6042e2da18210b5feca471e9da480c11ef", "43db8b6042e2da18210b5feca471e9da480c11ef",
"43df7b055f78ecc001599232613a3befbd3e076a", "43df7b055f78ecc001599232613a3befbd3e076a",
"43e9470aabd9f4243661eeb94715d661", "43e9470aabd9f4243661eeb94715d661",
"440b747378b0596d429570f7f89201aed2e9327e",
"440d16f31810555b2ec52c439d55dcbb60b9ffad", "440d16f31810555b2ec52c439d55dcbb60b9ffad",
"440ff68a35e03995ac55e457a67eb1680f9a7cdd", "440ff68a35e03995ac55e457a67eb1680f9a7cdd",
"44192d116fe72ed41247724ae41fb46c488929ae", "44192d116fe72ed41247724ae41fb46c488929ae",
@ -2072,6 +2081,7 @@
"477d1e884744bd9dadbd523bf7653caa", "477d1e884744bd9dadbd523bf7653caa",
"477f7bfbb85e66fd80f365b8b6decdf2", "477f7bfbb85e66fd80f365b8b6decdf2",
"4786ce4351b58cac6839fb4683cd291e", "4786ce4351b58cac6839fb4683cd291e",
"479195065720335e9016f1a9465e681c",
"47961937b34ce2d12bbd2e8ab5cc96c4c493746f", "47961937b34ce2d12bbd2e8ab5cc96c4c493746f",
"479c819fa2f318e9d2910842b045607b66c4bebcacdd49db4412859846145ab7", "479c819fa2f318e9d2910842b045607b66c4bebcacdd49db4412859846145ab7",
"47a2b188a1f084c58b8fb827038aad5b38e4f96d", "47a2b188a1f084c58b8fb827038aad5b38e4f96d",
@ -2565,7 +2575,6 @@
"57d8fbc13443745fbf4cab208930584b544fe510", "57d8fbc13443745fbf4cab208930584b544fe510",
"57de0583efd2b26e0361da99da9df4648def7ee8441c3b728afa9bcde0f9b26a", "57de0583efd2b26e0361da99da9df4648def7ee8441c3b728afa9bcde0f9b26a",
"57e03bb6bc2dfa1b10618baeda568a08", "57e03bb6bc2dfa1b10618baeda568a08",
"57eee860ebd1e466d5c85a33ab168162159017c5",
"57fe0ab300dc327f55fcf40cf3f8eec5e1d2c8b2", "57fe0ab300dc327f55fcf40cf3f8eec5e1d2c8b2",
"580bb3ab127f5e13e940ae7365e3c69a62ad204d7fdfa451592fe7c47dc02a95", "580bb3ab127f5e13e940ae7365e3c69a62ad204d7fdfa451592fe7c47dc02a95",
"580dff3c16e5f8ba06aabe7d156768256cf2536c", "580dff3c16e5f8ba06aabe7d156768256cf2536c",
@ -2816,6 +2825,7 @@
"602c3fbd1a20d30bf764130b50d28876386f8e66", "602c3fbd1a20d30bf764130b50d28876386f8e66",
"6033518ea020b06ac2ee4de4bb2973208776943d", "6033518ea020b06ac2ee4de4bb2973208776943d",
"6036330e1643a0cee19c8af780e0f3e8f59ca1a3", "6036330e1643a0cee19c8af780e0f3e8f59ca1a3",
"60388b773d33e9141648613f36b9149f015a114378bacf3e3694864ffda426ec",
"603ceecf1a21f3319bf275d71586a2a3", "603ceecf1a21f3319bf275d71586a2a3",
"603d69822381a0bfc274bbed67009bc7df133cb902fa242cf58bf727d23d5495", "603d69822381a0bfc274bbed67009bc7df133cb902fa242cf58bf727d23d5495",
"603eaafb53f2e76a7124f85f8b43a327", "603eaafb53f2e76a7124f85f8b43a327",
@ -2993,6 +3003,7 @@
"65ac9b17fb9ad079bd69fdf916a24100eed21063", "65ac9b17fb9ad079bd69fdf916a24100eed21063",
"65adc9c691d32155e68ce8d5193bf372295d25a04075457b31ee85a0441b23f8", "65adc9c691d32155e68ce8d5193bf372295d25a04075457b31ee85a0441b23f8",
"65cf938fd9fb34200c9f6df93ded73aa5b71c369cc74a7c491f0e47a1d5e74be", "65cf938fd9fb34200c9f6df93ded73aa5b71c369cc74a7c491f0e47a1d5e74be",
"65d17ecae5798c79db8e840fe98a53b9",
"65de322a1ef7affedeb7387138c26060825b08cc27e1992dd9eac8337297957b", "65de322a1ef7affedeb7387138c26060825b08cc27e1992dd9eac8337297957b",
"65e0db51904d0eb88e278d0af0557f9fbc568cbf8460544f0e193152ebd03755", "65e0db51904d0eb88e278d0af0557f9fbc568cbf8460544f0e193152ebd03755",
"65e1a6cf628279fca25c49156aa38715", "65e1a6cf628279fca25c49156aa38715",
@ -3311,6 +3322,7 @@
"701b432ac0cdd4d9cf95b4b884c32bf5cca90d44e0161abd13b934d68e380472", "701b432ac0cdd4d9cf95b4b884c32bf5cca90d44e0161abd13b934d68e380472",
"701eb23f95564cd5569cd20e5f05c2888900bae9ba03abf5abe57bfe04b54a60", "701eb23f95564cd5569cd20e5f05c2888900bae9ba03abf5abe57bfe04b54a60",
"70357b9e56d3fb3c6c009c38c7181454c462908dfbce6d54d60dfe1e506e14fd", "70357b9e56d3fb3c6c009c38c7181454c462908dfbce6d54d60dfe1e506e14fd",
"703d7a8f0ebf55aaa59f98eaf4a206004eb2516a",
"703ea0a173e2042355f1dfeb1079292d41ea68c046c9d45cc60dc5410104f478", "703ea0a173e2042355f1dfeb1079292d41ea68c046c9d45cc60dc5410104f478",
"704b659f2e7a06ead38dbf397bc4432b899fb88413e085ea572ba9c96ce60ae0", "704b659f2e7a06ead38dbf397bc4432b899fb88413e085ea572ba9c96ce60ae0",
"70541c2823e1a30b452d5f862833dc13", "70541c2823e1a30b452d5f862833dc13",
@ -3802,7 +3814,6 @@
"7f6889ffe8b02045e2cac99a2f2ee4f4c2eed24934b6521872d34bf812671c4b", "7f6889ffe8b02045e2cac99a2f2ee4f4c2eed24934b6521872d34bf812671c4b",
"7f755dc2b9ee99eb02d71cb1805059aaabb5eb3e8558dca8c09a1d0de258d767", "7f755dc2b9ee99eb02d71cb1805059aaabb5eb3e8558dca8c09a1d0de258d767",
"7f786418f0afcebeb2be04c2e5b2ee4c835929ae", "7f786418f0afcebeb2be04c2e5b2ee4c835929ae",
"7f79889c8f8312368b52173ed8fcd462",
"7f814cd1454900ac1331364b6b6de06b87cc8cb9936ec583100822721057d3a3", "7f814cd1454900ac1331364b6b6de06b87cc8cb9936ec583100822721057d3a3",
"7f8c9f7829dbe8b768cb4a60da4f9ab558d064eb", "7f8c9f7829dbe8b768cb4a60da4f9ab558d064eb",
"7f8cc76fbfd2b031aaabf0e6b0c8fd8c", "7f8cc76fbfd2b031aaabf0e6b0c8fd8c",
@ -3821,6 +3832,7 @@
"7fd90bbdfe628d97e254ca23871671f2", "7fd90bbdfe628d97e254ca23871671f2",
"7fdaceb62d40890bcf8c777b93d63e2d", "7fdaceb62d40890bcf8c777b93d63e2d",
"7fdce3bf4103c2684b3adbb5792884bd45c75094c217788863950346f79c90a3", "7fdce3bf4103c2684b3adbb5792884bd45c75094c217788863950346f79c90a3",
"7feb9374eab08d392717c647436dae06176a24c010607fda1cce5e5f0106b472",
"7ff0739409e68802c0149af180645ff7", "7ff0739409e68802c0149af180645ff7",
"80031c140e89f9be6792be2b49b74603", "80031c140e89f9be6792be2b49b74603",
"80042563db5f162e455632b589c60150f37ef7fedb774c6dc2ed5ba6fb865120", "80042563db5f162e455632b589c60150f37ef7fedb774c6dc2ed5ba6fb865120",
@ -3833,6 +3845,7 @@
"802e0b133d61441a0a6af799dc5e4b260972990d6dcb87dc0acfc585d9082610", "802e0b133d61441a0a6af799dc5e4b260972990d6dcb87dc0acfc585d9082610",
"8048eefc45bd5e2ef7254b79e7ed9968f5213b3a", "8048eefc45bd5e2ef7254b79e7ed9968f5213b3a",
"805577e05b81812579ae5102a79018b4", "805577e05b81812579ae5102a79018b4",
"806f1c72f6d67c9c114eff43d3d84100",
"8072f44e162fd430d83c4fff1c0b7f248258e4abb2aea8c4719aa5f3fd23c626", "8072f44e162fd430d83c4fff1c0b7f248258e4abb2aea8c4719aa5f3fd23c626",
"8081b85329c69843c7d9444e37bd367a", "8081b85329c69843c7d9444e37bd367a",
"808205ede5d170cc4aac49c119c29eb8", "808205ede5d170cc4aac49c119c29eb8",
@ -3960,7 +3973,9 @@
"84a850389f51a50ff60e54c7c4189d81", "84a850389f51a50ff60e54c7c4189d81",
"84aa25e9a6ef9acd2ea64743d0674af1", "84aa25e9a6ef9acd2ea64743d0674af1",
"84d3226619a7c2584d49b2cc7465fa85902efee3a5e8f36b36ec70ce3ea4a09f", "84d3226619a7c2584d49b2cc7465fa85902efee3a5e8f36b36ec70ce3ea4a09f",
"84dddd0196ac27bcc2f17d564769ea7e",
"84fae8c3aeebca606050bc11487fece7464c1210a4e6eaf261cecb013681bcab", "84fae8c3aeebca606050bc11487fece7464c1210a4e6eaf261cecb013681bcab",
"8503c71b1e484c60cc2913ea2d79c482626a1847",
"8504a0350036b18594b1f47b6a3bd3513e28b045bc7dbfe5b08d8f3049cd3cfe", "8504a0350036b18594b1f47b6a3bd3513e28b045bc7dbfe5b08d8f3049cd3cfe",
"850575bbf8a8a8d43b44886210f84a0746ca9493", "850575bbf8a8a8d43b44886210f84a0746ca9493",
"850772c2c9201656ca7adf33772aeb7e", "850772c2c9201656ca7adf33772aeb7e",
@ -4436,7 +4451,6 @@
"9470dc33d482850ed9e458ad48b62a4fde772c91", "9470dc33d482850ed9e458ad48b62a4fde772c91",
"9475d29be526627856fee40e191e13c8febda40f", "9475d29be526627856fee40e191e13c8febda40f",
"947de5a14bc7bef32f5f4d326d15aa5d", "947de5a14bc7bef32f5f4d326d15aa5d",
"94833a3766439d0b83bb4bf06233babd11f217a42ba7e8c7392c212171407350",
"948b7111af42f546d579cff5ce2bdec82134dd9914842bddb0c52872eb604e39", "948b7111af42f546d579cff5ce2bdec82134dd9914842bddb0c52872eb604e39",
"948d1daf1d124ace83f6826192036edc35c4d005", "948d1daf1d124ace83f6826192036edc35c4d005",
"948e1652586240d453287ab69caeb8f2f4f02117", "948e1652586240d453287ab69caeb8f2f4f02117",
@ -4796,7 +4810,6 @@
"9f9c14d22a083785ceb7d02a19e730f8", "9f9c14d22a083785ceb7d02a19e730f8",
"9f9f66c96b2af2fd1b478480373d3c1ae1ee10a3", "9f9f66c96b2af2fd1b478480373d3c1ae1ee10a3",
"9fa9fcb500f1a98355099f70034cf0d2ee743aba63f6eb67d55685ea9065328b", "9fa9fcb500f1a98355099f70034cf0d2ee743aba63f6eb67d55685ea9065328b",
"9fba32a6be7f0f7e9b321a01ec45a5e8",
"9fc43b631d46e2ffa6c11928a33a8dccbccf86c5d8dd461518541c52fe259266", "9fc43b631d46e2ffa6c11928a33a8dccbccf86c5d8dd461518541c52fe259266",
"9fcdf094368d1b025c4c5574f8c59db8df75d0c3", "9fcdf094368d1b025c4c5574f8c59db8df75d0c3",
"9fd9b25764b14052b1db155536e38867", "9fd9b25764b14052b1db155536e38867",
@ -4819,7 +4832,6 @@
"a01264a53c3d71887b7a6ea3d9a05eda", "a01264a53c3d71887b7a6ea3d9a05eda",
"a0252e6ffe0cc002518a6e3fb4105eda28ef1213", "a0252e6ffe0cc002518a6e3fb4105eda28ef1213",
"a026b5019219582e2748a65e7f70e159", "a026b5019219582e2748a65e7f70e159",
"a028356eaabdbbddfc78523dfc1287a0",
"a028df21db93af1ebd970e0e681cf902c20b2185", "a028df21db93af1ebd970e0e681cf902c20b2185",
"a02ddcec3086787e9de7152bf85ec9e7", "a02ddcec3086787e9de7152bf85ec9e7",
"a02ecf23fd9e2ff3880667b150b3dd030b126751", "a02ecf23fd9e2ff3880667b150b3dd030b126751",
@ -5319,6 +5331,7 @@
"b0a6ef0350e7c4c6056beea7af9d2d860b9ed102137b9729d3c23216d195546a", "b0a6ef0350e7c4c6056beea7af9d2d860b9ed102137b9729d3c23216d195546a",
"b0b05d7131d7881f78ba4172b442b7d774d04ff27d383be3e459a372473b1e15", "b0b05d7131d7881f78ba4172b442b7d774d04ff27d383be3e459a372473b1e15",
"b0c29d0244f981e999a15f59708655f0", "b0c29d0244f981e999a15f59708655f0",
"b0c2d2d13cdd56cdaa6ab6e2c04440be4a429c75",
"b0c7c6e2b6265989c1a104256ed756fe", "b0c7c6e2b6265989c1a104256ed756fe",
"b0cd6ae7b9e20ec5f830fee01f666d5d90e6e229d06bc46a30accedfec889648", "b0cd6ae7b9e20ec5f830fee01f666d5d90e6e229d06bc46a30accedfec889648",
"b0e6383fbdbfce5e5f9d7934fad0fc3c5e030f13", "b0e6383fbdbfce5e5f9d7934fad0fc3c5e030f13",
@ -5350,6 +5363,7 @@
"b19885909d55e47f196f36a55ccc4ea9afa052c3f8ee3e3a9ee041e562163584", "b19885909d55e47f196f36a55ccc4ea9afa052c3f8ee3e3a9ee041e562163584",
"b19b9bc90c06f08061824cc723c82ecab90fe543", "b19b9bc90c06f08061824cc723c82ecab90fe543",
"b1a8fd40b437ded972f574370622e2bbb2f9b9593bc0504d7e2030f2b24e67fa", "b1a8fd40b437ded972f574370622e2bbb2f9b9593bc0504d7e2030f2b24e67fa",
"b1aae1bcd555e8a3d1e3dda8ec84e757c552655344ce3feebbaf98e895dbedac",
"b1ac8cfb181b9c9354e1775fcbdfcfe7898c5cc9a17d76315b57c112eee55234", "b1ac8cfb181b9c9354e1775fcbdfcfe7898c5cc9a17d76315b57c112eee55234",
"b1ae92ee3e68ac9c00bee275604782301ebcd9b63cd3e0d5e8144fdf7415b9dd", "b1ae92ee3e68ac9c00bee275604782301ebcd9b63cd3e0d5e8144fdf7415b9dd",
"b1b3162f7cc867f6d47536df3232e706e031357c", "b1b3162f7cc867f6d47536df3232e706e031357c",
@ -5627,6 +5641,7 @@
"bac4f03dc635bdae067f7bfe77ab06d88b5ba4668b0f46157e53d6ca9310077f", "bac4f03dc635bdae067f7bfe77ab06d88b5ba4668b0f46157e53d6ca9310077f",
"bacafa90e4a81408ff7b1245b7e973213c83488c", "bacafa90e4a81408ff7b1245b7e973213c83488c",
"bad74f4d7ab128a49460692416540b5aeaa6b46a", "bad74f4d7ab128a49460692416540b5aeaa6b46a",
"bad7fe221e2bf54e832bde8871f0930c",
"bad88a984e396a294168e085077ffac2", "bad88a984e396a294168e085077ffac2",
"bae30b15dbb1544cf194d076b75b7bb9e3d6b760", "bae30b15dbb1544cf194d076b75b7bb9e3d6b760",
"bae5efbd8fff3de56dc2388803bde18572b32e6e", "bae5efbd8fff3de56dc2388803bde18572b32e6e",
@ -6004,6 +6019,7 @@
"c717d11a9035a26df21a6be9bada93ef", "c717d11a9035a26df21a6be9bada93ef",
"c71b01ff159489973f92e2f841d63987ebf24548", "c71b01ff159489973f92e2f841d63987ebf24548",
"c72a5a03842ff4a31ad26e31bbdbaee5", "c72a5a03842ff4a31ad26e31bbdbaee5",
"c72c7ac0fd729d4997a1a74fb7df6535",
"c7367403cf6d5c13d9690187926ef896ce6e74135080c9afd8c8b1532467ae50", "c7367403cf6d5c13d9690187926ef896ce6e74135080c9afd8c8b1532467ae50",
"c7422de21cebc92ecae6be9dcd7e711ee650d30aed711fbb0df6b2c784b6c4fb", "c7422de21cebc92ecae6be9dcd7e711ee650d30aed711fbb0df6b2c784b6c4fb",
"c743fe562a1eb4b3494d752e1443a97e", "c743fe562a1eb4b3494d752e1443a97e",
@ -6357,7 +6373,6 @@
"d29900b39c9f22ff2da8497ca624719d64a86ba9", "d29900b39c9f22ff2da8497ca624719d64a86ba9",
"d2a50723561f9bba7cef93b76315268a", "d2a50723561f9bba7cef93b76315268a",
"d2adf8385fe30160fc5169ec81f8cc33ab88ca23", "d2adf8385fe30160fc5169ec81f8cc33ab88ca23",
"d2bdb0760d4b69357c589ddf4108385923d820f6",
"d2c410cb8c94e566bfd4b4ff23c8868632743690", "d2c410cb8c94e566bfd4b4ff23c8868632743690",
"d2c686e02a4a50772ea9cdec7ffad03e", "d2c686e02a4a50772ea9cdec7ffad03e",
"d2c7cb6e13483c6004009b2d97455b7ff8f4afa6", "d2c7cb6e13483c6004009b2d97455b7ff8f4afa6",
@ -6369,6 +6384,7 @@
"d31520a86966839c76f79b23bbbb3c545b582b99", "d31520a86966839c76f79b23bbbb3c545b582b99",
"d3229995b6a61afbfd3cb245be9b129b", "d3229995b6a61afbfd3cb245be9b129b",
"d328f91b2d74c372820a01a0ce54d3bb", "d328f91b2d74c372820a01a0ce54d3bb",
"d33a847f640377be0ae1a429153a07e87c8827fa4840b5158bcdcb85e10a453a",
"d33d27b38c72bfa7914f068b10fd18d8e12a45f5e60b8c35d6728c2d17fe3c48", "d33d27b38c72bfa7914f068b10fd18d8e12a45f5e60b8c35d6728c2d17fe3c48",
"d34138ed1458ae7de4edea36ed3992e4f46e6ec9cf1e633e538de9ff0f38f8e5", "d34138ed1458ae7de4edea36ed3992e4f46e6ec9cf1e633e538de9ff0f38f8e5",
"d3533b732a518a6da68ef266085e11dfd114c0eb0092cd43530a44d54b913ed1", "d3533b732a518a6da68ef266085e11dfd114c0eb0092cd43530a44d54b913ed1",
@ -6424,6 +6440,7 @@
"d517bfad08702c10f086ddc192e702748bb716d6", "d517bfad08702c10f086ddc192e702748bb716d6",
"d51f1ed305821bb2541af3afc96e2ff2", "d51f1ed305821bb2541af3afc96e2ff2",
"d522f5744e0303cdd3cb101eb00fbfefa0c66985", "d522f5744e0303cdd3cb101eb00fbfefa0c66985",
"d52db7b240677eaa7154d588f74cbc1bfccda14e",
"d5309f1c2014ff3a18be707c7c1123bb", "d5309f1c2014ff3a18be707c7c1123bb",
"d53463db5346a1811dbd6478888a11a7", "d53463db5346a1811dbd6478888a11a7",
"d554ede7df29a86ea2b30021482ddfbd24cdec55", "d554ede7df29a86ea2b30021482ddfbd24cdec55",
@ -6516,7 +6533,6 @@
"d8e47c0b60e3dfe37b42242c719f63c2", "d8e47c0b60e3dfe37b42242c719f63c2",
"d8e79ebed997b764e6d0df624f2dbb4eff2a1b6049c6f1a1a1e88263c43db221", "d8e79ebed997b764e6d0df624f2dbb4eff2a1b6049c6f1a1a1e88263c43db221",
"d8f6ce367405c0d4bf179552fec2925d", "d8f6ce367405c0d4bf179552fec2925d",
"d8fae00729250ebbd727757a12232b1d4887ce9f195b344c18cfee068cfbe6ec",
"d933da413ce524080624a34ce85aa48e", "d933da413ce524080624a34ce85aa48e",
"d9352669e29349329af4745c55a6c159dc979ab745925170a1980668382acbc0", "d9352669e29349329af4745c55a6c159dc979ab745925170a1980668382acbc0",
"d93c3c309071a5fbbca28c45311f763aef2dbe62", "d93c3c309071a5fbbca28c45311f763aef2dbe62",
@ -6581,6 +6597,7 @@
"db3c8e2a7f1cba1419edaf68dc5a2428", "db3c8e2a7f1cba1419edaf68dc5a2428",
"db3dbba517e5bca21fa69b72e8867f74afb1b7aa", "db3dbba517e5bca21fa69b72e8867f74afb1b7aa",
"db4591f878f6672f5b70733a66ad7c9537b97e6f0af5ca49aab8ecb2ce02f86b", "db4591f878f6672f5b70733a66ad7c9537b97e6f0af5ca49aab8ecb2ce02f86b",
"db46e58630ad01fd265b0862d7026812",
"db476339ccbfcc9e4bd1d6cb606ca27f00679e1ef8a581e7236309b9d63ffe37", "db476339ccbfcc9e4bd1d6cb606ca27f00679e1ef8a581e7236309b9d63ffe37",
"db5555779be0e3f862ba9e92a28683743c3e9e0d", "db5555779be0e3f862ba9e92a28683743c3e9e0d",
"db6cbb8733beb493af39c702a9dd0674314e2ed3260cd5250a491480757d8ea2", "db6cbb8733beb493af39c702a9dd0674314e2ed3260cd5250a491480757d8ea2",
@ -6935,6 +6952,7 @@
"e803a4975e3df57967ece94de85dc141", "e803a4975e3df57967ece94de85dc141",
"e810ac30d9a776056fc1c994b76ded94", "e810ac30d9a776056fc1c994b76ded94",
"e81443141e3c1232d06465e11bbd1f6ca7181a3e037f815bb8faaeade6bebed1", "e81443141e3c1232d06465e11bbd1f6ca7181a3e037f815bb8faaeade6bebed1",
"e8170f7b6f7a6078dba3daa2b10a13ab",
"e81b01f9f5692cf3823c6fd35886542bfaeefc5ea94f4e246e42c4a9fc5fe8ab", "e81b01f9f5692cf3823c6fd35886542bfaeefc5ea94f4e246e42c4a9fc5fe8ab",
"e8202a24114b3ecc751be23f6cef9018721f3604", "e8202a24114b3ecc751be23f6cef9018721f3604",
"e8248ff3f0f591086bcf6dec9c37c535a0704643", "e8248ff3f0f591086bcf6dec9c37c535a0704643",
@ -7059,6 +7077,7 @@
"ebaf8abd199528193747a135cac339c1", "ebaf8abd199528193747a135cac339c1",
"ebb06ca92fbe3cf1245f81f438bb830d", "ebb06ca92fbe3cf1245f81f438bb830d",
"ebbf4dc600c17da04381defdcfc119c3f34efb4a04d0860910b813c7792d7585", "ebbf4dc600c17da04381defdcfc119c3f34efb4a04d0860910b813c7792d7585",
"ebcd0c73fe44b60e25789419c07b0b234b3f586e092316fb2cbda39c223224a3",
"ebe0015ef641c480954ed2a68842fccd2a8a79213aeed836ddfa6282e8ea8f6a", "ebe0015ef641c480954ed2a68842fccd2a8a79213aeed836ddfa6282e8ea8f6a",
"ebe54f29a97b7d327d9b253b8a7eaafc", "ebe54f29a97b7d327d9b253b8a7eaafc",
"ebe6f53aa744955e95ad712d4b9b464c9b273f24", "ebe6f53aa744955e95ad712d4b9b464c9b273f24",
@ -7187,7 +7206,6 @@
"efeb3869a192aabb11d10d1313bda0f6", "efeb3869a192aabb11d10d1313bda0f6",
"f0029e794d5180634b34aee29f291358232faa04", "f0029e794d5180634b34aee29f291358232faa04",
"f00322c97b57cea6d64a6a1d365c6c6b7f0eba3089876c1430e1063e8a82a676", "f00322c97b57cea6d64a6a1d365c6c6b7f0eba3089876c1430e1063e8a82a676",
"f00366a7a57a9aee3276ccc7b0d3c05f9c814706",
"f00cf0fae0b51ba38e8065bb40d90e977586334f", "f00cf0fae0b51ba38e8065bb40d90e977586334f",
"f01abafc75ddd87a52fdd230c4514ffc", "f01abafc75ddd87a52fdd230c4514ffc",
"f01c1aca392882af152e9f01ecccd0afddd8aa35bf895b003198b1e8c752ddb8", "f01c1aca392882af152e9f01ecccd0afddd8aa35bf895b003198b1e8c752ddb8",
@ -7686,5 +7704,5 @@
], ],
"name": "Fingerprint of known intermedicate of trusted certificates", "name": "Fingerprint of known intermedicate of trusted certificates",
"type": "string", "type": "string",
"version": 20200721 "version": 20200723
} }

322
lists/public-dns-hostname/list.json
View File

File diff suppressed because it is too large Load Diff

583
lists/public-dns-v4/list.json
View File

File diff suppressed because it is too large Load Diff

7
lists/public-dns-v6/list.json
View File

@ -1,6 +1,8 @@
{ {
"description": "Event contains one or more public IPv6 DNS resolvers as attribute with an IDS flag set", "description": "Event contains one or more public IPv6 DNS resolvers as attribute with an IDS flag set",
"list": [ "list": [
"2001:148f:fffe::1",
"2001:148f:ffff::1",
"2001:41d0:203:4b1f:2:2:2:2", "2001:41d0:203:4b1f:2:2:2:2",
"2001:428:101:100:205:171:2:65", "2001:428:101:100:205:171:2:65",
"2001:428::1", "2001:428::1",
@ -11,18 +13,15 @@
"2001:4b8:3:201::902", "2001:4b8:3:201::902",
"2001:67c:28a4::", "2001:67c:28a4::",
"2001:67c:2b0::1", "2001:67c:2b0::1",
"2001:b000:168::2",
"2001:de4::102", "2001:de4::102",
"2606:ed00:2:babe::10", "2606:ed00:2:babe::10",
"2607:5300:203:1797::53", "2607:5300:203:1797::53",
"2607:f130:0:d7::d41",
"2610:a1:1018::34", "2610:a1:1018::34",
"2a00:aa40:0:225::2", "2a00:aa40:0:225::2",
"2a01:238:42f6:ac00:2a29:4f7f:b6d:ef46", "2a01:238:42f6:ac00:2a29:4f7f:b6d:ef46",
"2a01:3a0:53:53::", "2a01:3a0:53:53::",
"2a01:4f8:141:316d::117", "2a01:4f8:141:316d::117",
"2a01:4f8:151:34aa::198", "2a01:4f8:151:34aa::198",
"2a01:4f8:c0c:1e44::1",
"2a01:4f8:c17:739a::2", "2a01:4f8:c17:739a::2",
"2a02:2970:1002::18", "2a02:2970:1002::18",
"2a02:c205:3001:4558::1", "2a02:c205:3001:4558::1",
@ -35,5 +34,5 @@
], ],
"name": "List of known IPv6 public DNS resolvers", "name": "List of known IPv6 public DNS resolvers",
"type": "string", "type": "string",
"version": 20200720 "version": 20200723
} }

35570
lists/tranco/list.json
View File

File diff suppressed because it is too large Load Diff

140
lists/tranco10k/list.json
View File

@ -31,7 +31,6 @@
"127.net", "127.net",
"1337x.to", "1337x.to",
"1377x.to", "1377x.to",
"13dl.net",
"163.com", "163.com",
"1688.com", "1688.com",
"16mb.com", "16mb.com",
@ -49,7 +48,6 @@
"1fichier.com", "1fichier.com",
"1gb.ru", "1gb.ru",
"1mg.com", "1mg.com",
"1mono.biz",
"1password.com", "1password.com",
"1plus1tv.ru", "1plus1tv.ru",
"1rx.io", "1rx.io",
@ -66,7 +64,9 @@
"247wallst.com", "247wallst.com",
"24h.com.vn", "24h.com.vn",
"24s.com", "24s.com",
"24video.vip",
"258.com", "258.com",
"25pp.com",
"263.net", "263.net",
"2checkout.com", "2checkout.com",
"2conv.com", "2conv.com",
@ -82,7 +82,6 @@
"350.org", "350.org",
"360.cn", "360.cn",
"360.com", "360.com",
"360buyimg.com",
"360doc.com", "360doc.com",
"360kan.com", "360kan.com",
"360safe.com", "360safe.com",
@ -127,6 +126,7 @@
"52pojie.cn", "52pojie.cn",
"53.com", "53.com",
"53kf.com", "53kf.com",
"55online.news",
"56.com", "56.com",
"58.com", "58.com",
"58pic.com", "58pic.com",
@ -178,11 +178,11 @@
"9r.cn", "9r.cn",
"9to5google.com", "9to5google.com",
"9to5mac.com", "9to5mac.com",
"a-course-in-miracles.net",
"a-msedge.net", "a-msedge.net",
"a.co", "a.co",
"a16z.com", "a16z.com",
"a1sewcraft.com", "a1sewcraft.com",
"a2hosting.com",
"a2z.com", "a2z.com",
"a3cloud.net", "a3cloud.net",
"a3yqjsrczwwp.com", "a3yqjsrczwwp.com",
@ -208,7 +208,6 @@
"aarp.org", "aarp.org",
"aarth.net", "aarth.net",
"aastocks.com", "aastocks.com",
"aau.dk",
"aawsat.com", "aawsat.com",
"aaxads.com", "aaxads.com",
"abadis.ir", "abadis.ir",
@ -249,6 +248,7 @@
"academia.edu", "academia.edu",
"academic.ru", "academic.ru",
"acast.com", "acast.com",
"accaglobal.com",
"accenture.com", "accenture.com",
"accesspressthemes.com", "accesspressthemes.com",
"accesstrade.net", "accesstrade.net",
@ -296,7 +296,6 @@
"adafruit.com", "adafruit.com",
"adage.com", "adage.com",
"adap.tv", "adap.tv",
"adaptedmind.com",
"adaware.com", "adaware.com",
"adb.org", "adb.org",
"adblockplus.org", "adblockplus.org",
@ -424,6 +423,7 @@
"ahrq.gov", "ahrq.gov",
"aicheren.com", "aicheren.com",
"aicpa.org", "aicpa.org",
"aiellocondomini.it",
"aif.ru", "aif.ru",
"aihelp.net", "aihelp.net",
"aip.org", "aip.org",
@ -548,6 +548,7 @@
"alternet.org", "alternet.org",
"altervista.org", "altervista.org",
"altibbi.com", "altibbi.com",
"altreeq.com",
"alturl.com", "alturl.com",
"alwafd.news", "alwafd.news",
"alwakeelnews.com", "alwakeelnews.com",
@ -673,6 +674,7 @@
"ap.gov.in", "ap.gov.in",
"ap.org", "ap.org",
"ap7am.com", "ap7am.com",
"apa.az",
"apa.org", "apa.org",
"apache-ssl.org", "apache-ssl.org",
"apache.org", "apache.org",
@ -729,7 +731,6 @@
"arabi21.com", "arabi21.com",
"arabianbusiness.com", "arabianbusiness.com",
"arabnews.com", "arabnews.com",
"aracne.biz",
"aralego.com", "aralego.com",
"aralego.net", "aralego.net",
"arbeitsagentur.de", "arbeitsagentur.de",
@ -789,6 +790,7 @@
"asda.com", "asda.com",
"asha.org", "asha.org",
"ashemaletube.com", "ashemaletube.com",
"asiae.co.kr",
"asianetnews.com", "asianetnews.com",
"asiaone.com", "asiaone.com",
"asics.com", "asics.com",
@ -880,6 +882,7 @@
"avcdn.net", "avcdn.net",
"avclub.com", "avclub.com",
"avct.cloud", "avct.cloud",
"avery.com",
"avg.com", "avg.com",
"avgle.com", "avgle.com",
"aviasales.ru", "aviasales.ru",
@ -907,7 +910,6 @@
"ayobandung.com", "ayobandung.com",
"az.gov", "az.gov",
"azcentral.com", "azcentral.com",
"azfamily.com",
"azlyrics.com", "azlyrics.com",
"azure.com", "azure.com",
"azure.net", "azure.net",
@ -972,6 +974,7 @@
"bankofamerica.com", "bankofamerica.com",
"bankofengland.co.uk", "bankofengland.co.uk",
"bankrate.com", "bankrate.com",
"bannersnack.com",
"banorte.com", "banorte.com",
"banquepopulaire.fr", "banquepopulaire.fr",
"banvenez.com", "banvenez.com",
@ -1061,6 +1064,7 @@
"berlin.de", "berlin.de",
"bershka.com", "bershka.com",
"besoccer.com", "besoccer.com",
"best2019-games-web4.com",
"bestbrides.org", "bestbrides.org",
"bestbuy.ca", "bestbuy.ca",
"bestbuy.com", "bestbuy.com",
@ -1124,7 +1128,6 @@
"bing.net", "bing.net",
"binged.it", "binged.it",
"binghamton.edu", "binghamton.edu",
"binomo.com",
"binomoofficial.com", "binomoofficial.com",
"binomoworld.com", "binomoworld.com",
"biobiochile.cl", "biobiochile.cl",
@ -1132,6 +1135,7 @@
"biologists.org", "biologists.org",
"biomedcentral.com", "biomedcentral.com",
"biorxiv.org", "biorxiv.org",
"biqle.ru",
"birmingham.ac.uk", "birmingham.ac.uk",
"birminghammail.co.uk", "birminghammail.co.uk",
"bis.org", "bis.org",
@ -1153,6 +1157,7 @@
"bitmoji.com", "bitmoji.com",
"bitnami.com", "bitnami.com",
"bitrix24.com", "bitrix24.com",
"bitrix24.net",
"bitrix24.ru", "bitrix24.ru",
"bittorrent.com", "bittorrent.com",
"bittrex.com", "bittrex.com",
@ -1167,6 +1172,7 @@
"bjnews.com.cn", "bjnews.com.cn",
"bjs.gov", "bjs.gov",
"bjx.com.cn", "bjx.com.cn",
"bkm.com.tr",
"bkrtx.com", "bkrtx.com",
"bl.uk", "bl.uk",
"blackberry.com", "blackberry.com",
@ -1439,6 +1445,7 @@
"buzzsprout.com", "buzzsprout.com",
"buzzsumo.com", "buzzsumo.com",
"byjus.com", "byjus.com",
"byrdie.com",
"byrenjia.com", "byrenjia.com",
"bytedance.com", "bytedance.com",
"bytefence.com", "bytefence.com",
@ -1559,6 +1566,7 @@
"carsensor.net", "carsensor.net",
"carsforsale.com", "carsforsale.com",
"carto.com", "carto.com",
"carview.co.jp",
"carwale.com", "carwale.com",
"cas.cn", "cas.cn",
"casadellibro.com", "casadellibro.com",
@ -1635,7 +1643,9 @@
"center4family.com", "center4family.com",
"centos-webpanel.com", "centos-webpanel.com",
"centos.org", "centos.org",
"centrelink.gov.au",
"centrum.cz", "centrum.cz",
"centrum24.pl",
"centurylink.com", "centurylink.com",
"cern.ch", "cern.ch",
"cert.org", "cert.org",
@ -1777,6 +1787,7 @@
"cint.com", "cint.com",
"cio.com", "cio.com",
"cirquedusoleil.com", "cirquedusoleil.com",
"cisa.gov",
"cisco.com", "cisco.com",
"ciscospark.com", "ciscospark.com",
"cision.com", "cision.com",
@ -1807,8 +1818,10 @@
"clarin.com", "clarin.com",
"clarium.io", "clarium.io",
"clarivate.com", "clarivate.com",
"claro.com.br",
"classdojo.com", "classdojo.com",
"classiccars.com", "classiccars.com",
"clave.gob.es",
"clck.ru", "clck.ru",
"clean.gg", "clean.gg",
"cleanpng.com", "cleanpng.com",
@ -2024,7 +2037,6 @@
"cookpad.com", "cookpad.com",
"coolmathgames.com", "coolmathgames.com",
"coolors.co", "coolors.co",
"cooltext.com",
"cootek.com", "cootek.com",
"cootekservice.com", "cootekservice.com",
"cootlogix.com", "cootlogix.com",
@ -2388,7 +2400,6 @@
"digital-photography-school.com", "digital-photography-school.com",
"digital.com", "digital.com",
"digitalcommerce360.com", "digitalcommerce360.com",
"digitaleast.mobi",
"digitalguardian.com", "digitalguardian.com",
"digitaljournal.com", "digitaljournal.com",
"digitalocean.com", "digitalocean.com",
@ -2470,7 +2481,6 @@
"docin.com", "docin.com",
"docker.com", "docker.com",
"docomo.ne.jp", "docomo.ne.jp",
"docsity.com",
"docstoc.com", "docstoc.com",
"doctissimo.fr", "doctissimo.fr",
"doctolib.fr", "doctolib.fr",
@ -2481,7 +2491,6 @@
"docusign.net", "docusign.net",
"doda.jp", "doda.jp",
"doe.gov", "doe.gov",
"dogdrip.net",
"dogpile.com", "dogpile.com",
"doi.gov", "doi.gov",
"doi.org", "doi.org",
@ -2512,7 +2521,6 @@
"doodlekit.com", "doodlekit.com",
"doorblog.jp", "doorblog.jp",
"doordash.com", "doordash.com",
"doostihaa.com",
"doramatv.live", "doramatv.live",
"doramy.club", "doramy.club",
"dostor.org", "dostor.org",
@ -2606,7 +2614,6 @@
"dwz.cn", "dwz.cn",
"dx.com", "dx.com",
"dxy.cn", "dxy.cn",
"dy2018.com",
"dyn.com", "dyn.com",
"dynamicdrive.com", "dynamicdrive.com",
"dynamics.com", "dynamics.com",
@ -2733,6 +2740,7 @@
"egerton.ac.ke", "egerton.ac.ke",
"eghtesadnews.com", "eghtesadnews.com",
"eghtesadonline.com", "eghtesadonline.com",
"egnyte.com",
"egov.kz", "egov.kz",
"ehow.com", "ehow.com",
"eia.gov", "eia.gov",
@ -2819,6 +2827,7 @@
"enable-javascript.com", "enable-javascript.com",
"enamad.ir", "enamad.ir",
"ename.com", "ename.com",
"ename.net",
"enchantedlearning.com", "enchantedlearning.com",
"encuentra24.com", "encuentra24.com",
"encyclopedia.com", "encyclopedia.com",
@ -2938,7 +2947,6 @@
"evernote.com", "evernote.com",
"everydayhealth.com", "everydayhealth.com",
"evidon.com", "evidon.com",
"evite.com",
"ew.com", "ew.com",
"ewaybillgst.gov.in", "ewaybillgst.gov.in",
"eweek.com", "eweek.com",
@ -2994,6 +3002,7 @@
"f6s.com", "f6s.com",
"f95zone.to", "f95zone.to",
"faa.gov", "faa.gov",
"faberlic.com",
"fabric.com", "fabric.com",
"facebook.com", "facebook.com",
"facebook.net", "facebook.net",
@ -3075,7 +3084,6 @@
"feedbooks.com", "feedbooks.com",
"feedburner.com", "feedburner.com",
"feedingamerica.org", "feedingamerica.org",
"feediran.com",
"feedly.com", "feedly.com",
"feedspot.com", "feedspot.com",
"fehobmasr.com", "fehobmasr.com",
@ -3265,6 +3273,7 @@
"freedesktop.org", "freedesktop.org",
"freedomhouse.org", "freedomhouse.org",
"freedownloadmanager.org", "freedownloadmanager.org",
"freee.co.jp",
"freefiremobile.com", "freefiremobile.com",
"freehostia.com", "freehostia.com",
"freejobalert.com", "freejobalert.com",
@ -3372,7 +3381,6 @@
"gandi.net", "gandi.net",
"gangde.net", "gangde.net",
"gansu.gov.cn", "gansu.gov.cn",
"gansudaily.com.cn",
"gao.gov", "gao.gov",
"gao7.com", "gao7.com",
"gap.com", "gap.com",
@ -3452,6 +3460,7 @@
"getcomposer.org", "getcomposer.org",
"getcourse.ru", "getcourse.ru",
"getdonspeg.work", "getdonspeg.work",
"getdrip.com",
"getdropbox.com", "getdropbox.com",
"getfirefox.com", "getfirefox.com",
"getfvid.com", "getfvid.com",
@ -3484,6 +3493,7 @@
"gigaom.com", "gigaom.com",
"gigazine.net", "gigazine.net",
"gigya.com", "gigya.com",
"gilt.com",
"gimp.org", "gimp.org",
"gingerall.com", "gingerall.com",
"gingersoftware.com", "gingersoftware.com",
@ -3507,7 +3517,6 @@
"glamour.com", "glamour.com",
"glassdoor.co.in", "glassdoor.co.in",
"glassdoor.com", "glassdoor.com",
"glaz.tv",
"gleam.io", "gleam.io",
"globalcitizen.org", "globalcitizen.org",
"globalnews.ca", "globalnews.ca",
@ -3611,6 +3620,7 @@
"google.com.hk", "google.com.hk",
"google.com.kh", "google.com.kh",
"google.com.kw", "google.com.kw",
"google.com.lb",
"google.com.ly", "google.com.ly",
"google.com.mm", "google.com.mm",
"google.com.mx", "google.com.mx",
@ -3709,6 +3719,7 @@
"grainger.com", "grainger.com",
"graizoah.com", "graizoah.com",
"gramfree.net", "gramfree.net",
"gramfree.one",
"gramho.com", "gramho.com",
"grammarly.com", "grammarly.com",
"grammarly.io", "grammarly.io",
@ -3828,6 +3839,7 @@
"hapitas.jp", "hapitas.jp",
"haplat.net", "haplat.net",
"haqqin.az", "haqqin.az",
"haraj.com.sa",
"harborfreight.com", "harborfreight.com",
"hardened-php.net", "hardened-php.net",
"hardrock.com", "hardrock.com",
@ -3855,9 +3867,9 @@
"hbomax.com", "hbomax.com",
"hbr.org", "hbr.org",
"hbs.edu", "hbs.edu",
"hbx.com",
"hc360.com", "hc360.com",
"hclips.com", "hclips.com",
"hd315.gov.cn",
"hdfcbank.com", "hdfcbank.com",
"hdrezka-ag.com", "hdrezka-ag.com",
"he.net", "he.net",
@ -3998,7 +4010,6 @@
"hostinger.com", "hostinger.com",
"hotair.com", "hotair.com",
"hotbot.com", "hotbot.com",
"hotbrides.org",
"hotcleaner.com", "hotcleaner.com",
"hotels.com", "hotels.com",
"hotelscombined.com", "hotelscombined.com",
@ -4057,6 +4068,7 @@
"huaban.com", "huaban.com",
"huamu.cn", "huamu.cn",
"huangye88.com", "huangye88.com",
"huanjugu.com",
"huanqiu.com", "huanqiu.com",
"huatu.com", "huatu.com",
"huawei.com", "huawei.com",
@ -4272,7 +4284,6 @@
"inc.com", "inc.com",
"incapdns.net", "incapdns.net",
"incometaxindiaefiling.gov.in", "incometaxindiaefiling.gov.in",
"indaytrending.com",
"indeed.co.uk", "indeed.co.uk",
"indeed.com", "indeed.com",
"indeedassessments.com", "indeedassessments.com",
@ -4381,7 +4392,6 @@
"internic.net", "internic.net",
"interpark.com", "interpark.com",
"interpol.int", "interpol.int",
"interracialdatingapp.biz",
"intesasanpaolo.com", "intesasanpaolo.com",
"intipseleb.com", "intipseleb.com",
"intoday.in", "intoday.in",
@ -4412,7 +4422,6 @@
"ipify.org", "ipify.org",
"ipinfo.io", "ipinfo.io",
"ipko.pl", "ipko.pl",
"ipl.org",
"iplanet.com", "iplanet.com",
"iponweb.net", "iponweb.net",
"ipredictive.com", "ipredictive.com",
@ -4458,7 +4467,6 @@
"italki.com", "italki.com",
"itar-tass.com", "itar-tass.com",
"itau.com.br", "itau.com.br",
"itavcn.com",
"itch.io", "itch.io",
"iteye.com", "iteye.com",
"ithome.com", "ithome.com",
@ -4506,6 +4514,7 @@
"jalopnik.com", "jalopnik.com",
"jamanetwork.com", "jamanetwork.com",
"jamendo.com", "jamendo.com",
"jamesclear.com",
"jamieoliver.com", "jamieoliver.com",
"jamnews.com", "jamnews.com",
"jandan.net", "jandan.net",
@ -4590,7 +4599,6 @@
"joann.com", "joann.com",
"jobcan.jp", "jobcan.jp",
"jobinja.ir", "jobinja.ir",
"jobkorea.co.kr",
"jobrapido.com", "jobrapido.com",
"jobsdb.com", "jobsdb.com",
"jobui.com", "jobui.com",
@ -4637,6 +4645,7 @@
"jst.go.jp", "jst.go.jp",
"jstor.org", "jstor.org",
"jstv.com", "jstv.com",
"juejin.im",
"jugantor.com", "jugantor.com",
"jugem.jp", "jugem.jp",
"juicyads.com", "juicyads.com",
@ -4686,7 +4695,6 @@
"kalerkantho.com", "kalerkantho.com",
"kaltura.com", "kaltura.com",
"kampyle.com", "kampyle.com",
"kanal247.com",
"kansas.com", "kansas.com",
"kansascity.com", "kansascity.com",
"kantei.go.jp", "kantei.go.jp",
@ -4759,13 +4767,12 @@
"kingcounty.gov", "kingcounty.gov",
"kingsoft.com", "kingsoft.com",
"kingston.com", "kingston.com",
"kingtime.jp",
"kinja.com", "kinja.com",
"kino-teatr.ru", "kino-teatr.ru",
"kinogo.by", "kinogo.by",
"kinogo.zone",
"kinokrad.co", "kinokrad.co",
"kinopoisk.ru", "kinopoisk.ru",
"kinozal.tv",
"kinsta.com", "kinsta.com",
"kiplinger.com", "kiplinger.com",
"kiro7.com", "kiro7.com",
@ -4864,6 +4871,7 @@
"l-msedge.net", "l-msedge.net",
"la-croix.com", "la-croix.com",
"labanquepostale.fr", "labanquepostale.fr",
"labcorp.com",
"labiennale.org", "labiennale.org",
"labirint.ru", "labirint.ru",
"labnol.org", "labnol.org",
@ -5017,6 +5025,7 @@
"lifehack.org", "lifehack.org",
"lifehacker.com", "lifehacker.com",
"lifehacker.ru", "lifehacker.ru",
"lifeinaday.youtube",
"lifesitenews.com", "lifesitenews.com",
"lifewire.com", "lifewire.com",
"lifo.gr", "lifo.gr",
@ -5194,6 +5203,7 @@
"macworld.com", "macworld.com",
"macys.com", "macys.com",
"made-in-china.com", "made-in-china.com",
"madeiramadeira.com.br",
"madison.com", "madison.com",
"madmimi.com", "madmimi.com",
"madrid.org", "madrid.org",
@ -5222,6 +5232,7 @@
"main.jp", "main.jp",
"maine.gov", "maine.gov",
"mainichi.jp", "mainichi.jp",
"maisonmargiela.com",
"majorgeeks.com", "majorgeeks.com",
"makeleio.gr", "makeleio.gr",
"makemytrip.com", "makemytrip.com",
@ -5268,6 +5279,7 @@
"marines.mil", "marines.mil",
"marinetraffic.com", "marinetraffic.com",
"marist.edu", "marist.edu",
"marketbeat.com",
"markethive.com", "markethive.com",
"marketingland.com", "marketingland.com",
"marketingprofs.com", "marketingprofs.com",
@ -5385,7 +5397,6 @@
"mega.nz", "mega.nz",
"megafon.ru", "megafon.ru",
"megaphone.fm", "megaphone.fm",
"megaup.net",
"megaupload.com", "megaupload.com",
"megogo.net", "megogo.net",
"mehrnews.com", "mehrnews.com",
@ -5402,6 +5413,7 @@
"memurlar.net", "memurlar.net",
"menards.com", "menards.com",
"mendeley.com", "mendeley.com",
"meneame.net",
"menshealth.com", "menshealth.com",
"mensjournal.com", "mensjournal.com",
"mentalfloss.com", "mentalfloss.com",
@ -5436,7 +5448,6 @@
"metafilter.com", "metafilter.com",
"metal-archives.com", "metal-archives.com",
"metapress.com", "metapress.com",
"meteoblue.com",
"meteofrance.com", "meteofrance.com",
"meterserver.vn", "meterserver.vn",
"meti.go.jp", "meti.go.jp",
@ -5483,6 +5494,7 @@
"microchip.com", "microchip.com",
"microfocus.com", "microfocus.com",
"microsoft.com", "microsoft.com",
"microsoft.net",
"microsoft.us", "microsoft.us",
"microsoftazuread-sso.com", "microsoftazuread-sso.com",
"microsoftonline-p.com", "microsoftonline-p.com",
@ -5517,7 +5529,6 @@
"mindspring.com", "mindspring.com",
"mindtools.com", "mindtools.com",
"mindvalley.com", "mindvalley.com",
"mine.nu",
"minecraft.net", "minecraft.net",
"mingpao.com", "mingpao.com",
"miniclip.com", "miniclip.com",
@ -5569,7 +5580,6 @@
"mobile.de", "mobile.de",
"mobile.ir", "mobile.ir",
"mobile01.com", "mobile01.com",
"mobilesystemservice.com",
"mobinsb.com", "mobinsb.com",
"mobirise.com", "mobirise.com",
"mobirise.info", "mobirise.info",
@ -5612,6 +5622,7 @@
"mongabay.com", "mongabay.com",
"mongodb.com", "mongodb.com",
"mono3.biz", "mono3.biz",
"mono4.biz",
"mono5.biz", "mono5.biz",
"mono7.biz", "mono7.biz",
"monografias.com", "monografias.com",
@ -5655,10 +5666,8 @@
"movavi.com", "movavi.com",
"movember.com", "movember.com",
"moveon.org", "moveon.org",
"moviefone.com",
"movieranker.com", "movieranker.com",
"movieweb.com", "movieweb.com",
"movistar.es",
"movs4u.live", "movs4u.live",
"moz.com", "moz.com",
"mozaws.net", "mozaws.net",
@ -5752,7 +5761,6 @@
"myapp.com", "myapp.com",
"myasianbride.net", "myasianbride.net",
"mybigcommerce.com", "mybigcommerce.com",
"mybuzz.fun",
"mycima.me", "mycima.me",
"mydailymagazine.com", "mydailymagazine.com",
"mydealz.de", "mydealz.de",
@ -5905,7 +5913,6 @@
"ndr.de", "ndr.de",
"ndrc.gov.cn", "ndrc.gov.cn",
"ndtv.com", "ndtv.com",
"nea.gov.cn",
"nea.org", "nea.org",
"nearme.com.cn", "nearme.com.cn",
"nebraska.gov", "nebraska.gov",
@ -5970,6 +5977,7 @@
"news-medical.net", "news-medical.net",
"news.cn", "news.cn",
"news.com.au", "news.com.au",
"news12.com",
"news18.com", "news18.com",
"news24.com", "news24.com",
"news247.gr", "news247.gr",
@ -5980,7 +5988,6 @@
"newscientist.com", "newscientist.com",
"newsday.com", "newsday.com",
"newser.com", "newser.com",
"newsgator.com",
"newsit.gr", "newsit.gr",
"newsmax.com", "newsmax.com",
"newsmth.net", "newsmth.net",
@ -6154,7 +6161,6 @@
"ntv.com.tr", "ntv.com.tr",
"ntv.io", "ntv.io",
"ntv.ru", "ntv.ru",
"ntvtelugu.com",
"nu.nl", "nu.nl",
"nuance.com", "nuance.com",
"nudevista.com", "nudevista.com",
@ -6196,7 +6202,6 @@
"oaspapps.com", "oaspapps.com",
"oath.cloud", "oath.cloud",
"oath.com", "oath.com",
"oberlin.edu",
"oberlo.com", "oberlo.com",
"obozrevatel.com", "obozrevatel.com",
"observalgerie.com", "observalgerie.com",
@ -6343,6 +6348,7 @@
"opentext.com", "opentext.com",
"openvpn.net", "openvpn.net",
"openweathermap.org", "openweathermap.org",
"openwrt.org",
"openx.com", "openx.com",
"openx.net", "openx.net",
"opera-mini.net", "opera-mini.net",
@ -6385,7 +6391,6 @@
"oschina.net", "oschina.net",
"osd.mil", "osd.mil",
"osha.gov", "osha.gov",
"ospserver.net",
"osticket.com", "osticket.com",
"osu.edu", "osu.edu",
"osxdaily.com", "osxdaily.com",
@ -6649,7 +6654,6 @@
"pingdom.net", "pingdom.net",
"pinimg.com", "pinimg.com",
"pinkbike.com", "pinkbike.com",
"pinterest.at",
"pinterest.ca", "pinterest.ca",
"pinterest.co.uk", "pinterest.co.uk",
"pinterest.com", "pinterest.com",
@ -6718,6 +6722,7 @@
"poetryfoundation.org", "poetryfoundation.org",
"poets.org", "poets.org",
"pof.com", "pof.com",
"point2homes.com",
"pojoksatu.id", "pojoksatu.id",
"pokemon.com", "pokemon.com",
"poki.com", "poki.com",
@ -6778,13 +6783,13 @@
"potterybarn.com", "potterybarn.com",
"powells.com", "powells.com",
"powerapp.download", "powerapp.download",
"powerapps.com",
"powerbi.com", "powerbi.com",
"powerlinks.com", "powerlinks.com",
"powerofpositivity.com", "powerofpositivity.com",
"powerthesaurus.org", "powerthesaurus.org",
"powtoon.com", "powtoon.com",
"poynter.org", "poynter.org",
"pozdravok.ru",
"poznan.pl", "poznan.pl",
"pp.ru", "pp.ru",
"pp.ua", "pp.ua",
@ -6890,6 +6895,7 @@
"publpush.com", "publpush.com",
"pubmatic.com", "pubmatic.com",
"pulitzer.org", "pulitzer.org",
"pullcf.com",
"pullcm.com", "pullcm.com",
"pulzo.com", "pulzo.com",
"puma.com", "puma.com",
@ -6947,7 +6953,6 @@
"qq.com", "qq.com",
"qr-code-generator.com", "qr-code-generator.com",
"qsl.net", "qsl.net",
"qt.io",
"qualaroo.com", "qualaroo.com",
"qualcomm.com", "qualcomm.com",
"qualtrics.com", "qualtrics.com",
@ -6956,7 +6961,6 @@
"quantamagazine.org", "quantamagazine.org",
"quantcast.com", "quantcast.com",
"quantcount.com", "quantcount.com",
"quantrimang.com",
"quantserve.com", "quantserve.com",
"qub.ac.uk", "qub.ac.uk",
"queensu.ca", "queensu.ca",
@ -7033,6 +7037,7 @@
"rasadeghtesadi.com", "rasadeghtesadi.com",
"rasadvarzeshi.com", "rasadvarzeshi.com",
"raspberrypi.org", "raspberrypi.org",
"ratemyprofessors.com",
"rateyourmusic.com", "rateyourmusic.com",
"ratopati.com", "ratopati.com",
"ravelry.com", "ravelry.com",
@ -7108,6 +7113,7 @@
"refinery29.com", "refinery29.com",
"reg.ru", "reg.ru",
"regions.com", "regions.com",
"register.it",
"regjeringen.no", "regjeringen.no",
"regnum.ru", "regnum.ru",
"regonline.com", "regonline.com",
@ -7142,7 +7148,6 @@
"resetera.com", "resetera.com",
"residentadvisor.net", "residentadvisor.net",
"reson8.com", "reson8.com",
"restream.io",
"resultados-futbol.com", "resultados-futbol.com",
"retaildive.com", "retaildive.com",
"retailmenot.com", "retailmenot.com",
@ -7156,7 +7161,6 @@
"reverso.net", "reverso.net",
"reviewjournal.com", "reviewjournal.com",
"revjet.com", "revjet.com",
"revolut.com",
"revopush.com", "revopush.com",
"revues.org", "revues.org",
"rezka.ag", "rezka.ag",
@ -7318,14 +7322,12 @@
"sabio.us", "sabio.us",
"sabq.org", "sabq.org",
"sacbee.com", "sacbee.com",
"sachsen.de",
"sacred-texts.com", "sacred-texts.com",
"sae.org", "sae.org",
"saednews.com", "saednews.com",
"safedog.cn", "safedog.cn",
"safefinder.com", "safefinder.com",
"safer-networking.org", "safer-networking.org",
"safeway.com",
"sagawa-exp.co.jp", "sagawa-exp.co.jp",
"sage.com", "sage.com",
"sagepub.com", "sagepub.com",
@ -7431,6 +7433,7 @@
"sc.gov", "sc.gov",
"sc.gov.cn", "sc.gov.cn",
"scamadviser.com", "scamadviser.com",
"scarlet-clicks.info",
"scdn.co", "scdn.co",
"scene7.com", "scene7.com",
"sch.gr", "sch.gr",
@ -7544,6 +7547,7 @@
"sendgrid.net", "sendgrid.net",
"sendinblue.com", "sendinblue.com",
"sendo.vn", "sendo.vn",
"sendpulse.com",
"sendspace.com", "sendspace.com",
"seneweb.com", "seneweb.com",
"sennheiser.com", "sennheiser.com",
@ -7566,6 +7570,7 @@
"servimg.com", "servimg.com",
"serving-sys.com", "serving-sys.com",
"setare.com", "setare.com",
"setareparsi.com",
"setn.com", "setn.com",
"seu.edu.cn", "seu.edu.cn",
"seventeen.com", "seventeen.com",
@ -7648,7 +7653,6 @@
"shopifysvc.com", "shopifysvc.com",
"shopstyle.co.uk", "shopstyle.co.uk",
"shopstyle.com", "shopstyle.com",
"shoptime.com.br",
"shorturl.at", "shorturl.at",
"shrm.org", "shrm.org",
"shueisha.co.jp", "shueisha.co.jp",
@ -7792,7 +7796,6 @@
"snapads.com", "snapads.com",
"snapchat.com", "snapchat.com",
"snapdeal.com", "snapdeal.com",
"snapfish.com",
"snapwidget.com", "snapwidget.com",
"snazzymaps.com", "snazzymaps.com",
"sncf.com", "sncf.com",
@ -7892,6 +7895,7 @@
"splice.com", "splice.com",
"splunk.com", "splunk.com",
"spo-msedge.net", "spo-msedge.net",
"spocket.co",
"spokeo.com", "spokeo.com",
"spokesman.com", "spokesman.com",
"sponichi.co.jp", "sponichi.co.jp",
@ -7963,7 +7967,6 @@
"stackpathcdn.com", "stackpathcdn.com",
"stackpathdns.com", "stackpathdns.com",
"stacksocial.com", "stacksocial.com",
"stan.com.au",
"standaard.be", "standaard.be",
"standard.co.uk", "standard.co.uk",
"standardchartered.com", "standardchartered.com",
@ -8000,7 +8003,6 @@
"steam-chat.com", "steam-chat.com",
"steamcommunity.com", "steamcommunity.com",
"steamcontent.com", "steamcontent.com",
"steamdb.info",
"steampowered.com", "steampowered.com",
"steamstatic.com", "steamstatic.com",
"steemit.com", "steemit.com",
@ -8029,7 +8031,6 @@
"storify.com", "storify.com",
"storm.mg", "storm.mg",
"storygize.net", "storygize.net",
"straightdope.com",
"straitstimes.com", "straitstimes.com",
"strangled.net", "strangled.net",
"strath.ac.uk", "strath.ac.uk",
@ -8049,7 +8050,6 @@
"studfile.net", "studfile.net",
"studiopress.com", "studiopress.com",
"studocu.com", "studocu.com",
"studopedia.ru",
"study.com", "study.com",
"stuff.co.nz", "stuff.co.nz",
"stumbleupon.com", "stumbleupon.com",
@ -8162,13 +8162,11 @@
"tabnak.ir", "tabnak.ir",
"taboola.com", "taboola.com",
"tadalafilprof.com", "tadalafilprof.com",
"tagar.id",
"tagesanzeiger.ch", "tagesanzeiger.ch",
"tagesschau.de", "tagesschau.de",
"tagesspiegel.de", "tagesspiegel.de",
"tagged.com", "tagged.com",
"tahiamasr.com", "tahiamasr.com",
"taifeng-chem.com",
"taipeitimes.com", "taipeitimes.com",
"taiwannews.com.tw", "taiwannews.com.tw",
"takealot.com", "takealot.com",
@ -8226,6 +8224,7 @@
"teamtreehouse.com", "teamtreehouse.com",
"teamusa.org", "teamusa.org",
"teamviewer.com", "teamviewer.com",
"techacademy.jp",
"techadvisor.co.uk", "techadvisor.co.uk",
"techbang.com", "techbang.com",
"techcrunch.com", "techcrunch.com",
@ -8249,6 +8248,7 @@
"techwalla.com", "techwalla.com",
"techweb.com.cn", "techweb.com.cn",
"tecmundo.com.br", "tecmundo.com.br",
"tecnoblog.net",
"ted.com", "ted.com",
"teenvogue.com", "teenvogue.com",
"teepublic.com", "teepublic.com",
@ -8306,7 +8306,6 @@
"test.de", "test.de",
"testbook.com", "testbook.com",
"texas.gov", "texas.gov",
"texasmonthly.com",
"texastribune.org", "texastribune.org",
"text.ru", "text.ru",
"textnow.com", "textnow.com",
@ -8318,6 +8317,7 @@
"thanhnien.vn", "thanhnien.vn",
"thawte.com", "thawte.com",
"the-scientist.com", "the-scientist.com",
"the-sun.com",
"theadvocate.com", "theadvocate.com",
"theage.com.au", "theage.com.au",
"theathletic.com", "theathletic.com",
@ -8396,7 +8396,6 @@
"theonion.com", "theonion.com",
"theoutnet.com", "theoutnet.com",
"thepaper.cn", "thepaper.cn",
"theparisreview.org",
"thepetitionsite.com", "thepetitionsite.com",
"thepirate-bay.org", "thepirate-bay.org",
"thepiratebay.org", "thepiratebay.org",
@ -8416,7 +8415,6 @@
"theroot.com", "theroot.com",
"thesaurus.com", "thesaurus.com",
"thesimpledollar.com", "thesimpledollar.com",
"thesimsresource.com",
"thesmokinggun.com", "thesmokinggun.com",
"thespruce.com", "thespruce.com",
"thespruceeats.com", "thespruceeats.com",
@ -8468,7 +8466,6 @@
"thriveglobal.com", "thriveglobal.com",
"thrtle.com", "thrtle.com",
"thumbtack.com", "thumbtack.com",
"thumbzilla.com",
"thunderbird.net", "thunderbird.net",
"ti.com", "ti.com",
"tianqi.com", "tianqi.com",
@ -8625,6 +8622,7 @@
"transunion.com", "transunion.com",
"travelandleisure.com", "travelandleisure.com",
"travelchannel.com", "travelchannel.com",
"travelermaster.com",
"travelocity.com", "travelocity.com",
"traveloka.com", "traveloka.com",
"travis-ci.org", "travis-ci.org",
@ -8655,7 +8653,6 @@
"tripadvisor.ca", "tripadvisor.ca",
"tripadvisor.co.uk", "tripadvisor.co.uk",
"tripadvisor.com", "tripadvisor.com",
"tripadvisor.com.au",
"tripadvisor.de", "tripadvisor.de",
"tripadvisor.es", "tripadvisor.es",
"tripadvisor.fr", "tripadvisor.fr",
@ -8677,7 +8674,6 @@
"truste.com", "truste.com",
"trustedreviews.com", "trustedreviews.com",
"trustpilot.com", "trustpilot.com",
"trustutn.org",
"trustwave.com", "trustwave.com",
"trustx.org", "trustx.org",
"truthout.org", "truthout.org",
@ -8731,7 +8727,6 @@
"tutorialspoint.com", "tutorialspoint.com",
"tutsplus.com", "tutsplus.com",
"tutu.ru", "tutu.ru",
"tuv.com",
"tuwien.ac.at", "tuwien.ac.at",
"tv-asahi.co.jp", "tv-asahi.co.jp",
"tv-tokyo.co.jp", "tv-tokyo.co.jp",
@ -8837,7 +8832,6 @@
"udn.com", "udn.com",
"uea.ac.uk", "uea.ac.uk",
"uefa.com", "uefa.com",
"ueuodgnrhb.com",
"ufc.com", "ufc.com",
"ufl.edu", "ufl.edu",
"ufreegames.com", "ufreegames.com",
@ -8894,7 +8888,6 @@
"umengcloud.com", "umengcloud.com",
"umich.edu", "umich.edu",
"umkc.edu", "umkc.edu",
"umm.ac.id",
"umm.edu", "umm.edu",
"umms.org", "umms.org",
"umn.edu", "umn.edu",
@ -9007,6 +9000,7 @@
"unwto.org", "unwto.org",
"uny.ac.id", "uny.ac.id",
"uoa.gr", "uoa.gr",
"uoc.edu",
"uodoo.com", "uodoo.com",
"uoguelph.ca", "uoguelph.ca",
"uol.com.br", "uol.com.br",
@ -9045,6 +9039,7 @@
"urdupoint.com", "urdupoint.com",
"uri.edu", "uri.edu",
"url.cn", "url.cn",
"urldefense.com",
"urssaf.fr", "urssaf.fr",
"us-cert.gov", "us-cert.gov",
"us.com", "us.com",
@ -9102,6 +9097,7 @@
"uta.edu", "uta.edu",
"utah.edu", "utah.edu",
"utah.gov", "utah.gov",
"utas.edu.au",
"utdallas.edu", "utdallas.edu",
"utep.edu", "utep.edu",
"utexas.edu", "utexas.edu",
@ -9136,6 +9132,7 @@
"v2ex.com", "v2ex.com",
"va.gov", "va.gov",
"vagina.nl", "vagina.nl",
"valentino.com",
"validcbdoil.com", "validcbdoil.com",
"value-domain.com", "value-domain.com",
"valuecommerce.com", "valuecommerce.com",
@ -9157,7 +9154,6 @@
"vbspbx.com", "vbspbx.com",
"vbulletin.com", "vbulletin.com",
"vc.ru", "vc.ru",
"vchecks.me",
"vcu.edu", "vcu.edu",
"vecteezy.com", "vecteezy.com",
"vector.co.jp", "vector.co.jp",
@ -9208,6 +9204,7 @@
"videopress.com", "videopress.com",
"vidible.tv", "vidible.tv",
"vidio.com", "vidio.com",
"vidiq.com",
"vidstream.kim", "vidstream.kim",
"vidstream.online", "vidstream.online",
"vidstream.to", "vidstream.to",
@ -9225,6 +9222,7 @@
"vimeopro.com", "vimeopro.com",
"vindicosuite.com", "vindicosuite.com",
"vine.co", "vine.co",
"vingle.net",
"vip.com", "vip.com",
"virgilio.it", "virgilio.it",
"virgin.com", "virgin.com",
@ -9331,6 +9329,7 @@
"walgreens.com", "walgreens.com",
"walkerland.com.tw", "walkerland.com.tw",
"walkme.com", "walkme.com",
"walkscore.com",
"walla.co.il", "walla.co.il",
"wallapop.com", "wallapop.com",
"wallethub.com", "wallethub.com",
@ -9711,7 +9710,6 @@
"xhamster.one", "xhamster.one",
"xhamster3.desi", "xhamster3.desi",
"xhamsterlive.com", "xhamsterlive.com",
"xhamsterpremium.com",
"xhby.net", "xhby.net",
"xiachufang.com", "xiachufang.com",
"xiami.com", "xiami.com",
@ -9832,6 +9830,7 @@
"yjc.ir", "yjc.ir",
"yjtag.jp", "yjtag.jp",
"yle.fi", "yle.fi",
"ylsw.com",
"ymp4.download", "ymp4.download",
"yn.gov.cn", "yn.gov.cn",
"yna.co.kr", "yna.co.kr",
@ -9868,9 +9867,9 @@
"youradchoices.com", "youradchoices.com",
"yourdictionary.com", "yourdictionary.com",
"yourdomain.com", "yourdomain.com",
"yournewtab.com",
"youronlinechoices.com", "youronlinechoices.com",
"youronlinechoices.eu", "youronlinechoices.eu",
"yoursportspot.com",
"yourstory.com", "yourstory.com",
"yourtango.com", "yourtango.com",
"yousendit.com", "yousendit.com",
@ -9996,6 +9995,7 @@
"zozo.jp", "zozo.jp",
"zscaler.com", "zscaler.com",
"zulily.com", "zulily.com",
"zum.com",
"zumiez.com", "zumiez.com",
"zurb.com", "zurb.com",
"zxart.cn", "zxart.cn",
@ -10010,5 +10010,5 @@
], ],
"name": "Top 10K most-used sites from Tranco", "name": "Top 10K most-used sites from Tranco",
"type": "hostname", "type": "hostname",
"version": 20200720 "version": 20200723
} }

4
tools/generate-amazon-aws.py
View File

@ -3,11 +3,11 @@
import json import json
from generator import download_to_file, get_version, write_to_file from generator import download_to_file, get_version, write_to_file, get_abspath_source_file
def process(file, dst): def process(file, dst):
with open(file, 'r') as json_file: with open(get_abspath_source_file(file), 'r') as json_file:
amazon_aws_ip_list = json.load(json_file) amazon_aws_ip_list = json.load(json_file)
l = [] l = []

4
tools/generate-cisco.py
View File

@ -3,7 +3,7 @@
import zipfile import zipfile
from generator import download_to_file, get_version, write_to_file from generator import download_to_file, get_version, write_to_file, get_abspath_source_file
def process(file): def process(file):
@ -53,7 +53,7 @@ def generate(sites, warninglist, dst):
def get_lists(file): def get_lists(file):
with zipfile.ZipFile(file, 'r') as cisco_lists: with zipfile.ZipFile(get_abspath_source_file(file), 'r') as cisco_lists:
for name in cisco_lists.namelist(): for name in cisco_lists.namelist():
if name == "top-1m.csv": if name == "top-1m.csv":
with cisco_lists.open(name) as cisco_list: with cisco_lists.open(name) as cisco_list:

4
tools/generate-cloudflare.py
View File

@ -1,7 +1,7 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
from generator import download_to_file, get_version, write_to_file from generator import download_to_file, get_version, write_to_file, get_abspath_source_file
def process(files, dst): def process(files, dst):
@ -15,7 +15,7 @@ def process(files, dst):
} }
for file in files: for file in files:
with open(file, 'r') as f: with open(get_abspath_source_file(file), 'r') as f:
ips = f.readlines() ips = f.readlines()
for ip in ips: for ip in ips:
warninglist['list'].append(ip.strip()) warninglist['list'].append(ip.strip())

4
tools/generate-crl-ip-list.py
View File

@ -1,7 +1,7 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
from generator import download_to_file, get_version, write_to_file from generator import download_to_file, get_version, write_to_file, get_abspath_source_file
def process(files, dst): def process(files, dst):
@ -16,7 +16,7 @@ def process(files, dst):
} }
for file in files: for file in files:
with open(file, 'r') as f: with open(get_abspath_source_file(file), 'r') as f:
ips = f.readlines() ips = f.readlines()
for ip in ips: for ip in ips:
warninglist['list'].append(ip.strip()) warninglist['list'].append(ip.strip())

4
tools/generate-microsoft-azure.py
View File

@ -5,7 +5,7 @@ import json
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from generator import download, download_to_file, get_version, write_to_file from generator import download, download_to_file, get_version, write_to_file, get_abspath_source_file
def get_json_url(page): def get_json_url(page):
@ -26,7 +26,7 @@ def process(file, dst):
'type': 'cidr' 'type': 'cidr'
} }
with open(file, 'r') as json_file: with open(get_abspath_source_file(file), 'r') as json_file:
ms_azure_ip_list = json.load(json_file) ms_azure_ip_list = json.load(json_file)
for value in ms_azure_ip_list['values']: for value in ms_azure_ip_list['values']:

4
tools/generate-publicdns.py
View File

@ -5,7 +5,7 @@ import csv
import ipaddress import ipaddress
import logging import logging
from generator import download_to_file, get_version, write_to_file from generator import download_to_file, get_version, write_to_file, get_abspath_source_file
def process(file): def process(file):
@ -52,7 +52,7 @@ def generate(data_list, warninglist, dst):
def get_lists(file): def get_lists(file):
with open(file) as csv_file: with open(get_abspath_source_file(file)) as csv_file:
servers_list = csv.reader(csv_file, delimiter=',', quotechar='"') servers_list = csv.reader(csv_file, delimiter=',', quotechar='"')
lipv4 = [] lipv4 = []

5
tools/generate_alexa.py
View File

@ -3,11 +3,12 @@
import zipfile import zipfile
from generator import download_to_file, get_version, write_to_file from generator import (download_to_file, get_abspath_source_file, get_version,
write_to_file)
def process(file, dst): def process(file, dst):
with zipfile.ZipFile(file, 'r') as alexa_lists: with zipfile.ZipFile(get_abspath_source_file(file), 'r') as alexa_lists:
for name in alexa_lists.namelist(): for name in alexa_lists.namelist():
if name == "top-1m.csv": if name == "top-1m.csv":
with alexa_lists.open(name) as top: with alexa_lists.open(name) as top:

4
tools/generate_majestic-million.py
View File

@ -1,12 +1,12 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
from generator import download_to_file, get_version, write_to_file from generator import download_to_file, get_version, write_to_file, get_abspath_source_file
def process(file, dst): def process(file, dst):
with open(file, newline='\n', encoding='utf-8', errors='replace') as csv_file: with open(get_abspath_source_file(file), newline='\n', encoding='utf-8', errors='replace') as csv_file:
sites = csv_file.readlines()[:10000] sites = csv_file.readlines()[:10000]
warninglist = { warninglist = {

83
tools/generate_moz-top500.py
View File

@ -1,69 +1,44 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import requests
import datetime
import json
import csv import csv
import os
from generator import download_to_file, get_version, write_to_file, get_abspath_source_file
# TODO: Include MozRank # TODO: Include MozRank
def process(files, dst):
moz_url_domains = "https://moz.com/top500/domains/csv" warninglist = {
moz_url_pages = "https://moz.com/top500/pages/csv" 'description': "Event contains one or more entries from the top 500 of the most used domains (Mozilla).",
'version': get_version(),
'name': "Top 500 domains and pages from https://moz.com/top500",
'type': 'hostname',
'list': [],
'matching_attributes': ['hostname', 'domain', 'uri', 'url']
}
moz_file_domains = "/tmp/top500.domains.csv" for file in files:
moz_file_pages = "/tmp/top500.pages.csv" with open(get_abspath_source_file(file)) as csv_file:
csv_reader = csv.reader(csv_file, delimiter=',')
for row in csv_reader:
v = row[1]
warninglist['list'].append(v.rstrip().rstrip('/'))
user_agent = {"User-agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:46.0) Gecko/20100101 Firefox/46.0"} write_to_file(warninglist, dst)
rDomains = requests.get(moz_url_domains, headers=user_agent)
rPages = requests.get(moz_url_pages, headers=user_agent)
open(moz_file_domains, 'wb').write(rDomains.content)
open(moz_file_pages, 'wb').write(rPages.content)
moz_warninglist = {} if __name__ == '__main__':
version = int(datetime.date.today().strftime('%Y%m%d')) moz_domains_url = "https://moz.com/top-500/download/?table=top500Domains"
#moz_pages_url = "https://moz.com/top500/pages/csv"
moz_warninglist['description'] = "Event contains one or more entries from the top 500 of the most used domains (Mozilla)." moz_domains_file = "moz-top500.domains.csv"
d = datetime.datetime.now() #moz_pages_file = "moz-top500.pages.csv"
moz_warninglist['version'] = version
moz_warninglist['name'] = "Top 500 domains and pages from https://moz.com/top500"
moz_warninglist['type'] = 'hostname'
moz_warninglist['list'] = []
moz_warninglist['matching_attributes'] = ['hostname', 'domain', 'uri', 'url']
with open(moz_file_domains) as csv_file: moz_dst = 'moz-top500'
csv_reader = csv.reader(csv_file, delimiter=',')
line_count = 0
for row in csv_reader:
if line_count == 0:
#print(f'Column names are {", ".join(row)}')
line_count += 1
else:
#print(f'\t{row[0]}. {row[1]}, MozTrust: {row[5]}.')
v = row[1]
moz_warninglist['list'].append(v.rstrip().rstrip('/'))
line_count += 1
with open(moz_file_pages) as csv_file: download_to_file(moz_domains_url, moz_domains_file)
csv_reader = csv.reader(csv_file, delimiter=',') #download_to_file(moz_pages_url, moz_pages_file)
line_count = 0
for row in csv_reader:
if line_count == 0:
#print(f'Column names are {", ".join(row)}')
line_count += 1
else:
#print(f'\t{row[0]}. {row[1]}, MozTrust: {row[5]}.')
v = row[1]
moz_warninglist['list'].append(v.rstrip().rstrip('/'))
line_count += 1
moz_warninglist['list'] = sorted(set(moz_warninglist['list'])) #process([moz_domains_file, moz_pages_file], moz_dst)
print(json.dumps(moz_warninglist)) process([moz_domains_file], moz_dst)
try:
os.remove(moz_file_domains)
os.remove(moz_file_pages)
except:
print(f'Perhaps {moz_file_domains}/{moz_file_pages} does not exist.')

4
tools/generate_mozilla_certificates.py
View File

@ -5,7 +5,7 @@ import csv
from OpenSSL.crypto import FILETYPE_PEM, load_certificate from OpenSSL.crypto import FILETYPE_PEM, load_certificate
from generator import download_to_file, get_version, write_to_file from generator import download_to_file, get_version, write_to_file, get_abspath_source_file
def gethash(cert, digest): def gethash(cert, digest):
@ -14,7 +14,7 @@ def gethash(cert, digest):
def process(file, dst, type): def process(file, dst, type):
hashes = set() hashes = set()
with open(file, 'r') as f_in: with open(get_abspath_source_file(file), 'r') as f_in:
for obj in csv.DictReader(f_in): for obj in csv.DictReader(f_in):
pem = obj['PEM Info'].strip("'").replace( pem = obj['PEM Info'].strip("'").replace(
'\r', '').replace('\n\n', '\n') '\r', '').replace('\n\n', '\n')

4
tools/generate_tranco.py
View File

@ -3,7 +3,7 @@
import zipfile import zipfile
from generator import download_to_file, get_version, write_to_file from generator import download_to_file, get_version, write_to_file, get_abspath_source_file
def process(file): def process(file):
@ -42,7 +42,7 @@ def generate(sites, warninglist, dst):
def get_lists(file): def get_lists(file):
with zipfile.ZipFile(file, 'r') as tranco_lists: with zipfile.ZipFile(get_abspath_source_file(file), 'r') as tranco_lists:
for name in tranco_lists.namelist(): for name in tranco_lists.namelist():
if name == 'top-1m.csv': if name == 'top-1m.csv':
with tranco_lists.open(name) as tranco: with tranco_lists.open(name) as tranco:

97
tools/generator.py
View File

@ -3,49 +3,90 @@
import datetime import datetime
import json import json
from inspect import currentframe, getframeinfo
from os import path
import logging import logging
from inspect import currentframe, getframeinfo, getmodulename, stack
from os import mkdir, path
import requests import requests
from dateutil.parser import parse as parsedate from dateutil.parser import parse as parsedate
def init_logging():
rel_path = getframeinfo(currentframe()).filename
current_folder = path.dirname(path.abspath(rel_path))
LOG_DIR = path.join(current_folder, '../generators.log')
logFormatter = logging.Formatter(
"[%(asctime)s] %(levelname)s::%(funcName)s()::%(message)s")
rootLogger = logging.getLogger()
rootLogger.setLevel(logging.INFO)
# Log to file
fileHandler = logging.FileHandler(LOG_DIR)
fileHandler.setFormatter(logFormatter)
rootLogger.addHandler(fileHandler)
# Log to console too
''' consoleHandler = logging.StreamHandler()
consoleHandler.setFormatter(logFormatter)
rootLogger.addHandler(consoleHandler) '''
return rootLogger
init_logging()
def download_to_file(url, file): def download_to_file(url, file):
user_agent = {"User-agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:46.0) Gecko/20100101 Firefox/46.0"} frame_records = stack()[1]
caller = getmodulename(frame_records[1]).upper()
user_agent = {
"User-agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:46.0) Gecko/20100101 Firefox/46.0"}
try: try:
r = requests.head(url, headers=user_agent) r = requests.head(url, headers=user_agent)
url_datetime = parsedate(r.headers['Last-Modified']).astimezone() url_datetime = parsedate(r.headers['Last-Modified']).astimezone()
file_datetime = datetime.datetime.fromtimestamp( file_datetime = datetime.datetime.fromtimestamp(
path.getmtime(file)).astimezone() path.getmtime(get_abspath_source_file(file))).astimezone()
if(url_datetime > file_datetime): if(url_datetime > file_datetime):
logging.info('{} File on server is newer, so downloading update to {}'.format(
caller, get_abspath_source_file(file)))
actual_download_to_file(url, file, user_agent) actual_download_to_file(url, file, user_agent)
except KeyError as ex: else:
logging.warning(str(ex)) logging.info(
'{} File on server is older, nothing to do'.format(caller))
except KeyError as exc:
logging.warning('{} KeyError in the headers. the {} header was not sent by server {}. Downloading file'.format(
caller, str(exc), url))
actual_download_to_file(url, file, user_agent) actual_download_to_file(url, file, user_agent)
except FileNotFoundError as exc:
logging.info(
"{} File didn't exist, so downloading {} from {}".format(caller, file, url))
actual_download_to_file(url, file, user_agent)
except Exception as exc:
logging.warning(
'{} General exception occured: {}.'.format(caller, str(exc)))
actual_download_to_file(url, file, user_agent)
def actual_download_to_file(url, file, user_agent): def actual_download_to_file(url, file, user_agent):
r = requests.get(url, headers=user_agent) r = requests.get(url, headers=user_agent)
with open(file, 'wb') as fd: with open(get_abspath_source_file(file), 'wb') as fd:
for chunk in r.iter_content(4096): for chunk in r.iter_content(4096):
fd.write(chunk) fd.write(chunk)
def process_stream(url): def process_stream(url):
r = requests.get(url, stream=True) r = requests.get(url, stream=True)
data_list = [] data_list = []
for line in r.iter_lines(): for line in r.iter_lines():
v = line.decode('utf-8') v = line.decode('utf-8')
if not v.startswith("#"): if not v.startswith("#"):
if v: if v:
data_list.append(v) data_list.append(v)
return data_list return data_list
def download(url): def download(url):
user_agent = { user_agent = {
"User-agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:46.0) Gecko/20100101 Firefox/46.0"} "User-agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:46.0) Gecko/20100101 Firefox/46.0"}
@ -60,6 +101,15 @@ def get_abspath_list_file(dst):
return path.abspath(path.realpath(real_path)) return path.abspath(path.realpath(real_path))
def get_abspath_source_file(dst):
rel_path = getframeinfo(currentframe()).filename
current_folder = path.dirname(path.abspath(rel_path))
tmp_path = path.join(current_folder, '../tmp/')
if not path.exists(tmp_path):
mkdir(tmp_path)
return path.abspath(path.realpath(path.join(tmp_path, '{dst}'.format(dst=dst))))
def get_version(): def get_version():
return int(datetime.date.today().strftime('%Y%m%d')) return int(datetime.date.today().strftime('%Y%m%d'))
@ -70,7 +120,24 @@ def unique_sorted_warninglist(warninglist):
def write_to_file(warninglist, dst): def write_to_file(warninglist, dst):
with open(get_abspath_list_file(dst), 'w') as data_file: frame_records = stack()[1]
json.dump(unique_sorted_warninglist(warninglist), caller = getmodulename(frame_records[1]).upper()
data_file, indent=2, sort_keys=True)
data_file.write("\n") try:
with open(get_abspath_list_file(dst), 'w') as data_file:
json.dump(unique_sorted_warninglist(warninglist),
data_file, indent=2, sort_keys=True)
data_file.write("\n")
logging.info('New warninglist written to {}.'.format(
get_abspath_list_file(dst)))
except Exception as exc:
logging.error(
'{} General exception occured: {}.'.format(caller, str(exc)))
def main():
init_logging()
if __name__ == '__main__':
main()