Refactor last scripts, logging, central directory for downloads
* Refactored generate_moz-top50.py * Download all file to new /tmp file to centralize all downloads * Add central logging to generators.log file * Create Bash script that generates all warninglists * Add /tmp folder and extra files to .gitignore * Start adding exception handling in download_to_file and write_to_filepull/162/head
Kevin Holvoet
parent
610292e90f
commit
a9c163ef20
|
|
@ -1,4 +1,5 @@
|
|||
# files download by generators
|
||||
tmp/*
|
||||
alexa_top-1m.csv.zip
|
||||
amazon_ip-ranges.json
|
||||
cisco_top-1m.csv.zip
|
||||
|
|
@ -18,3 +19,5 @@ top500.pages.csv
|
|||
top-1m.csv.zip
|
||||
public-dns-nameservers.csv
|
||||
ms-azure.json
|
||||
moz-top500.pages.csv
|
||||
moz-top500.domains.csv
|
||||
|
|
|
|||
|
|
@ -0,0 +1,32 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
set -x
|
||||
|
||||
pushd tools
|
||||
python3 generate_alexa.py
|
||||
python3 generate-amazon-aws.py
|
||||
python3 generate-cisco.py
|
||||
python3 generate-cloudflare.py
|
||||
python3 generate-covid.py
|
||||
python3 generate-crl-ip-list.py
|
||||
python3 generate-disposal.py
|
||||
# TODO: Google page on Wikipedia does not exist anymore
|
||||
# Suggestion came to use a passivetotal whois search for org:Google LLC
|
||||
#python3 generate-google.py > lists/google/list.json
|
||||
python3 generate_majestic-million.py
|
||||
python3 generate-microsoft-azure.py
|
||||
python3 generate_mozilla_certificates.py
|
||||
python3 generate_moz-top500.py
|
||||
# Deprecate?
|
||||
#python3 generate-office365-cn.py > lists/microsoft-office365-cn/list.json
|
||||
python3 generate-office365.py
|
||||
python3 generate-publicdns.py
|
||||
python3 generate-tlds.py
|
||||
python3 generate_tranco.py
|
||||
python3 generate-university-domain-list.py
|
||||
python3 generate-vpn.py
|
||||
python3 generate-wikimedia.py
|
||||
popd
|
||||
|
||||
./jq_all_the_things.sh
|
||||
|
|
@ -134,6 +134,10 @@
|
|||
"13.34.23.32/27",
|
||||
"13.34.23.64/27",
|
||||
"13.34.23.96/27",
|
||||
"13.34.24.128/27",
|
||||
"13.34.24.160/27",
|
||||
"13.34.24.192/27",
|
||||
"13.34.24.96/27",
|
||||
"13.35.0.0/16",
|
||||
"13.48.0.0/15",
|
||||
"13.48.186.128/27",
|
||||
|
|
@ -164,6 +168,7 @@
|
|||
"140.179.0.0/16",
|
||||
"143.204.0.0/16",
|
||||
"144.220.0.0/16",
|
||||
"15.152.0.0/16",
|
||||
"15.161.0.0/16",
|
||||
"15.161.135.164/30",
|
||||
"15.164.0.0/15",
|
||||
|
|
@ -226,6 +231,9 @@
|
|||
"15.221.20.0/22",
|
||||
"15.221.24.0/21",
|
||||
"15.221.3.0/24",
|
||||
"15.221.33.0/24",
|
||||
"15.221.34.0/24",
|
||||
"15.221.35.0/24",
|
||||
"15.221.4.0/23",
|
||||
"15.221.8.0/21",
|
||||
"15.222.0.0/15",
|
||||
|
|
@ -382,6 +390,7 @@
|
|||
"15.230.40.0/24",
|
||||
"15.230.41.0/24",
|
||||
"15.230.42.0/24",
|
||||
"15.230.43.0/24",
|
||||
"15.230.5.0/24",
|
||||
"15.230.6.0/24",
|
||||
"15.230.64.0/26",
|
||||
|
|
@ -414,6 +423,22 @@
|
|||
"150.222.118.0/24",
|
||||
"150.222.119.0/24",
|
||||
"150.222.12.0/24",
|
||||
"150.222.120.20/31",
|
||||
"150.222.120.224/31",
|
||||
"150.222.120.226/31",
|
||||
"150.222.120.228/31",
|
||||
"150.222.120.230/31",
|
||||
"150.222.120.232/31",
|
||||
"150.222.120.234/31",
|
||||
"150.222.120.240/31",
|
||||
"150.222.120.242/31",
|
||||
"150.222.120.244/31",
|
||||
"150.222.120.246/31",
|
||||
"150.222.120.248/31",
|
||||
"150.222.120.250/31",
|
||||
"150.222.120.252/32",
|
||||
"150.222.120.255/32",
|
||||
"150.222.120.62/31",
|
||||
"150.222.121.0/24",
|
||||
"150.222.122.100/31",
|
||||
"150.222.122.102/31",
|
||||
|
|
@ -494,6 +519,7 @@
|
|||
"150.222.222.0/24",
|
||||
"150.222.223.0/24",
|
||||
"150.222.224.0/24",
|
||||
"150.222.226.0/24",
|
||||
"150.222.227.0/24",
|
||||
"150.222.228.0/24",
|
||||
"150.222.229.0/24",
|
||||
|
|
@ -545,6 +571,10 @@
|
|||
"150.222.244.35/32",
|
||||
"150.222.244.37/32",
|
||||
"150.222.245.122/31",
|
||||
"150.222.3.176/32",
|
||||
"150.222.3.177/32",
|
||||
"150.222.3.178/32",
|
||||
"150.222.3.179/32",
|
||||
"150.222.5.0/24",
|
||||
"150.222.6.0/24",
|
||||
"150.222.66.0/24",
|
||||
|
|
@ -623,7 +653,6 @@
|
|||
"176.34.64.0/18",
|
||||
"177.71.128.0/17",
|
||||
"177.71.207.128/26",
|
||||
"177.71.207.16/29",
|
||||
"177.72.240.0/21",
|
||||
"178.236.0.0/20",
|
||||
"18.130.0.0/16",
|
||||
|
|
@ -1009,11 +1038,42 @@
|
|||
"2600:1ffc:8000::/40",
|
||||
"2600:1ffc:c000::/40",
|
||||
"2600:1ffc:e000::/40",
|
||||
"2600:1ffd:807b::/48",
|
||||
"2600:1ffd:807f::/48",
|
||||
"2600:1ffd:80c8::/48",
|
||||
"2600:1ffd:80cb::/48",
|
||||
"2600:1ffd:80d0::/48",
|
||||
"2600:1ffd:80e1::/48",
|
||||
"2600:1ffd:80e8::/48",
|
||||
"2600:1ffd:80f0::/48",
|
||||
"2600:1ffd:8143::/48",
|
||||
"2600:1ffd:8149::/48",
|
||||
"2600:1ffd:814b::/48",
|
||||
"2600:1ffd:8152::/48",
|
||||
"2600:1ffd:8165::/48",
|
||||
"2600:1ffd:8166::/48",
|
||||
"2600:1ffd:816c::/48",
|
||||
"2600:1ffd:8188::/48",
|
||||
"2600:1ffd:818b::/48",
|
||||
"2600:1ffd:818f::/48",
|
||||
"2600:1ffd:8190::/48",
|
||||
"2600:1ffd:857f::/48",
|
||||
"2600:1ffd:8195::/48",
|
||||
"2600:1ffd:819f::/48",
|
||||
"2600:1ffd:81a0::/48",
|
||||
"2600:1ffd:81a3::/48",
|
||||
"2600:1ffd:81a5::/48",
|
||||
"2600:1ffd:81a7::/48",
|
||||
"2600:1ffd:81ab::/48",
|
||||
"2600:1ffd:81ad::/48",
|
||||
"2600:1ffd:81bb::/48",
|
||||
"2600:1ffd:81be::/48",
|
||||
"2600:1ffd:84a0::/48",
|
||||
"2600:1ffd:84a4::/48",
|
||||
"2600:1ffd:84ab::/48",
|
||||
"2600:1ffd:84ae::/48",
|
||||
"2600:1ffd:84af::/48",
|
||||
"2600:1ffd:84b4::/48",
|
||||
"2600:1ffd:84bd::/48",
|
||||
"2600:1ffe:1000::/40",
|
||||
"2600:1ffe:2000::/40",
|
||||
"2600:1ffe:4000::/40",
|
||||
|
|
@ -1216,7 +1276,14 @@
|
|||
"3.216.144.0/23",
|
||||
"3.216.148.0/22",
|
||||
"3.217.228.0/22",
|
||||
"3.218.180.0/22",
|
||||
"3.218.180.0/25",
|
||||
"3.218.180.128/25",
|
||||
"3.218.181.0/25",
|
||||
"3.218.181.128/25",
|
||||
"3.218.182.0/25",
|
||||
"3.218.182.128/25",
|
||||
"3.218.183.0/25",
|
||||
"3.218.183.128/25",
|
||||
"3.224.0.0/12",
|
||||
"3.227.250.128/25",
|
||||
"3.231.2.0/25",
|
||||
|
|
@ -2423,7 +2490,8 @@
|
|||
"99.83.64.0/21",
|
||||
"99.83.98.0/24",
|
||||
"99.84.0.0/16",
|
||||
"99.86.0.0/16"
|
||||
"99.86.0.0/16",
|
||||
"99.87.0.0/22"
|
||||
],
|
||||
"matching_attributes": [
|
||||
"ip-src",
|
||||
|
|
@ -2432,5 +2500,5 @@
|
|||
],
|
||||
"name": "List of known Amazon AWS IP address ranges",
|
||||
"type": "cidr",
|
||||
"version": 20200721
|
||||
"version": 20200727
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
|
|
@ -387,6 +387,6 @@
|
|||
"domain|ip"
|
||||
],
|
||||
"name": "CRL Warninglist",
|
||||
"type": "substring",
|
||||
"version": 20190301
|
||||
"type": "string",
|
||||
"version": 20200727
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
|
|
@ -55,6 +55,7 @@
|
|||
"01ed4f29eccef3110717c21d3e4498f409472c39",
|
||||
"02007a05ced36899aa8a03a2cf307f1c0449fc31",
|
||||
"0207056d172c80bdfb6dc45be9e5808846078d1e6eef1b6ed70259ab332a64c1",
|
||||
"0208ba5266fb27f3de0d2e9ac509f2c550c2159a",
|
||||
"022886ecf3bce37b2e114a7a34823c81",
|
||||
"0230feebdb5087ea5b75b046e25c8542738efe86",
|
||||
"0238e4e67c1a837b5dc674afc154a4ad",
|
||||
|
|
@ -151,6 +152,7 @@
|
|||
"04e8f12522427c5e23aa79221cf6b328",
|
||||
"04e9aabd14bf0d39c472c788a4b4c642",
|
||||
"04ecba8f92bff7458c4a5e7c69261fc7e2ef52d5af54fbdd92b17141bbe0651f",
|
||||
"04eeea8e50b4775b3c24797262917ee50002ec4c75b56cdf3ee1c18cfca5ba52",
|
||||
"04f7a1c112607e593928fec786cc6021bd6ebd95001f1c693b8c4b384eb19f6a",
|
||||
"0503e0da86c78af88dc720362432f202b51ab1a8",
|
||||
"05177d018c5cf4ea0dfb630a0c846f50e8d245df",
|
||||
|
|
@ -177,6 +179,7 @@
|
|||
"05c7bcf83f02a84ff931441c640dc308",
|
||||
"05cb0545b3d382ae1bc23e2a1eb0cd590c20a862",
|
||||
"05cf90f70d1e94a4c919f575ba9144d8",
|
||||
"05e4005db0c382f3bd66b47729e9011577601bf6f7b287e9a52ced710d258346",
|
||||
"05e69942d6af1baed73f67c731872195",
|
||||
"05e8dea8556df54e4f709376573ec990",
|
||||
"05ee9c2ac66f75d964ac5f1a3c7de75d",
|
||||
|
|
@ -252,6 +255,7 @@
|
|||
"084c318595e957ec218866984b9f2246",
|
||||
"0855414af5f5fd7e264f8b002a39cced67e5952e89b61b680cc847baa34944de",
|
||||
"085a0714524df7ae9919a6f597884439e12a533e61552d29d6a18c35096fc548",
|
||||
"085a74d4683d0fccde44677cfa579b732c9ce2da6921ad3fe70f814ee7c0bf52",
|
||||
"086d6e110dd75c37ea9f0530a93b67453733e4ef38dc53fc2c8077db75267971",
|
||||
"08745487e891c19e3078c1f2a07e452950ef36f6",
|
||||
"087feac17c75ea0ab5952f0017814ace3c7107ec",
|
||||
|
|
@ -328,7 +332,6 @@
|
|||
"0ad741d98a84c3b9c4e7552e15bff1e369b6806839455dbafcac638124d5f100",
|
||||
"0af61aa1400809d1f612b53ce61961098b787767",
|
||||
"0af912694434515de92ae4c4337b56fd",
|
||||
"0afa0f7bbfb8eeb76f451139138f8806cc56a6285aad7cc11957f4bd09ae48c1",
|
||||
"0afc8ca591cd34e06b30b23ffbcb5ec44939fc86",
|
||||
"0b1d4bac054b0dcb5866cf73f1856959b3ffdee50456001ffc6df0e35748ab92",
|
||||
"0b2280ef840adf325520b70d8785512f5148a21f",
|
||||
|
|
@ -582,6 +585,7 @@
|
|||
"13bc6504fbf77b74048ec5d46aff63b0",
|
||||
"13c6b7395bc64fc06cef2ce53f66d4b9800c52cd8816ee37532f3ab24b8165cc",
|
||||
"13c8bf84a1196bd07d16c41829199210",
|
||||
"13cbf05d4dca3e6bcce1f0403a0bc9f467d108b3",
|
||||
"13cbf88815ef41e879bedcb8b68e75e8051aa62fce6799eea34bd33d343bb32c",
|
||||
"13db41e68e0b6126302f4789b9373177",
|
||||
"13dd43e3c9357d31dbef2b6f4ff99cde",
|
||||
|
|
@ -725,6 +729,7 @@
|
|||
"18176016b41268270cb34175d67422c421a0e9b5",
|
||||
"1817d47b242f12389f140e6d10810a3f4fdd394f",
|
||||
"181dc8be7e91edf163e66cc3fad14d979a6dae53b556daee8f7dca0e114c14f6",
|
||||
"1820404404c65eff36dfd3a5210c81aa6c294bee",
|
||||
"183244b8712297be207bf17d76bd7055",
|
||||
"1835b0e482ea65536fc010e4bc13c060f65668165fba97e2f542ce96ca6dfefc",
|
||||
"18370b91824ea99e398f7086c76b0cd5",
|
||||
|
|
@ -740,6 +745,7 @@
|
|||
"187de1f9e61a2ec4f90c16bbed97e95ac8fbaa31",
|
||||
"188b39ceeecb6e612e01b1f70bfd30035f703e28",
|
||||
"188e7931b6d781af22b645f9f695239fe44da8b759cba662aecccdecd924b7e1",
|
||||
"189286494b786a5ab8dc7dbc58905aca",
|
||||
"1892884bb13fbff75ca6c982e46545021025c5b2",
|
||||
"1893205276c4448e00ecbd613e5fc75cb672acc9",
|
||||
"18958d03afb409687a1bc263860d0d735a25a004ab60e0f0e45d6333587437ae",
|
||||
|
|
@ -1156,6 +1162,7 @@
|
|||
"27efe682c7479c38133a0943666fbc9103fcbb1a919634e439711cbea7e3cfc7",
|
||||
"27fa1be4f4b56a142a47a56a7e94b72502059b4022e4623e748ca1ef151cf222",
|
||||
"280bbc34a98cc79606a89690e52ab72fdc5fff275dc366c4827b1656313e9790",
|
||||
"280c44a7fe1e31eba23e0fc6a7eac611fa5be1e8",
|
||||
"281bd27daa72be30316ba5b52ca0fd66",
|
||||
"2822861d1e3d1a3d4e28cef50c362aab",
|
||||
"282f2953d96ea3194d14bad6a1046018",
|
||||
|
|
@ -1536,6 +1543,7 @@
|
|||
"345eff15b7a49add451b65a7f4bdc6ae",
|
||||
"346071ce2ee312d1551e9bd9e88a38cb",
|
||||
"3473aecd1d6248fd45fee88a2d1758397f6c4a45aefdc8ce97c764500d214560",
|
||||
"347d18dccc2efc51a920e7a7fbb07bfbda35613681f82dca5c4c72bb0c83c035",
|
||||
"3486ed23622155459e9b25ff3f21ad7627987387",
|
||||
"348fda2b99cd5ae0b92ffa52fd6b3d72",
|
||||
"34b6d26a5f8f1c1c064cc24ebcd0f8cf",
|
||||
|
|
@ -1971,6 +1979,7 @@
|
|||
"43db8b6042e2da18210b5feca471e9da480c11ef",
|
||||
"43df7b055f78ecc001599232613a3befbd3e076a",
|
||||
"43e9470aabd9f4243661eeb94715d661",
|
||||
"440b747378b0596d429570f7f89201aed2e9327e",
|
||||
"440d16f31810555b2ec52c439d55dcbb60b9ffad",
|
||||
"440ff68a35e03995ac55e457a67eb1680f9a7cdd",
|
||||
"44192d116fe72ed41247724ae41fb46c488929ae",
|
||||
|
|
@ -2072,6 +2081,7 @@
|
|||
"477d1e884744bd9dadbd523bf7653caa",
|
||||
"477f7bfbb85e66fd80f365b8b6decdf2",
|
||||
"4786ce4351b58cac6839fb4683cd291e",
|
||||
"479195065720335e9016f1a9465e681c",
|
||||
"47961937b34ce2d12bbd2e8ab5cc96c4c493746f",
|
||||
"479c819fa2f318e9d2910842b045607b66c4bebcacdd49db4412859846145ab7",
|
||||
"47a2b188a1f084c58b8fb827038aad5b38e4f96d",
|
||||
|
|
@ -2565,7 +2575,6 @@
|
|||
"57d8fbc13443745fbf4cab208930584b544fe510",
|
||||
"57de0583efd2b26e0361da99da9df4648def7ee8441c3b728afa9bcde0f9b26a",
|
||||
"57e03bb6bc2dfa1b10618baeda568a08",
|
||||
"57eee860ebd1e466d5c85a33ab168162159017c5",
|
||||
"57fe0ab300dc327f55fcf40cf3f8eec5e1d2c8b2",
|
||||
"580bb3ab127f5e13e940ae7365e3c69a62ad204d7fdfa451592fe7c47dc02a95",
|
||||
"580dff3c16e5f8ba06aabe7d156768256cf2536c",
|
||||
|
|
@ -2816,6 +2825,7 @@
|
|||
"602c3fbd1a20d30bf764130b50d28876386f8e66",
|
||||
"6033518ea020b06ac2ee4de4bb2973208776943d",
|
||||
"6036330e1643a0cee19c8af780e0f3e8f59ca1a3",
|
||||
"60388b773d33e9141648613f36b9149f015a114378bacf3e3694864ffda426ec",
|
||||
"603ceecf1a21f3319bf275d71586a2a3",
|
||||
"603d69822381a0bfc274bbed67009bc7df133cb902fa242cf58bf727d23d5495",
|
||||
"603eaafb53f2e76a7124f85f8b43a327",
|
||||
|
|
@ -2993,6 +3003,7 @@
|
|||
"65ac9b17fb9ad079bd69fdf916a24100eed21063",
|
||||
"65adc9c691d32155e68ce8d5193bf372295d25a04075457b31ee85a0441b23f8",
|
||||
"65cf938fd9fb34200c9f6df93ded73aa5b71c369cc74a7c491f0e47a1d5e74be",
|
||||
"65d17ecae5798c79db8e840fe98a53b9",
|
||||
"65de322a1ef7affedeb7387138c26060825b08cc27e1992dd9eac8337297957b",
|
||||
"65e0db51904d0eb88e278d0af0557f9fbc568cbf8460544f0e193152ebd03755",
|
||||
"65e1a6cf628279fca25c49156aa38715",
|
||||
|
|
@ -3311,6 +3322,7 @@
|
|||
"701b432ac0cdd4d9cf95b4b884c32bf5cca90d44e0161abd13b934d68e380472",
|
||||
"701eb23f95564cd5569cd20e5f05c2888900bae9ba03abf5abe57bfe04b54a60",
|
||||
"70357b9e56d3fb3c6c009c38c7181454c462908dfbce6d54d60dfe1e506e14fd",
|
||||
"703d7a8f0ebf55aaa59f98eaf4a206004eb2516a",
|
||||
"703ea0a173e2042355f1dfeb1079292d41ea68c046c9d45cc60dc5410104f478",
|
||||
"704b659f2e7a06ead38dbf397bc4432b899fb88413e085ea572ba9c96ce60ae0",
|
||||
"70541c2823e1a30b452d5f862833dc13",
|
||||
|
|
@ -3802,7 +3814,6 @@
|
|||
"7f6889ffe8b02045e2cac99a2f2ee4f4c2eed24934b6521872d34bf812671c4b",
|
||||
"7f755dc2b9ee99eb02d71cb1805059aaabb5eb3e8558dca8c09a1d0de258d767",
|
||||
"7f786418f0afcebeb2be04c2e5b2ee4c835929ae",
|
||||
"7f79889c8f8312368b52173ed8fcd462",
|
||||
"7f814cd1454900ac1331364b6b6de06b87cc8cb9936ec583100822721057d3a3",
|
||||
"7f8c9f7829dbe8b768cb4a60da4f9ab558d064eb",
|
||||
"7f8cc76fbfd2b031aaabf0e6b0c8fd8c",
|
||||
|
|
@ -3821,6 +3832,7 @@
|
|||
"7fd90bbdfe628d97e254ca23871671f2",
|
||||
"7fdaceb62d40890bcf8c777b93d63e2d",
|
||||
"7fdce3bf4103c2684b3adbb5792884bd45c75094c217788863950346f79c90a3",
|
||||
"7feb9374eab08d392717c647436dae06176a24c010607fda1cce5e5f0106b472",
|
||||
"7ff0739409e68802c0149af180645ff7",
|
||||
"80031c140e89f9be6792be2b49b74603",
|
||||
"80042563db5f162e455632b589c60150f37ef7fedb774c6dc2ed5ba6fb865120",
|
||||
|
|
@ -3833,6 +3845,7 @@
|
|||
"802e0b133d61441a0a6af799dc5e4b260972990d6dcb87dc0acfc585d9082610",
|
||||
"8048eefc45bd5e2ef7254b79e7ed9968f5213b3a",
|
||||
"805577e05b81812579ae5102a79018b4",
|
||||
"806f1c72f6d67c9c114eff43d3d84100",
|
||||
"8072f44e162fd430d83c4fff1c0b7f248258e4abb2aea8c4719aa5f3fd23c626",
|
||||
"8081b85329c69843c7d9444e37bd367a",
|
||||
"808205ede5d170cc4aac49c119c29eb8",
|
||||
|
|
@ -3960,7 +3973,9 @@
|
|||
"84a850389f51a50ff60e54c7c4189d81",
|
||||
"84aa25e9a6ef9acd2ea64743d0674af1",
|
||||
"84d3226619a7c2584d49b2cc7465fa85902efee3a5e8f36b36ec70ce3ea4a09f",
|
||||
"84dddd0196ac27bcc2f17d564769ea7e",
|
||||
"84fae8c3aeebca606050bc11487fece7464c1210a4e6eaf261cecb013681bcab",
|
||||
"8503c71b1e484c60cc2913ea2d79c482626a1847",
|
||||
"8504a0350036b18594b1f47b6a3bd3513e28b045bc7dbfe5b08d8f3049cd3cfe",
|
||||
"850575bbf8a8a8d43b44886210f84a0746ca9493",
|
||||
"850772c2c9201656ca7adf33772aeb7e",
|
||||
|
|
@ -4436,7 +4451,6 @@
|
|||
"9470dc33d482850ed9e458ad48b62a4fde772c91",
|
||||
"9475d29be526627856fee40e191e13c8febda40f",
|
||||
"947de5a14bc7bef32f5f4d326d15aa5d",
|
||||
"94833a3766439d0b83bb4bf06233babd11f217a42ba7e8c7392c212171407350",
|
||||
"948b7111af42f546d579cff5ce2bdec82134dd9914842bddb0c52872eb604e39",
|
||||
"948d1daf1d124ace83f6826192036edc35c4d005",
|
||||
"948e1652586240d453287ab69caeb8f2f4f02117",
|
||||
|
|
@ -4796,7 +4810,6 @@
|
|||
"9f9c14d22a083785ceb7d02a19e730f8",
|
||||
"9f9f66c96b2af2fd1b478480373d3c1ae1ee10a3",
|
||||
"9fa9fcb500f1a98355099f70034cf0d2ee743aba63f6eb67d55685ea9065328b",
|
||||
"9fba32a6be7f0f7e9b321a01ec45a5e8",
|
||||
"9fc43b631d46e2ffa6c11928a33a8dccbccf86c5d8dd461518541c52fe259266",
|
||||
"9fcdf094368d1b025c4c5574f8c59db8df75d0c3",
|
||||
"9fd9b25764b14052b1db155536e38867",
|
||||
|
|
@ -4819,7 +4832,6 @@
|
|||
"a01264a53c3d71887b7a6ea3d9a05eda",
|
||||
"a0252e6ffe0cc002518a6e3fb4105eda28ef1213",
|
||||
"a026b5019219582e2748a65e7f70e159",
|
||||
"a028356eaabdbbddfc78523dfc1287a0",
|
||||
"a028df21db93af1ebd970e0e681cf902c20b2185",
|
||||
"a02ddcec3086787e9de7152bf85ec9e7",
|
||||
"a02ecf23fd9e2ff3880667b150b3dd030b126751",
|
||||
|
|
@ -5319,6 +5331,7 @@
|
|||
"b0a6ef0350e7c4c6056beea7af9d2d860b9ed102137b9729d3c23216d195546a",
|
||||
"b0b05d7131d7881f78ba4172b442b7d774d04ff27d383be3e459a372473b1e15",
|
||||
"b0c29d0244f981e999a15f59708655f0",
|
||||
"b0c2d2d13cdd56cdaa6ab6e2c04440be4a429c75",
|
||||
"b0c7c6e2b6265989c1a104256ed756fe",
|
||||
"b0cd6ae7b9e20ec5f830fee01f666d5d90e6e229d06bc46a30accedfec889648",
|
||||
"b0e6383fbdbfce5e5f9d7934fad0fc3c5e030f13",
|
||||
|
|
@ -5350,6 +5363,7 @@
|
|||
"b19885909d55e47f196f36a55ccc4ea9afa052c3f8ee3e3a9ee041e562163584",
|
||||
"b19b9bc90c06f08061824cc723c82ecab90fe543",
|
||||
"b1a8fd40b437ded972f574370622e2bbb2f9b9593bc0504d7e2030f2b24e67fa",
|
||||
"b1aae1bcd555e8a3d1e3dda8ec84e757c552655344ce3feebbaf98e895dbedac",
|
||||
"b1ac8cfb181b9c9354e1775fcbdfcfe7898c5cc9a17d76315b57c112eee55234",
|
||||
"b1ae92ee3e68ac9c00bee275604782301ebcd9b63cd3e0d5e8144fdf7415b9dd",
|
||||
"b1b3162f7cc867f6d47536df3232e706e031357c",
|
||||
|
|
@ -5627,6 +5641,7 @@
|
|||
"bac4f03dc635bdae067f7bfe77ab06d88b5ba4668b0f46157e53d6ca9310077f",
|
||||
"bacafa90e4a81408ff7b1245b7e973213c83488c",
|
||||
"bad74f4d7ab128a49460692416540b5aeaa6b46a",
|
||||
"bad7fe221e2bf54e832bde8871f0930c",
|
||||
"bad88a984e396a294168e085077ffac2",
|
||||
"bae30b15dbb1544cf194d076b75b7bb9e3d6b760",
|
||||
"bae5efbd8fff3de56dc2388803bde18572b32e6e",
|
||||
|
|
@ -6004,6 +6019,7 @@
|
|||
"c717d11a9035a26df21a6be9bada93ef",
|
||||
"c71b01ff159489973f92e2f841d63987ebf24548",
|
||||
"c72a5a03842ff4a31ad26e31bbdbaee5",
|
||||
"c72c7ac0fd729d4997a1a74fb7df6535",
|
||||
"c7367403cf6d5c13d9690187926ef896ce6e74135080c9afd8c8b1532467ae50",
|
||||
"c7422de21cebc92ecae6be9dcd7e711ee650d30aed711fbb0df6b2c784b6c4fb",
|
||||
"c743fe562a1eb4b3494d752e1443a97e",
|
||||
|
|
@ -6357,7 +6373,6 @@
|
|||
"d29900b39c9f22ff2da8497ca624719d64a86ba9",
|
||||
"d2a50723561f9bba7cef93b76315268a",
|
||||
"d2adf8385fe30160fc5169ec81f8cc33ab88ca23",
|
||||
"d2bdb0760d4b69357c589ddf4108385923d820f6",
|
||||
"d2c410cb8c94e566bfd4b4ff23c8868632743690",
|
||||
"d2c686e02a4a50772ea9cdec7ffad03e",
|
||||
"d2c7cb6e13483c6004009b2d97455b7ff8f4afa6",
|
||||
|
|
@ -6369,6 +6384,7 @@
|
|||
"d31520a86966839c76f79b23bbbb3c545b582b99",
|
||||
"d3229995b6a61afbfd3cb245be9b129b",
|
||||
"d328f91b2d74c372820a01a0ce54d3bb",
|
||||
"d33a847f640377be0ae1a429153a07e87c8827fa4840b5158bcdcb85e10a453a",
|
||||
"d33d27b38c72bfa7914f068b10fd18d8e12a45f5e60b8c35d6728c2d17fe3c48",
|
||||
"d34138ed1458ae7de4edea36ed3992e4f46e6ec9cf1e633e538de9ff0f38f8e5",
|
||||
"d3533b732a518a6da68ef266085e11dfd114c0eb0092cd43530a44d54b913ed1",
|
||||
|
|
@ -6424,6 +6440,7 @@
|
|||
"d517bfad08702c10f086ddc192e702748bb716d6",
|
||||
"d51f1ed305821bb2541af3afc96e2ff2",
|
||||
"d522f5744e0303cdd3cb101eb00fbfefa0c66985",
|
||||
"d52db7b240677eaa7154d588f74cbc1bfccda14e",
|
||||
"d5309f1c2014ff3a18be707c7c1123bb",
|
||||
"d53463db5346a1811dbd6478888a11a7",
|
||||
"d554ede7df29a86ea2b30021482ddfbd24cdec55",
|
||||
|
|
@ -6516,7 +6533,6 @@
|
|||
"d8e47c0b60e3dfe37b42242c719f63c2",
|
||||
"d8e79ebed997b764e6d0df624f2dbb4eff2a1b6049c6f1a1a1e88263c43db221",
|
||||
"d8f6ce367405c0d4bf179552fec2925d",
|
||||
"d8fae00729250ebbd727757a12232b1d4887ce9f195b344c18cfee068cfbe6ec",
|
||||
"d933da413ce524080624a34ce85aa48e",
|
||||
"d9352669e29349329af4745c55a6c159dc979ab745925170a1980668382acbc0",
|
||||
"d93c3c309071a5fbbca28c45311f763aef2dbe62",
|
||||
|
|
@ -6581,6 +6597,7 @@
|
|||
"db3c8e2a7f1cba1419edaf68dc5a2428",
|
||||
"db3dbba517e5bca21fa69b72e8867f74afb1b7aa",
|
||||
"db4591f878f6672f5b70733a66ad7c9537b97e6f0af5ca49aab8ecb2ce02f86b",
|
||||
"db46e58630ad01fd265b0862d7026812",
|
||||
"db476339ccbfcc9e4bd1d6cb606ca27f00679e1ef8a581e7236309b9d63ffe37",
|
||||
"db5555779be0e3f862ba9e92a28683743c3e9e0d",
|
||||
"db6cbb8733beb493af39c702a9dd0674314e2ed3260cd5250a491480757d8ea2",
|
||||
|
|
@ -6935,6 +6952,7 @@
|
|||
"e803a4975e3df57967ece94de85dc141",
|
||||
"e810ac30d9a776056fc1c994b76ded94",
|
||||
"e81443141e3c1232d06465e11bbd1f6ca7181a3e037f815bb8faaeade6bebed1",
|
||||
"e8170f7b6f7a6078dba3daa2b10a13ab",
|
||||
"e81b01f9f5692cf3823c6fd35886542bfaeefc5ea94f4e246e42c4a9fc5fe8ab",
|
||||
"e8202a24114b3ecc751be23f6cef9018721f3604",
|
||||
"e8248ff3f0f591086bcf6dec9c37c535a0704643",
|
||||
|
|
@ -7059,6 +7077,7 @@
|
|||
"ebaf8abd199528193747a135cac339c1",
|
||||
"ebb06ca92fbe3cf1245f81f438bb830d",
|
||||
"ebbf4dc600c17da04381defdcfc119c3f34efb4a04d0860910b813c7792d7585",
|
||||
"ebcd0c73fe44b60e25789419c07b0b234b3f586e092316fb2cbda39c223224a3",
|
||||
"ebe0015ef641c480954ed2a68842fccd2a8a79213aeed836ddfa6282e8ea8f6a",
|
||||
"ebe54f29a97b7d327d9b253b8a7eaafc",
|
||||
"ebe6f53aa744955e95ad712d4b9b464c9b273f24",
|
||||
|
|
@ -7187,7 +7206,6 @@
|
|||
"efeb3869a192aabb11d10d1313bda0f6",
|
||||
"f0029e794d5180634b34aee29f291358232faa04",
|
||||
"f00322c97b57cea6d64a6a1d365c6c6b7f0eba3089876c1430e1063e8a82a676",
|
||||
"f00366a7a57a9aee3276ccc7b0d3c05f9c814706",
|
||||
"f00cf0fae0b51ba38e8065bb40d90e977586334f",
|
||||
"f01abafc75ddd87a52fdd230c4514ffc",
|
||||
"f01c1aca392882af152e9f01ecccd0afddd8aa35bf895b003198b1e8c752ddb8",
|
||||
|
|
@ -7686,5 +7704,5 @@
|
|||
],
|
||||
"name": "Fingerprint of known intermedicate of trusted certificates",
|
||||
"type": "string",
|
||||
"version": 20200721
|
||||
"version": 20200723
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
|
|
@ -1,6 +1,8 @@
|
|||
{
|
||||
"description": "Event contains one or more public IPv6 DNS resolvers as attribute with an IDS flag set",
|
||||
"list": [
|
||||
"2001:148f:fffe::1",
|
||||
"2001:148f:ffff::1",
|
||||
"2001:41d0:203:4b1f:2:2:2:2",
|
||||
"2001:428:101:100:205:171:2:65",
|
||||
"2001:428::1",
|
||||
|
|
@ -11,18 +13,15 @@
|
|||
"2001:4b8:3:201::902",
|
||||
"2001:67c:28a4::",
|
||||
"2001:67c:2b0::1",
|
||||
"2001:b000:168::2",
|
||||
"2001:de4::102",
|
||||
"2606:ed00:2:babe::10",
|
||||
"2607:5300:203:1797::53",
|
||||
"2607:f130:0:d7::d41",
|
||||
"2610:a1:1018::34",
|
||||
"2a00:aa40:0:225::2",
|
||||
"2a01:238:42f6:ac00:2a29:4f7f:b6d:ef46",
|
||||
"2a01:3a0:53:53::",
|
||||
"2a01:4f8:141:316d::117",
|
||||
"2a01:4f8:151:34aa::198",
|
||||
"2a01:4f8:c0c:1e44::1",
|
||||
"2a01:4f8:c17:739a::2",
|
||||
"2a02:2970:1002::18",
|
||||
"2a02:c205:3001:4558::1",
|
||||
|
|
@ -35,5 +34,5 @@
|
|||
],
|
||||
"name": "List of known IPv6 public DNS resolvers",
|
||||
"type": "string",
|
||||
"version": 20200720
|
||||
"version": 20200723
|
||||
}
|
||||
|
|
|
|||
35570
lists/tranco/list.json
35570
lists/tranco/list.json
File diff suppressed because it is too large
Load Diff
|
|
@ -31,7 +31,6 @@
|
|||
"127.net",
|
||||
"1337x.to",
|
||||
"1377x.to",
|
||||
"13dl.net",
|
||||
"163.com",
|
||||
"1688.com",
|
||||
"16mb.com",
|
||||
|
|
@ -49,7 +48,6 @@
|
|||
"1fichier.com",
|
||||
"1gb.ru",
|
||||
"1mg.com",
|
||||
"1mono.biz",
|
||||
"1password.com",
|
||||
"1plus1tv.ru",
|
||||
"1rx.io",
|
||||
|
|
@ -66,7 +64,9 @@
|
|||
"247wallst.com",
|
||||
"24h.com.vn",
|
||||
"24s.com",
|
||||
"24video.vip",
|
||||
"258.com",
|
||||
"25pp.com",
|
||||
"263.net",
|
||||
"2checkout.com",
|
||||
"2conv.com",
|
||||
|
|
@ -82,7 +82,6 @@
|
|||
"350.org",
|
||||
"360.cn",
|
||||
"360.com",
|
||||
"360buyimg.com",
|
||||
"360doc.com",
|
||||
"360kan.com",
|
||||
"360safe.com",
|
||||
|
|
@ -127,6 +126,7 @@
|
|||
"52pojie.cn",
|
||||
"53.com",
|
||||
"53kf.com",
|
||||
"55online.news",
|
||||
"56.com",
|
||||
"58.com",
|
||||
"58pic.com",
|
||||
|
|
@ -178,11 +178,11 @@
|
|||
"9r.cn",
|
||||
"9to5google.com",
|
||||
"9to5mac.com",
|
||||
"a-course-in-miracles.net",
|
||||
"a-msedge.net",
|
||||
"a.co",
|
||||
"a16z.com",
|
||||
"a1sewcraft.com",
|
||||
"a2hosting.com",
|
||||
"a2z.com",
|
||||
"a3cloud.net",
|
||||
"a3yqjsrczwwp.com",
|
||||
|
|
@ -208,7 +208,6 @@
|
|||
"aarp.org",
|
||||
"aarth.net",
|
||||
"aastocks.com",
|
||||
"aau.dk",
|
||||
"aawsat.com",
|
||||
"aaxads.com",
|
||||
"abadis.ir",
|
||||
|
|
@ -249,6 +248,7 @@
|
|||
"academia.edu",
|
||||
"academic.ru",
|
||||
"acast.com",
|
||||
"accaglobal.com",
|
||||
"accenture.com",
|
||||
"accesspressthemes.com",
|
||||
"accesstrade.net",
|
||||
|
|
@ -296,7 +296,6 @@
|
|||
"adafruit.com",
|
||||
"adage.com",
|
||||
"adap.tv",
|
||||
"adaptedmind.com",
|
||||
"adaware.com",
|
||||
"adb.org",
|
||||
"adblockplus.org",
|
||||
|
|
@ -424,6 +423,7 @@
|
|||
"ahrq.gov",
|
||||
"aicheren.com",
|
||||
"aicpa.org",
|
||||
"aiellocondomini.it",
|
||||
"aif.ru",
|
||||
"aihelp.net",
|
||||
"aip.org",
|
||||
|
|
@ -548,6 +548,7 @@
|
|||
"alternet.org",
|
||||
"altervista.org",
|
||||
"altibbi.com",
|
||||
"altreeq.com",
|
||||
"alturl.com",
|
||||
"alwafd.news",
|
||||
"alwakeelnews.com",
|
||||
|
|
@ -673,6 +674,7 @@
|
|||
"ap.gov.in",
|
||||
"ap.org",
|
||||
"ap7am.com",
|
||||
"apa.az",
|
||||
"apa.org",
|
||||
"apache-ssl.org",
|
||||
"apache.org",
|
||||
|
|
@ -729,7 +731,6 @@
|
|||
"arabi21.com",
|
||||
"arabianbusiness.com",
|
||||
"arabnews.com",
|
||||
"aracne.biz",
|
||||
"aralego.com",
|
||||
"aralego.net",
|
||||
"arbeitsagentur.de",
|
||||
|
|
@ -789,6 +790,7 @@
|
|||
"asda.com",
|
||||
"asha.org",
|
||||
"ashemaletube.com",
|
||||
"asiae.co.kr",
|
||||
"asianetnews.com",
|
||||
"asiaone.com",
|
||||
"asics.com",
|
||||
|
|
@ -880,6 +882,7 @@
|
|||
"avcdn.net",
|
||||
"avclub.com",
|
||||
"avct.cloud",
|
||||
"avery.com",
|
||||
"avg.com",
|
||||
"avgle.com",
|
||||
"aviasales.ru",
|
||||
|
|
@ -907,7 +910,6 @@
|
|||
"ayobandung.com",
|
||||
"az.gov",
|
||||
"azcentral.com",
|
||||
"azfamily.com",
|
||||
"azlyrics.com",
|
||||
"azure.com",
|
||||
"azure.net",
|
||||
|
|
@ -972,6 +974,7 @@
|
|||
"bankofamerica.com",
|
||||
"bankofengland.co.uk",
|
||||
"bankrate.com",
|
||||
"bannersnack.com",
|
||||
"banorte.com",
|
||||
"banquepopulaire.fr",
|
||||
"banvenez.com",
|
||||
|
|
@ -1061,6 +1064,7 @@
|
|||
"berlin.de",
|
||||
"bershka.com",
|
||||
"besoccer.com",
|
||||
"best2019-games-web4.com",
|
||||
"bestbrides.org",
|
||||
"bestbuy.ca",
|
||||
"bestbuy.com",
|
||||
|
|
@ -1124,7 +1128,6 @@
|
|||
"bing.net",
|
||||
"binged.it",
|
||||
"binghamton.edu",
|
||||
"binomo.com",
|
||||
"binomoofficial.com",
|
||||
"binomoworld.com",
|
||||
"biobiochile.cl",
|
||||
|
|
@ -1132,6 +1135,7 @@
|
|||
"biologists.org",
|
||||
"biomedcentral.com",
|
||||
"biorxiv.org",
|
||||
"biqle.ru",
|
||||
"birmingham.ac.uk",
|
||||
"birminghammail.co.uk",
|
||||
"bis.org",
|
||||
|
|
@ -1153,6 +1157,7 @@
|
|||
"bitmoji.com",
|
||||
"bitnami.com",
|
||||
"bitrix24.com",
|
||||
"bitrix24.net",
|
||||
"bitrix24.ru",
|
||||
"bittorrent.com",
|
||||
"bittrex.com",
|
||||
|
|
@ -1167,6 +1172,7 @@
|
|||
"bjnews.com.cn",
|
||||
"bjs.gov",
|
||||
"bjx.com.cn",
|
||||
"bkm.com.tr",
|
||||
"bkrtx.com",
|
||||
"bl.uk",
|
||||
"blackberry.com",
|
||||
|
|
@ -1439,6 +1445,7 @@
|
|||
"buzzsprout.com",
|
||||
"buzzsumo.com",
|
||||
"byjus.com",
|
||||
"byrdie.com",
|
||||
"byrenjia.com",
|
||||
"bytedance.com",
|
||||
"bytefence.com",
|
||||
|
|
@ -1559,6 +1566,7 @@
|
|||
"carsensor.net",
|
||||
"carsforsale.com",
|
||||
"carto.com",
|
||||
"carview.co.jp",
|
||||
"carwale.com",
|
||||
"cas.cn",
|
||||
"casadellibro.com",
|
||||
|
|
@ -1635,7 +1643,9 @@
|
|||
"center4family.com",
|
||||
"centos-webpanel.com",
|
||||
"centos.org",
|
||||
"centrelink.gov.au",
|
||||
"centrum.cz",
|
||||
"centrum24.pl",
|
||||
"centurylink.com",
|
||||
"cern.ch",
|
||||
"cert.org",
|
||||
|
|
@ -1777,6 +1787,7 @@
|
|||
"cint.com",
|
||||
"cio.com",
|
||||
"cirquedusoleil.com",
|
||||
"cisa.gov",
|
||||
"cisco.com",
|
||||
"ciscospark.com",
|
||||
"cision.com",
|
||||
|
|
@ -1807,8 +1818,10 @@
|
|||
"clarin.com",
|
||||
"clarium.io",
|
||||
"clarivate.com",
|
||||
"claro.com.br",
|
||||
"classdojo.com",
|
||||
"classiccars.com",
|
||||
"clave.gob.es",
|
||||
"clck.ru",
|
||||
"clean.gg",
|
||||
"cleanpng.com",
|
||||
|
|
@ -2024,7 +2037,6 @@
|
|||
"cookpad.com",
|
||||
"coolmathgames.com",
|
||||
"coolors.co",
|
||||
"cooltext.com",
|
||||
"cootek.com",
|
||||
"cootekservice.com",
|
||||
"cootlogix.com",
|
||||
|
|
@ -2388,7 +2400,6 @@
|
|||
"digital-photography-school.com",
|
||||
"digital.com",
|
||||
"digitalcommerce360.com",
|
||||
"digitaleast.mobi",
|
||||
"digitalguardian.com",
|
||||
"digitaljournal.com",
|
||||
"digitalocean.com",
|
||||
|
|
@ -2470,7 +2481,6 @@
|
|||
"docin.com",
|
||||
"docker.com",
|
||||
"docomo.ne.jp",
|
||||
"docsity.com",
|
||||
"docstoc.com",
|
||||
"doctissimo.fr",
|
||||
"doctolib.fr",
|
||||
|
|
@ -2481,7 +2491,6 @@
|
|||
"docusign.net",
|
||||
"doda.jp",
|
||||
"doe.gov",
|
||||
"dogdrip.net",
|
||||
"dogpile.com",
|
||||
"doi.gov",
|
||||
"doi.org",
|
||||
|
|
@ -2512,7 +2521,6 @@
|
|||
"doodlekit.com",
|
||||
"doorblog.jp",
|
||||
"doordash.com",
|
||||
"doostihaa.com",
|
||||
"doramatv.live",
|
||||
"doramy.club",
|
||||
"dostor.org",
|
||||
|
|
@ -2606,7 +2614,6 @@
|
|||
"dwz.cn",
|
||||
"dx.com",
|
||||
"dxy.cn",
|
||||
"dy2018.com",
|
||||
"dyn.com",
|
||||
"dynamicdrive.com",
|
||||
"dynamics.com",
|
||||
|
|
@ -2733,6 +2740,7 @@
|
|||
"egerton.ac.ke",
|
||||
"eghtesadnews.com",
|
||||
"eghtesadonline.com",
|
||||
"egnyte.com",
|
||||
"egov.kz",
|
||||
"ehow.com",
|
||||
"eia.gov",
|
||||
|
|
@ -2819,6 +2827,7 @@
|
|||
"enable-javascript.com",
|
||||
"enamad.ir",
|
||||
"ename.com",
|
||||
"ename.net",
|
||||
"enchantedlearning.com",
|
||||
"encuentra24.com",
|
||||
"encyclopedia.com",
|
||||
|
|
@ -2938,7 +2947,6 @@
|
|||
"evernote.com",
|
||||
"everydayhealth.com",
|
||||
"evidon.com",
|
||||
"evite.com",
|
||||
"ew.com",
|
||||
"ewaybillgst.gov.in",
|
||||
"eweek.com",
|
||||
|
|
@ -2994,6 +3002,7 @@
|
|||
"f6s.com",
|
||||
"f95zone.to",
|
||||
"faa.gov",
|
||||
"faberlic.com",
|
||||
"fabric.com",
|
||||
"facebook.com",
|
||||
"facebook.net",
|
||||
|
|
@ -3075,7 +3084,6 @@
|
|||
"feedbooks.com",
|
||||
"feedburner.com",
|
||||
"feedingamerica.org",
|
||||
"feediran.com",
|
||||
"feedly.com",
|
||||
"feedspot.com",
|
||||
"fehobmasr.com",
|
||||
|
|
@ -3265,6 +3273,7 @@
|
|||
"freedesktop.org",
|
||||
"freedomhouse.org",
|
||||
"freedownloadmanager.org",
|
||||
"freee.co.jp",
|
||||
"freefiremobile.com",
|
||||
"freehostia.com",
|
||||
"freejobalert.com",
|
||||
|
|
@ -3372,7 +3381,6 @@
|
|||
"gandi.net",
|
||||
"gangde.net",
|
||||
"gansu.gov.cn",
|
||||
"gansudaily.com.cn",
|
||||
"gao.gov",
|
||||
"gao7.com",
|
||||
"gap.com",
|
||||
|
|
@ -3452,6 +3460,7 @@
|
|||
"getcomposer.org",
|
||||
"getcourse.ru",
|
||||
"getdonspeg.work",
|
||||
"getdrip.com",
|
||||
"getdropbox.com",
|
||||
"getfirefox.com",
|
||||
"getfvid.com",
|
||||
|
|
@ -3484,6 +3493,7 @@
|
|||
"gigaom.com",
|
||||
"gigazine.net",
|
||||
"gigya.com",
|
||||
"gilt.com",
|
||||
"gimp.org",
|
||||
"gingerall.com",
|
||||
"gingersoftware.com",
|
||||
|
|
@ -3507,7 +3517,6 @@
|
|||
"glamour.com",
|
||||
"glassdoor.co.in",
|
||||
"glassdoor.com",
|
||||
"glaz.tv",
|
||||
"gleam.io",
|
||||
"globalcitizen.org",
|
||||
"globalnews.ca",
|
||||
|
|
@ -3611,6 +3620,7 @@
|
|||
"google.com.hk",
|
||||
"google.com.kh",
|
||||
"google.com.kw",
|
||||
"google.com.lb",
|
||||
"google.com.ly",
|
||||
"google.com.mm",
|
||||
"google.com.mx",
|
||||
|
|
@ -3709,6 +3719,7 @@
|
|||
"grainger.com",
|
||||
"graizoah.com",
|
||||
"gramfree.net",
|
||||
"gramfree.one",
|
||||
"gramho.com",
|
||||
"grammarly.com",
|
||||
"grammarly.io",
|
||||
|
|
@ -3828,6 +3839,7 @@
|
|||
"hapitas.jp",
|
||||
"haplat.net",
|
||||
"haqqin.az",
|
||||
"haraj.com.sa",
|
||||
"harborfreight.com",
|
||||
"hardened-php.net",
|
||||
"hardrock.com",
|
||||
|
|
@ -3855,9 +3867,9 @@
|
|||
"hbomax.com",
|
||||
"hbr.org",
|
||||
"hbs.edu",
|
||||
"hbx.com",
|
||||
"hc360.com",
|
||||
"hclips.com",
|
||||
"hd315.gov.cn",
|
||||
"hdfcbank.com",
|
||||
"hdrezka-ag.com",
|
||||
"he.net",
|
||||
|
|
@ -3998,7 +4010,6 @@
|
|||
"hostinger.com",
|
||||
"hotair.com",
|
||||
"hotbot.com",
|
||||
"hotbrides.org",
|
||||
"hotcleaner.com",
|
||||
"hotels.com",
|
||||
"hotelscombined.com",
|
||||
|
|
@ -4057,6 +4068,7 @@
|
|||
"huaban.com",
|
||||
"huamu.cn",
|
||||
"huangye88.com",
|
||||
"huanjugu.com",
|
||||
"huanqiu.com",
|
||||
"huatu.com",
|
||||
"huawei.com",
|
||||
|
|
@ -4272,7 +4284,6 @@
|
|||
"inc.com",
|
||||
"incapdns.net",
|
||||
"incometaxindiaefiling.gov.in",
|
||||
"indaytrending.com",
|
||||
"indeed.co.uk",
|
||||
"indeed.com",
|
||||
"indeedassessments.com",
|
||||
|
|
@ -4381,7 +4392,6 @@
|
|||
"internic.net",
|
||||
"interpark.com",
|
||||
"interpol.int",
|
||||
"interracialdatingapp.biz",
|
||||
"intesasanpaolo.com",
|
||||
"intipseleb.com",
|
||||
"intoday.in",
|
||||
|
|
@ -4412,7 +4422,6 @@
|
|||
"ipify.org",
|
||||
"ipinfo.io",
|
||||
"ipko.pl",
|
||||
"ipl.org",
|
||||
"iplanet.com",
|
||||
"iponweb.net",
|
||||
"ipredictive.com",
|
||||
|
|
@ -4458,7 +4467,6 @@
|
|||
"italki.com",
|
||||
"itar-tass.com",
|
||||
"itau.com.br",
|
||||
"itavcn.com",
|
||||
"itch.io",
|
||||
"iteye.com",
|
||||
"ithome.com",
|
||||
|
|
@ -4506,6 +4514,7 @@
|
|||
"jalopnik.com",
|
||||
"jamanetwork.com",
|
||||
"jamendo.com",
|
||||
"jamesclear.com",
|
||||
"jamieoliver.com",
|
||||
"jamnews.com",
|
||||
"jandan.net",
|
||||
|
|
@ -4590,7 +4599,6 @@
|
|||
"joann.com",
|
||||
"jobcan.jp",
|
||||
"jobinja.ir",
|
||||
"jobkorea.co.kr",
|
||||
"jobrapido.com",
|
||||
"jobsdb.com",
|
||||
"jobui.com",
|
||||
|
|
@ -4637,6 +4645,7 @@
|
|||
"jst.go.jp",
|
||||
"jstor.org",
|
||||
"jstv.com",
|
||||
"juejin.im",
|
||||
"jugantor.com",
|
||||
"jugem.jp",
|
||||
"juicyads.com",
|
||||
|
|
@ -4686,7 +4695,6 @@
|
|||
"kalerkantho.com",
|
||||
"kaltura.com",
|
||||
"kampyle.com",
|
||||
"kanal247.com",
|
||||
"kansas.com",
|
||||
"kansascity.com",
|
||||
"kantei.go.jp",
|
||||
|
|
@ -4759,13 +4767,12 @@
|
|||
"kingcounty.gov",
|
||||
"kingsoft.com",
|
||||
"kingston.com",
|
||||
"kingtime.jp",
|
||||
"kinja.com",
|
||||
"kino-teatr.ru",
|
||||
"kinogo.by",
|
||||
"kinogo.zone",
|
||||
"kinokrad.co",
|
||||
"kinopoisk.ru",
|
||||
"kinozal.tv",
|
||||
"kinsta.com",
|
||||
"kiplinger.com",
|
||||
"kiro7.com",
|
||||
|
|
@ -4864,6 +4871,7 @@
|
|||
"l-msedge.net",
|
||||
"la-croix.com",
|
||||
"labanquepostale.fr",
|
||||
"labcorp.com",
|
||||
"labiennale.org",
|
||||
"labirint.ru",
|
||||
"labnol.org",
|
||||
|
|
@ -5017,6 +5025,7 @@
|
|||
"lifehack.org",
|
||||
"lifehacker.com",
|
||||
"lifehacker.ru",
|
||||
"lifeinaday.youtube",
|
||||
"lifesitenews.com",
|
||||
"lifewire.com",
|
||||
"lifo.gr",
|
||||
|
|
@ -5194,6 +5203,7 @@
|
|||
"macworld.com",
|
||||
"macys.com",
|
||||
"made-in-china.com",
|
||||
"madeiramadeira.com.br",
|
||||
"madison.com",
|
||||
"madmimi.com",
|
||||
"madrid.org",
|
||||
|
|
@ -5222,6 +5232,7 @@
|
|||
"main.jp",
|
||||
"maine.gov",
|
||||
"mainichi.jp",
|
||||
"maisonmargiela.com",
|
||||
"majorgeeks.com",
|
||||
"makeleio.gr",
|
||||
"makemytrip.com",
|
||||
|
|
@ -5268,6 +5279,7 @@
|
|||
"marines.mil",
|
||||
"marinetraffic.com",
|
||||
"marist.edu",
|
||||
"marketbeat.com",
|
||||
"markethive.com",
|
||||
"marketingland.com",
|
||||
"marketingprofs.com",
|
||||
|
|
@ -5385,7 +5397,6 @@
|
|||
"mega.nz",
|
||||
"megafon.ru",
|
||||
"megaphone.fm",
|
||||
"megaup.net",
|
||||
"megaupload.com",
|
||||
"megogo.net",
|
||||
"mehrnews.com",
|
||||
|
|
@ -5402,6 +5413,7 @@
|
|||
"memurlar.net",
|
||||
"menards.com",
|
||||
"mendeley.com",
|
||||
"meneame.net",
|
||||
"menshealth.com",
|
||||
"mensjournal.com",
|
||||
"mentalfloss.com",
|
||||
|
|
@ -5436,7 +5448,6 @@
|
|||
"metafilter.com",
|
||||
"metal-archives.com",
|
||||
"metapress.com",
|
||||
"meteoblue.com",
|
||||
"meteofrance.com",
|
||||
"meterserver.vn",
|
||||
"meti.go.jp",
|
||||
|
|
@ -5483,6 +5494,7 @@
|
|||
"microchip.com",
|
||||
"microfocus.com",
|
||||
"microsoft.com",
|
||||
"microsoft.net",
|
||||
"microsoft.us",
|
||||
"microsoftazuread-sso.com",
|
||||
"microsoftonline-p.com",
|
||||
|
|
@ -5517,7 +5529,6 @@
|
|||
"mindspring.com",
|
||||
"mindtools.com",
|
||||
"mindvalley.com",
|
||||
"mine.nu",
|
||||
"minecraft.net",
|
||||
"mingpao.com",
|
||||
"miniclip.com",
|
||||
|
|
@ -5569,7 +5580,6 @@
|
|||
"mobile.de",
|
||||
"mobile.ir",
|
||||
"mobile01.com",
|
||||
"mobilesystemservice.com",
|
||||
"mobinsb.com",
|
||||
"mobirise.com",
|
||||
"mobirise.info",
|
||||
|
|
@ -5612,6 +5622,7 @@
|
|||
"mongabay.com",
|
||||
"mongodb.com",
|
||||
"mono3.biz",
|
||||
"mono4.biz",
|
||||
"mono5.biz",
|
||||
"mono7.biz",
|
||||
"monografias.com",
|
||||
|
|
@ -5655,10 +5666,8 @@
|
|||
"movavi.com",
|
||||
"movember.com",
|
||||
"moveon.org",
|
||||
"moviefone.com",
|
||||
"movieranker.com",
|
||||
"movieweb.com",
|
||||
"movistar.es",
|
||||
"movs4u.live",
|
||||
"moz.com",
|
||||
"mozaws.net",
|
||||
|
|
@ -5752,7 +5761,6 @@
|
|||
"myapp.com",
|
||||
"myasianbride.net",
|
||||
"mybigcommerce.com",
|
||||
"mybuzz.fun",
|
||||
"mycima.me",
|
||||
"mydailymagazine.com",
|
||||
"mydealz.de",
|
||||
|
|
@ -5905,7 +5913,6 @@
|
|||
"ndr.de",
|
||||
"ndrc.gov.cn",
|
||||
"ndtv.com",
|
||||
"nea.gov.cn",
|
||||
"nea.org",
|
||||
"nearme.com.cn",
|
||||
"nebraska.gov",
|
||||
|
|
@ -5970,6 +5977,7 @@
|
|||
"news-medical.net",
|
||||
"news.cn",
|
||||
"news.com.au",
|
||||
"news12.com",
|
||||
"news18.com",
|
||||
"news24.com",
|
||||
"news247.gr",
|
||||
|
|
@ -5980,7 +5988,6 @@
|
|||
"newscientist.com",
|
||||
"newsday.com",
|
||||
"newser.com",
|
||||
"newsgator.com",
|
||||
"newsit.gr",
|
||||
"newsmax.com",
|
||||
"newsmth.net",
|
||||
|
|
@ -6154,7 +6161,6 @@
|
|||
"ntv.com.tr",
|
||||
"ntv.io",
|
||||
"ntv.ru",
|
||||
"ntvtelugu.com",
|
||||
"nu.nl",
|
||||
"nuance.com",
|
||||
"nudevista.com",
|
||||
|
|
@ -6196,7 +6202,6 @@
|
|||
"oaspapps.com",
|
||||
"oath.cloud",
|
||||
"oath.com",
|
||||
"oberlin.edu",
|
||||
"oberlo.com",
|
||||
"obozrevatel.com",
|
||||
"observalgerie.com",
|
||||
|
|
@ -6343,6 +6348,7 @@
|
|||
"opentext.com",
|
||||
"openvpn.net",
|
||||
"openweathermap.org",
|
||||
"openwrt.org",
|
||||
"openx.com",
|
||||
"openx.net",
|
||||
"opera-mini.net",
|
||||
|
|
@ -6385,7 +6391,6 @@
|
|||
"oschina.net",
|
||||
"osd.mil",
|
||||
"osha.gov",
|
||||
"ospserver.net",
|
||||
"osticket.com",
|
||||
"osu.edu",
|
||||
"osxdaily.com",
|
||||
|
|
@ -6649,7 +6654,6 @@
|
|||
"pingdom.net",
|
||||
"pinimg.com",
|
||||
"pinkbike.com",
|
||||
"pinterest.at",
|
||||
"pinterest.ca",
|
||||
"pinterest.co.uk",
|
||||
"pinterest.com",
|
||||
|
|
@ -6718,6 +6722,7 @@
|
|||
"poetryfoundation.org",
|
||||
"poets.org",
|
||||
"pof.com",
|
||||
"point2homes.com",
|
||||
"pojoksatu.id",
|
||||
"pokemon.com",
|
||||
"poki.com",
|
||||
|
|
@ -6778,13 +6783,13 @@
|
|||
"potterybarn.com",
|
||||
"powells.com",
|
||||
"powerapp.download",
|
||||
"powerapps.com",
|
||||
"powerbi.com",
|
||||
"powerlinks.com",
|
||||
"powerofpositivity.com",
|
||||
"powerthesaurus.org",
|
||||
"powtoon.com",
|
||||
"poynter.org",
|
||||
"pozdravok.ru",
|
||||
"poznan.pl",
|
||||
"pp.ru",
|
||||
"pp.ua",
|
||||
|
|
@ -6890,6 +6895,7 @@
|
|||
"publpush.com",
|
||||
"pubmatic.com",
|
||||
"pulitzer.org",
|
||||
"pullcf.com",
|
||||
"pullcm.com",
|
||||
"pulzo.com",
|
||||
"puma.com",
|
||||
|
|
@ -6947,7 +6953,6 @@
|
|||
"qq.com",
|
||||
"qr-code-generator.com",
|
||||
"qsl.net",
|
||||
"qt.io",
|
||||
"qualaroo.com",
|
||||
"qualcomm.com",
|
||||
"qualtrics.com",
|
||||
|
|
@ -6956,7 +6961,6 @@
|
|||
"quantamagazine.org",
|
||||
"quantcast.com",
|
||||
"quantcount.com",
|
||||
"quantrimang.com",
|
||||
"quantserve.com",
|
||||
"qub.ac.uk",
|
||||
"queensu.ca",
|
||||
|
|
@ -7033,6 +7037,7 @@
|
|||
"rasadeghtesadi.com",
|
||||
"rasadvarzeshi.com",
|
||||
"raspberrypi.org",
|
||||
"ratemyprofessors.com",
|
||||
"rateyourmusic.com",
|
||||
"ratopati.com",
|
||||
"ravelry.com",
|
||||
|
|
@ -7108,6 +7113,7 @@
|
|||
"refinery29.com",
|
||||
"reg.ru",
|
||||
"regions.com",
|
||||
"register.it",
|
||||
"regjeringen.no",
|
||||
"regnum.ru",
|
||||
"regonline.com",
|
||||
|
|
@ -7142,7 +7148,6 @@
|
|||
"resetera.com",
|
||||
"residentadvisor.net",
|
||||
"reson8.com",
|
||||
"restream.io",
|
||||
"resultados-futbol.com",
|
||||
"retaildive.com",
|
||||
"retailmenot.com",
|
||||
|
|
@ -7156,7 +7161,6 @@
|
|||
"reverso.net",
|
||||
"reviewjournal.com",
|
||||
"revjet.com",
|
||||
"revolut.com",
|
||||
"revopush.com",
|
||||
"revues.org",
|
||||
"rezka.ag",
|
||||
|
|
@ -7318,14 +7322,12 @@
|
|||
"sabio.us",
|
||||
"sabq.org",
|
||||
"sacbee.com",
|
||||
"sachsen.de",
|
||||
"sacred-texts.com",
|
||||
"sae.org",
|
||||
"saednews.com",
|
||||
"safedog.cn",
|
||||
"safefinder.com",
|
||||
"safer-networking.org",
|
||||
"safeway.com",
|
||||
"sagawa-exp.co.jp",
|
||||
"sage.com",
|
||||
"sagepub.com",
|
||||
|
|
@ -7431,6 +7433,7 @@
|
|||
"sc.gov",
|
||||
"sc.gov.cn",
|
||||
"scamadviser.com",
|
||||
"scarlet-clicks.info",
|
||||
"scdn.co",
|
||||
"scene7.com",
|
||||
"sch.gr",
|
||||
|
|
@ -7544,6 +7547,7 @@
|
|||
"sendgrid.net",
|
||||
"sendinblue.com",
|
||||
"sendo.vn",
|
||||
"sendpulse.com",
|
||||
"sendspace.com",
|
||||
"seneweb.com",
|
||||
"sennheiser.com",
|
||||
|
|
@ -7566,6 +7570,7 @@
|
|||
"servimg.com",
|
||||
"serving-sys.com",
|
||||
"setare.com",
|
||||
"setareparsi.com",
|
||||
"setn.com",
|
||||
"seu.edu.cn",
|
||||
"seventeen.com",
|
||||
|
|
@ -7648,7 +7653,6 @@
|
|||
"shopifysvc.com",
|
||||
"shopstyle.co.uk",
|
||||
"shopstyle.com",
|
||||
"shoptime.com.br",
|
||||
"shorturl.at",
|
||||
"shrm.org",
|
||||
"shueisha.co.jp",
|
||||
|
|
@ -7792,7 +7796,6 @@
|
|||
"snapads.com",
|
||||
"snapchat.com",
|
||||
"snapdeal.com",
|
||||
"snapfish.com",
|
||||
"snapwidget.com",
|
||||
"snazzymaps.com",
|
||||
"sncf.com",
|
||||
|
|
@ -7892,6 +7895,7 @@
|
|||
"splice.com",
|
||||
"splunk.com",
|
||||
"spo-msedge.net",
|
||||
"spocket.co",
|
||||
"spokeo.com",
|
||||
"spokesman.com",
|
||||
"sponichi.co.jp",
|
||||
|
|
@ -7963,7 +7967,6 @@
|
|||
"stackpathcdn.com",
|
||||
"stackpathdns.com",
|
||||
"stacksocial.com",
|
||||
"stan.com.au",
|
||||
"standaard.be",
|
||||
"standard.co.uk",
|
||||
"standardchartered.com",
|
||||
|
|
@ -8000,7 +8003,6 @@
|
|||
"steam-chat.com",
|
||||
"steamcommunity.com",
|
||||
"steamcontent.com",
|
||||
"steamdb.info",
|
||||
"steampowered.com",
|
||||
"steamstatic.com",
|
||||
"steemit.com",
|
||||
|
|
@ -8029,7 +8031,6 @@
|
|||
"storify.com",
|
||||
"storm.mg",
|
||||
"storygize.net",
|
||||
"straightdope.com",
|
||||
"straitstimes.com",
|
||||
"strangled.net",
|
||||
"strath.ac.uk",
|
||||
|
|
@ -8049,7 +8050,6 @@
|
|||
"studfile.net",
|
||||
"studiopress.com",
|
||||
"studocu.com",
|
||||
"studopedia.ru",
|
||||
"study.com",
|
||||
"stuff.co.nz",
|
||||
"stumbleupon.com",
|
||||
|
|
@ -8162,13 +8162,11 @@
|
|||
"tabnak.ir",
|
||||
"taboola.com",
|
||||
"tadalafilprof.com",
|
||||
"tagar.id",
|
||||
"tagesanzeiger.ch",
|
||||
"tagesschau.de",
|
||||
"tagesspiegel.de",
|
||||
"tagged.com",
|
||||
"tahiamasr.com",
|
||||
"taifeng-chem.com",
|
||||
"taipeitimes.com",
|
||||
"taiwannews.com.tw",
|
||||
"takealot.com",
|
||||
|
|
@ -8226,6 +8224,7 @@
|
|||
"teamtreehouse.com",
|
||||
"teamusa.org",
|
||||
"teamviewer.com",
|
||||
"techacademy.jp",
|
||||
"techadvisor.co.uk",
|
||||
"techbang.com",
|
||||
"techcrunch.com",
|
||||
|
|
@ -8249,6 +8248,7 @@
|
|||
"techwalla.com",
|
||||
"techweb.com.cn",
|
||||
"tecmundo.com.br",
|
||||
"tecnoblog.net",
|
||||
"ted.com",
|
||||
"teenvogue.com",
|
||||
"teepublic.com",
|
||||
|
|
@ -8306,7 +8306,6 @@
|
|||
"test.de",
|
||||
"testbook.com",
|
||||
"texas.gov",
|
||||
"texasmonthly.com",
|
||||
"texastribune.org",
|
||||
"text.ru",
|
||||
"textnow.com",
|
||||
|
|
@ -8318,6 +8317,7 @@
|
|||
"thanhnien.vn",
|
||||
"thawte.com",
|
||||
"the-scientist.com",
|
||||
"the-sun.com",
|
||||
"theadvocate.com",
|
||||
"theage.com.au",
|
||||
"theathletic.com",
|
||||
|
|
@ -8396,7 +8396,6 @@
|
|||
"theonion.com",
|
||||
"theoutnet.com",
|
||||
"thepaper.cn",
|
||||
"theparisreview.org",
|
||||
"thepetitionsite.com",
|
||||
"thepirate-bay.org",
|
||||
"thepiratebay.org",
|
||||
|
|
@ -8416,7 +8415,6 @@
|
|||
"theroot.com",
|
||||
"thesaurus.com",
|
||||
"thesimpledollar.com",
|
||||
"thesimsresource.com",
|
||||
"thesmokinggun.com",
|
||||
"thespruce.com",
|
||||
"thespruceeats.com",
|
||||
|
|
@ -8468,7 +8466,6 @@
|
|||
"thriveglobal.com",
|
||||
"thrtle.com",
|
||||
"thumbtack.com",
|
||||
"thumbzilla.com",
|
||||
"thunderbird.net",
|
||||
"ti.com",
|
||||
"tianqi.com",
|
||||
|
|
@ -8625,6 +8622,7 @@
|
|||
"transunion.com",
|
||||
"travelandleisure.com",
|
||||
"travelchannel.com",
|
||||
"travelermaster.com",
|
||||
"travelocity.com",
|
||||
"traveloka.com",
|
||||
"travis-ci.org",
|
||||
|
|
@ -8655,7 +8653,6 @@
|
|||
"tripadvisor.ca",
|
||||
"tripadvisor.co.uk",
|
||||
"tripadvisor.com",
|
||||
"tripadvisor.com.au",
|
||||
"tripadvisor.de",
|
||||
"tripadvisor.es",
|
||||
"tripadvisor.fr",
|
||||
|
|
@ -8677,7 +8674,6 @@
|
|||
"truste.com",
|
||||
"trustedreviews.com",
|
||||
"trustpilot.com",
|
||||
"trustutn.org",
|
||||
"trustwave.com",
|
||||
"trustx.org",
|
||||
"truthout.org",
|
||||
|
|
@ -8731,7 +8727,6 @@
|
|||
"tutorialspoint.com",
|
||||
"tutsplus.com",
|
||||
"tutu.ru",
|
||||
"tuv.com",
|
||||
"tuwien.ac.at",
|
||||
"tv-asahi.co.jp",
|
||||
"tv-tokyo.co.jp",
|
||||
|
|
@ -8837,7 +8832,6 @@
|
|||
"udn.com",
|
||||
"uea.ac.uk",
|
||||
"uefa.com",
|
||||
"ueuodgnrhb.com",
|
||||
"ufc.com",
|
||||
"ufl.edu",
|
||||
"ufreegames.com",
|
||||
|
|
@ -8894,7 +8888,6 @@
|
|||
"umengcloud.com",
|
||||
"umich.edu",
|
||||
"umkc.edu",
|
||||
"umm.ac.id",
|
||||
"umm.edu",
|
||||
"umms.org",
|
||||
"umn.edu",
|
||||
|
|
@ -9007,6 +9000,7 @@
|
|||
"unwto.org",
|
||||
"uny.ac.id",
|
||||
"uoa.gr",
|
||||
"uoc.edu",
|
||||
"uodoo.com",
|
||||
"uoguelph.ca",
|
||||
"uol.com.br",
|
||||
|
|
@ -9045,6 +9039,7 @@
|
|||
"urdupoint.com",
|
||||
"uri.edu",
|
||||
"url.cn",
|
||||
"urldefense.com",
|
||||
"urssaf.fr",
|
||||
"us-cert.gov",
|
||||
"us.com",
|
||||
|
|
@ -9102,6 +9097,7 @@
|
|||
"uta.edu",
|
||||
"utah.edu",
|
||||
"utah.gov",
|
||||
"utas.edu.au",
|
||||
"utdallas.edu",
|
||||
"utep.edu",
|
||||
"utexas.edu",
|
||||
|
|
@ -9136,6 +9132,7 @@
|
|||
"v2ex.com",
|
||||
"va.gov",
|
||||
"vagina.nl",
|
||||
"valentino.com",
|
||||
"validcbdoil.com",
|
||||
"value-domain.com",
|
||||
"valuecommerce.com",
|
||||
|
|
@ -9157,7 +9154,6 @@
|
|||
"vbspbx.com",
|
||||
"vbulletin.com",
|
||||
"vc.ru",
|
||||
"vchecks.me",
|
||||
"vcu.edu",
|
||||
"vecteezy.com",
|
||||
"vector.co.jp",
|
||||
|
|
@ -9208,6 +9204,7 @@
|
|||
"videopress.com",
|
||||
"vidible.tv",
|
||||
"vidio.com",
|
||||
"vidiq.com",
|
||||
"vidstream.kim",
|
||||
"vidstream.online",
|
||||
"vidstream.to",
|
||||
|
|
@ -9225,6 +9222,7 @@
|
|||
"vimeopro.com",
|
||||
"vindicosuite.com",
|
||||
"vine.co",
|
||||
"vingle.net",
|
||||
"vip.com",
|
||||
"virgilio.it",
|
||||
"virgin.com",
|
||||
|
|
@ -9331,6 +9329,7 @@
|
|||
"walgreens.com",
|
||||
"walkerland.com.tw",
|
||||
"walkme.com",
|
||||
"walkscore.com",
|
||||
"walla.co.il",
|
||||
"wallapop.com",
|
||||
"wallethub.com",
|
||||
|
|
@ -9711,7 +9710,6 @@
|
|||
"xhamster.one",
|
||||
"xhamster3.desi",
|
||||
"xhamsterlive.com",
|
||||
"xhamsterpremium.com",
|
||||
"xhby.net",
|
||||
"xiachufang.com",
|
||||
"xiami.com",
|
||||
|
|
@ -9832,6 +9830,7 @@
|
|||
"yjc.ir",
|
||||
"yjtag.jp",
|
||||
"yle.fi",
|
||||
"ylsw.com",
|
||||
"ymp4.download",
|
||||
"yn.gov.cn",
|
||||
"yna.co.kr",
|
||||
|
|
@ -9868,9 +9867,9 @@
|
|||
"youradchoices.com",
|
||||
"yourdictionary.com",
|
||||
"yourdomain.com",
|
||||
"yournewtab.com",
|
||||
"youronlinechoices.com",
|
||||
"youronlinechoices.eu",
|
||||
"yoursportspot.com",
|
||||
"yourstory.com",
|
||||
"yourtango.com",
|
||||
"yousendit.com",
|
||||
|
|
@ -9996,6 +9995,7 @@
|
|||
"zozo.jp",
|
||||
"zscaler.com",
|
||||
"zulily.com",
|
||||
"zum.com",
|
||||
"zumiez.com",
|
||||
"zurb.com",
|
||||
"zxart.cn",
|
||||
|
|
@ -10010,5 +10010,5 @@
|
|||
],
|
||||
"name": "Top 10K most-used sites from Tranco",
|
||||
"type": "hostname",
|
||||
"version": 20200720
|
||||
"version": 20200723
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,11 +3,11 @@
|
|||
|
||||
import json
|
||||
|
||||
from generator import download_to_file, get_version, write_to_file
|
||||
from generator import download_to_file, get_version, write_to_file, get_abspath_source_file
|
||||
|
||||
|
||||
def process(file, dst):
|
||||
with open(file, 'r') as json_file:
|
||||
with open(get_abspath_source_file(file), 'r') as json_file:
|
||||
amazon_aws_ip_list = json.load(json_file)
|
||||
l = []
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
|
||||
import zipfile
|
||||
|
||||
from generator import download_to_file, get_version, write_to_file
|
||||
from generator import download_to_file, get_version, write_to_file, get_abspath_source_file
|
||||
|
||||
|
||||
def process(file):
|
||||
|
|
@ -53,7 +53,7 @@ def generate(sites, warninglist, dst):
|
|||
|
||||
|
||||
def get_lists(file):
|
||||
with zipfile.ZipFile(file, 'r') as cisco_lists:
|
||||
with zipfile.ZipFile(get_abspath_source_file(file), 'r') as cisco_lists:
|
||||
for name in cisco_lists.namelist():
|
||||
if name == "top-1m.csv":
|
||||
with cisco_lists.open(name) as cisco_list:
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from generator import download_to_file, get_version, write_to_file
|
||||
from generator import download_to_file, get_version, write_to_file, get_abspath_source_file
|
||||
|
||||
|
||||
def process(files, dst):
|
||||
|
|
@ -15,7 +15,7 @@ def process(files, dst):
|
|||
}
|
||||
|
||||
for file in files:
|
||||
with open(file, 'r') as f:
|
||||
with open(get_abspath_source_file(file), 'r') as f:
|
||||
ips = f.readlines()
|
||||
for ip in ips:
|
||||
warninglist['list'].append(ip.strip())
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from generator import download_to_file, get_version, write_to_file
|
||||
from generator import download_to_file, get_version, write_to_file, get_abspath_source_file
|
||||
|
||||
|
||||
def process(files, dst):
|
||||
|
|
@ -16,7 +16,7 @@ def process(files, dst):
|
|||
}
|
||||
|
||||
for file in files:
|
||||
with open(file, 'r') as f:
|
||||
with open(get_abspath_source_file(file), 'r') as f:
|
||||
ips = f.readlines()
|
||||
for ip in ips:
|
||||
warninglist['list'].append(ip.strip())
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import json
|
|||
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from generator import download, download_to_file, get_version, write_to_file
|
||||
from generator import download, download_to_file, get_version, write_to_file, get_abspath_source_file
|
||||
|
||||
|
||||
def get_json_url(page):
|
||||
|
|
@ -26,7 +26,7 @@ def process(file, dst):
|
|||
'type': 'cidr'
|
||||
}
|
||||
|
||||
with open(file, 'r') as json_file:
|
||||
with open(get_abspath_source_file(file), 'r') as json_file:
|
||||
ms_azure_ip_list = json.load(json_file)
|
||||
|
||||
for value in ms_azure_ip_list['values']:
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import csv
|
|||
import ipaddress
|
||||
import logging
|
||||
|
||||
from generator import download_to_file, get_version, write_to_file
|
||||
from generator import download_to_file, get_version, write_to_file, get_abspath_source_file
|
||||
|
||||
|
||||
def process(file):
|
||||
|
|
@ -52,7 +52,7 @@ def generate(data_list, warninglist, dst):
|
|||
|
||||
def get_lists(file):
|
||||
|
||||
with open(file) as csv_file:
|
||||
with open(get_abspath_source_file(file)) as csv_file:
|
||||
servers_list = csv.reader(csv_file, delimiter=',', quotechar='"')
|
||||
|
||||
lipv4 = []
|
||||
|
|
|
|||
|
|
@ -3,11 +3,12 @@
|
|||
|
||||
import zipfile
|
||||
|
||||
from generator import download_to_file, get_version, write_to_file
|
||||
from generator import (download_to_file, get_abspath_source_file, get_version,
|
||||
write_to_file)
|
||||
|
||||
|
||||
def process(file, dst):
|
||||
with zipfile.ZipFile(file, 'r') as alexa_lists:
|
||||
with zipfile.ZipFile(get_abspath_source_file(file), 'r') as alexa_lists:
|
||||
for name in alexa_lists.namelist():
|
||||
if name == "top-1m.csv":
|
||||
with alexa_lists.open(name) as top:
|
||||
|
|
|
|||
|
|
@ -1,12 +1,12 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from generator import download_to_file, get_version, write_to_file
|
||||
from generator import download_to_file, get_version, write_to_file, get_abspath_source_file
|
||||
|
||||
|
||||
def process(file, dst):
|
||||
|
||||
with open(file, newline='\n', encoding='utf-8', errors='replace') as csv_file:
|
||||
with open(get_abspath_source_file(file), newline='\n', encoding='utf-8', errors='replace') as csv_file:
|
||||
sites = csv_file.readlines()[:10000]
|
||||
|
||||
warninglist = {
|
||||
|
|
|
|||
|
|
@ -1,69 +1,44 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import requests
|
||||
import datetime
|
||||
import json
|
||||
import csv
|
||||
import os
|
||||
|
||||
from generator import download_to_file, get_version, write_to_file, get_abspath_source_file
|
||||
|
||||
|
||||
# TODO: Include MozRank
|
||||
def process(files, dst):
|
||||
|
||||
moz_url_domains = "https://moz.com/top500/domains/csv"
|
||||
moz_url_pages = "https://moz.com/top500/pages/csv"
|
||||
warninglist = {
|
||||
'description': "Event contains one or more entries from the top 500 of the most used domains (Mozilla).",
|
||||
'version': get_version(),
|
||||
'name': "Top 500 domains and pages from https://moz.com/top500",
|
||||
'type': 'hostname',
|
||||
'list': [],
|
||||
'matching_attributes': ['hostname', 'domain', 'uri', 'url']
|
||||
}
|
||||
|
||||
moz_file_domains = "/tmp/top500.domains.csv"
|
||||
moz_file_pages = "/tmp/top500.pages.csv"
|
||||
for file in files:
|
||||
with open(get_abspath_source_file(file)) as csv_file:
|
||||
csv_reader = csv.reader(csv_file, delimiter=',')
|
||||
for row in csv_reader:
|
||||
v = row[1]
|
||||
warninglist['list'].append(v.rstrip().rstrip('/'))
|
||||
|
||||
user_agent = {"User-agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:46.0) Gecko/20100101 Firefox/46.0"}
|
||||
write_to_file(warninglist, dst)
|
||||
|
||||
rDomains = requests.get(moz_url_domains, headers=user_agent)
|
||||
rPages = requests.get(moz_url_pages, headers=user_agent)
|
||||
open(moz_file_domains, 'wb').write(rDomains.content)
|
||||
open(moz_file_pages, 'wb').write(rPages.content)
|
||||
|
||||
moz_warninglist = {}
|
||||
version = int(datetime.date.today().strftime('%Y%m%d'))
|
||||
if __name__ == '__main__':
|
||||
moz_domains_url = "https://moz.com/top-500/download/?table=top500Domains"
|
||||
#moz_pages_url = "https://moz.com/top500/pages/csv"
|
||||
|
||||
moz_warninglist['description'] = "Event contains one or more entries from the top 500 of the most used domains (Mozilla)."
|
||||
d = datetime.datetime.now()
|
||||
moz_warninglist['version'] = version
|
||||
moz_warninglist['name'] = "Top 500 domains and pages from https://moz.com/top500"
|
||||
moz_warninglist['type'] = 'hostname'
|
||||
moz_warninglist['list'] = []
|
||||
moz_warninglist['matching_attributes'] = ['hostname', 'domain', 'uri', 'url']
|
||||
moz_domains_file = "moz-top500.domains.csv"
|
||||
#moz_pages_file = "moz-top500.pages.csv"
|
||||
|
||||
with open(moz_file_domains) as csv_file:
|
||||
csv_reader = csv.reader(csv_file, delimiter=',')
|
||||
line_count = 0
|
||||
for row in csv_reader:
|
||||
if line_count == 0:
|
||||
#print(f'Column names are {", ".join(row)}')
|
||||
line_count += 1
|
||||
else:
|
||||
#print(f'\t{row[0]}. {row[1]}, MozTrust: {row[5]}.')
|
||||
v = row[1]
|
||||
moz_warninglist['list'].append(v.rstrip().rstrip('/'))
|
||||
line_count += 1
|
||||
moz_dst = 'moz-top500'
|
||||
|
||||
with open(moz_file_pages) as csv_file:
|
||||
csv_reader = csv.reader(csv_file, delimiter=',')
|
||||
line_count = 0
|
||||
for row in csv_reader:
|
||||
if line_count == 0:
|
||||
#print(f'Column names are {", ".join(row)}')
|
||||
line_count += 1
|
||||
else:
|
||||
#print(f'\t{row[0]}. {row[1]}, MozTrust: {row[5]}.')
|
||||
v = row[1]
|
||||
moz_warninglist['list'].append(v.rstrip().rstrip('/'))
|
||||
line_count += 1
|
||||
download_to_file(moz_domains_url, moz_domains_file)
|
||||
#download_to_file(moz_pages_url, moz_pages_file)
|
||||
|
||||
moz_warninglist['list'] = sorted(set(moz_warninglist['list']))
|
||||
print(json.dumps(moz_warninglist))
|
||||
|
||||
try:
|
||||
os.remove(moz_file_domains)
|
||||
os.remove(moz_file_pages)
|
||||
except:
|
||||
print(f'Perhaps {moz_file_domains}/{moz_file_pages} does not exist.')
|
||||
#process([moz_domains_file, moz_pages_file], moz_dst)
|
||||
process([moz_domains_file], moz_dst)
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import csv
|
|||
|
||||
from OpenSSL.crypto import FILETYPE_PEM, load_certificate
|
||||
|
||||
from generator import download_to_file, get_version, write_to_file
|
||||
from generator import download_to_file, get_version, write_to_file, get_abspath_source_file
|
||||
|
||||
|
||||
def gethash(cert, digest):
|
||||
|
|
@ -14,7 +14,7 @@ def gethash(cert, digest):
|
|||
|
||||
def process(file, dst, type):
|
||||
hashes = set()
|
||||
with open(file, 'r') as f_in:
|
||||
with open(get_abspath_source_file(file), 'r') as f_in:
|
||||
for obj in csv.DictReader(f_in):
|
||||
pem = obj['PEM Info'].strip("'").replace(
|
||||
'\r', '').replace('\n\n', '\n')
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
|
||||
import zipfile
|
||||
|
||||
from generator import download_to_file, get_version, write_to_file
|
||||
from generator import download_to_file, get_version, write_to_file, get_abspath_source_file
|
||||
|
||||
|
||||
def process(file):
|
||||
|
|
@ -42,7 +42,7 @@ def generate(sites, warninglist, dst):
|
|||
|
||||
|
||||
def get_lists(file):
|
||||
with zipfile.ZipFile(file, 'r') as tranco_lists:
|
||||
with zipfile.ZipFile(get_abspath_source_file(file), 'r') as tranco_lists:
|
||||
for name in tranco_lists.namelist():
|
||||
if name == 'top-1m.csv':
|
||||
with tranco_lists.open(name) as tranco:
|
||||
|
|
|
|||
|
|
@ -3,49 +3,90 @@
|
|||
|
||||
import datetime
|
||||
import json
|
||||
from inspect import currentframe, getframeinfo
|
||||
from os import path
|
||||
import logging
|
||||
from inspect import currentframe, getframeinfo, getmodulename, stack
|
||||
from os import mkdir, path
|
||||
|
||||
import requests
|
||||
from dateutil.parser import parse as parsedate
|
||||
|
||||
|
||||
def init_logging():
|
||||
rel_path = getframeinfo(currentframe()).filename
|
||||
current_folder = path.dirname(path.abspath(rel_path))
|
||||
LOG_DIR = path.join(current_folder, '../generators.log')
|
||||
|
||||
logFormatter = logging.Formatter(
|
||||
"[%(asctime)s] %(levelname)s::%(funcName)s()::%(message)s")
|
||||
rootLogger = logging.getLogger()
|
||||
rootLogger.setLevel(logging.INFO)
|
||||
# Log to file
|
||||
fileHandler = logging.FileHandler(LOG_DIR)
|
||||
fileHandler.setFormatter(logFormatter)
|
||||
rootLogger.addHandler(fileHandler)
|
||||
# Log to console too
|
||||
''' consoleHandler = logging.StreamHandler()
|
||||
consoleHandler.setFormatter(logFormatter)
|
||||
rootLogger.addHandler(consoleHandler) '''
|
||||
return rootLogger
|
||||
|
||||
|
||||
init_logging()
|
||||
|
||||
|
||||
def download_to_file(url, file):
|
||||
user_agent = {"User-agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:46.0) Gecko/20100101 Firefox/46.0"}
|
||||
frame_records = stack()[1]
|
||||
caller = getmodulename(frame_records[1]).upper()
|
||||
|
||||
user_agent = {
|
||||
"User-agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:46.0) Gecko/20100101 Firefox/46.0"}
|
||||
try:
|
||||
r = requests.head(url, headers=user_agent)
|
||||
url_datetime = parsedate(r.headers['Last-Modified']).astimezone()
|
||||
file_datetime = datetime.datetime.fromtimestamp(
|
||||
path.getmtime(file)).astimezone()
|
||||
path.getmtime(get_abspath_source_file(file))).astimezone()
|
||||
|
||||
if(url_datetime > file_datetime):
|
||||
logging.info('{} File on server is newer, so downloading update to {}'.format(
|
||||
caller, get_abspath_source_file(file)))
|
||||
actual_download_to_file(url, file, user_agent)
|
||||
except KeyError as ex:
|
||||
logging.warning(str(ex))
|
||||
else:
|
||||
logging.info(
|
||||
'{} File on server is older, nothing to do'.format(caller))
|
||||
except KeyError as exc:
|
||||
logging.warning('{} KeyError in the headers. the {} header was not sent by server {}. Downloading file'.format(
|
||||
caller, str(exc), url))
|
||||
actual_download_to_file(url, file, user_agent)
|
||||
|
||||
except FileNotFoundError as exc:
|
||||
logging.info(
|
||||
"{} File didn't exist, so downloading {} from {}".format(caller, file, url))
|
||||
actual_download_to_file(url, file, user_agent)
|
||||
except Exception as exc:
|
||||
logging.warning(
|
||||
'{} General exception occured: {}.'.format(caller, str(exc)))
|
||||
actual_download_to_file(url, file, user_agent)
|
||||
|
||||
|
||||
def actual_download_to_file(url, file, user_agent):
|
||||
r = requests.get(url, headers=user_agent)
|
||||
with open(file, 'wb') as fd:
|
||||
with open(get_abspath_source_file(file), 'wb') as fd:
|
||||
for chunk in r.iter_content(4096):
|
||||
fd.write(chunk)
|
||||
|
||||
|
||||
def process_stream(url):
|
||||
r = requests.get(url, stream=True)
|
||||
|
||||
|
||||
data_list = []
|
||||
for line in r.iter_lines():
|
||||
v = line.decode('utf-8')
|
||||
if not v.startswith("#"):
|
||||
if v:
|
||||
data_list.append(v)
|
||||
|
||||
|
||||
return data_list
|
||||
|
||||
|
||||
|
||||
def download(url):
|
||||
user_agent = {
|
||||
"User-agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:46.0) Gecko/20100101 Firefox/46.0"}
|
||||
|
|
@ -60,6 +101,15 @@ def get_abspath_list_file(dst):
|
|||
return path.abspath(path.realpath(real_path))
|
||||
|
||||
|
||||
def get_abspath_source_file(dst):
|
||||
rel_path = getframeinfo(currentframe()).filename
|
||||
current_folder = path.dirname(path.abspath(rel_path))
|
||||
tmp_path = path.join(current_folder, '../tmp/')
|
||||
if not path.exists(tmp_path):
|
||||
mkdir(tmp_path)
|
||||
return path.abspath(path.realpath(path.join(tmp_path, '{dst}'.format(dst=dst))))
|
||||
|
||||
|
||||
def get_version():
|
||||
return int(datetime.date.today().strftime('%Y%m%d'))
|
||||
|
||||
|
|
@ -70,7 +120,24 @@ def unique_sorted_warninglist(warninglist):
|
|||
|
||||
|
||||
def write_to_file(warninglist, dst):
|
||||
with open(get_abspath_list_file(dst), 'w') as data_file:
|
||||
json.dump(unique_sorted_warninglist(warninglist),
|
||||
data_file, indent=2, sort_keys=True)
|
||||
data_file.write("\n")
|
||||
frame_records = stack()[1]
|
||||
caller = getmodulename(frame_records[1]).upper()
|
||||
|
||||
try:
|
||||
with open(get_abspath_list_file(dst), 'w') as data_file:
|
||||
json.dump(unique_sorted_warninglist(warninglist),
|
||||
data_file, indent=2, sort_keys=True)
|
||||
data_file.write("\n")
|
||||
logging.info('New warninglist written to {}.'.format(
|
||||
get_abspath_list_file(dst)))
|
||||
except Exception as exc:
|
||||
logging.error(
|
||||
'{} General exception occured: {}.'.format(caller, str(exc)))
|
||||
|
||||
|
||||
def main():
|
||||
init_logging()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
|
|||
Loading…
Reference in New Issue