From e7401c9cbe4ca3e3abf99f64d861a0e8f6457f07 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sun, 13 Jun 2021 14:49:57 +0200 Subject: [PATCH] new: List of known SMTP sending IP ranges --- generate_all.sh | 1 + lists/smtp-sending-ips/list.json | 1024 ++++++++++++++++++++ tools/generate-crl-ip-domains.py | 12 +- tools/generate-google-gmail-sending-ips.py | 37 +- tools/generate-smtp-sending-ips.py | 87 ++ tools/generator.py | 100 +- 6 files changed, 1212 insertions(+), 49 deletions(-) create mode 100644 lists/smtp-sending-ips/list.json create mode 100644 tools/generate-smtp-sending-ips.py diff --git a/generate_all.sh b/generate_all.sh index ebab75f..6628d07 100755 --- a/generate_all.sh +++ b/generate_all.sh @@ -30,6 +30,7 @@ python3 generate-wikimedia.py python3 generate-second-level-tlds.py python3 generate-google-gcp.py python3 generate-google-gmail-sending-ips.py +python3 generate-smtp-sending-ips.py popd ./jq_all_the_things.sh diff --git a/lists/smtp-sending-ips/list.json b/lists/smtp-sending-ips/list.json new file mode 100644 index 0000000..e1663c3 --- /dev/null +++ b/lists/smtp-sending-ips/list.json @@ -0,0 +1,1024 @@ +{ + "description": "List of IP ranges for known SMTP servers.", + "list": [ + "10.69.69.68/31", + "101.71.154.208/28", + "103.129.252.0/26", + "103.129.255.0/24", + "103.151.192.0/23", + "103.18.109.138/32", + "103.7.29.0/24", + "103.74.28.0/24", + "103.9.8.121/32", + "103.9.8.122/31", + "104.130.122.0/23", + "104.130.96.0/28", + "104.17.225.56/32", + "104.17.226.56/32", + "104.215.186.3/32", + "104.47.0.0/17", + "106.10.148.0/24", + "106.10.150.0/23", + "106.10.167.0/24", + "106.10.176.0/24", + "106.10.240.0/22", + "106.10.244.0/24", + "106.55.201.0/24", + "108.177.8.0/21", + "108.177.96.0/19", + "109.244.180.0/24", + "110.234.248.0/21", + "111.161.68.0/24", + "111.221.112.0/21", + "111.221.23.128/25", + "111.221.26.0/27", + "111.221.66.0/25", + "111.221.69.128/25", + "111.91.135.0/27", + "111.91.135.48/30", + "112.19.199.64/29", + "112.19.242.64/29", + "113.108.11.0/24", + "113.108.214.0/27", + "113.108.225.0/24", + "113.108.226.64/26", + "113.108.23.0/24", + "113.96.223.0/24", + "114.110.48.0/20", + "114.111.32.0/24", + "114.111.38.0/23", + "114.111.40.0/24", + "114.111.45.0/24", + "114.111.47.64/26", + "114.111.64.0/18", + "114.134.80.128/26", + "115.124.30.0/24", + "115.236.118.192/27", + "115.236.121.128/26", + "115.236.127.0/24", + "116.214.12.0/24", + "117.53.114.0/23", + "118.151.224.0/19", + "119.147.193.0/24", + "119.147.194.0/24", + "119.205.231.213/32", + "119.205.231.214/32", + "119.205.232.147/32", + "119.205.232.148/32", + "12.130.152.112/28", + "121.0.18.0/23", + "121.0.30.0/24", + "121.14.1.15/32", + "121.14.1.16/31", + "121.14.129.0/24", + "121.14.133.0/24", + "121.14.19.0/24", + "121.14.53.0/24", + "121.189.42.0/24", + "121.195.178.48/28", + "121.51.40.128/25", + "121.51.6.0/25", + "122.13.158.0/25", + "123.125.104.0/24", + "123.125.50.0/24", + "123.126.65.0/24", + "123.126.96.0/23", + "123.58.177.0/24", + "123.58.178.0/24", + "123.58.189.128/25", + "124.108.96.0/24", + "124.83.128.0/17", + "125.209.192.0/18", + "125.39.104.0/24", + "128.140.168.0/21", + "128.65.127.0/26", + "129.232.203.80/28", + "129.91.190.200/29", + "129.91.222.0/29", + "13.115.33.207/32", + "13.124.252.14/32", + "13.125.100.84/32", + "13.125.80.34/32", + "13.230.129.231/32", + "130.211.0.0/22", + "131.0.24.40/29", + "131.0.24.64/28", + "131.0.24.8/29", + "131.0.26.192/26", + "131.117.171.0/28", + "131.253.121.20/32", + "131.253.121.52/32", + "134.170.27.8/32", + "135.84.80.192/26", + "135.84.82.0/24", + "136.143.182.0/23", + "136.143.184.0/24", + "136.143.188.0/24", + "137.188.0.0/16", + "139.178.64.159/32", + "139.178.64.195/32", + "139.180.17.0/24", + "14.17.21.0/24", + "14.17.43.0/24", + "14.17.44.0/24", + "14.215.153.0/24", + "14.215.154.0/23", + "14.215.91.0/24", + "14.29.82.0/25", + "140.108.1.0/24", + "140.108.22.0/26", + "140.108.26.0/24", + "141.146.154.96/27", + "141.146.165.0/27", + "141.193.32.0/23", + "141.8.132.0/24", + "143.244.80.0/20", + "143.47.149.160/27", + "143.47.167.160/27", + "143.55.224.0/20", + "144.178.36.0/24", + "144.178.38.0/24", + "146.188.102.26/31", + "146.188.97.43/32", + "146.20.112.0/26", + "146.20.113.0/24", + "146.20.191.0/24", + "147.135.98.120/32", + "147.149.100.78/32", + "147.149.100.81/32", + "147.149.196.177/32", + "147.149.196.181/32", + "147.75.65.173/32", + "147.75.65.174/32", + "147.75.98.190/32", + "148.105.8.0/21", + "148.163.149.93/32", + "148.163.153.92/32", + "148.235.52.0/26", + "148.87.89.32/28", + "149.72.0.0/16", + "15.222.199.59/32", + "157.55.0.192/26", + "157.55.1.128/26", + "157.55.11.0/25", + "157.55.157.128/25", + "157.55.2.0/25", + "157.55.225.0/25", + "157.55.49.0/25", + "157.55.61.0/24", + "157.55.9.128/25", + "157.56.232.0/21", + "157.56.24.0/25", + "157.56.240.0/20", + "158.247.16.0/20", + "158.69.163.48/29", + "159.135.132.128/25", + "159.135.140.80/29", + "159.135.224.0/20", + "160.92.124.64/26", + "160.92.140.180/31", + "160.92.141.148/31", + "160.92.141.150/32", + "161.38.192.0/20", + "162.115.0.0/16", + "162.144.69.95/32", + "162.144.91.17/32", + "162.144.91.18/32", + "162.208.119.181/32", + "162.62.52.214/32", + "162.62.55.67/32", + "162.62.57.137/32", + "162.62.57.210/32", + "162.62.57.252/32", + "162.62.57.49/32", + "162.62.57.64/32", + "162.62.57.87/32", + "162.62.58.211/32", + "162.62.58.216/32", + "162.62.58.69/32", + "162.62.63.194/32", + "163.177.87.0/24", + "163.47.180.0/23", + "166.78.230.128/25", + "166.78.68.0/22", + "167.220.67.238/32", + "167.89.0.0/17", + "168.245.0.0/17", + "17.110.0.0/15", + "17.120.0.0/16", + "17.133.0.0/16", + "17.139.0.0/16", + "17.142.0.0/15", + "17.151.1.0/24", + "17.158.0.0/15", + "17.162.0.0/15", + "17.164.0.0/16", + "17.171.37.0/24", + "17.172.0.0/16", + "17.179.168.0/23", + "17.36.0.0/16", + "17.41.0.0/16", + "17.58.0.0/16", + "172.217.0.0/19", + "172.217.128.0/19", + "172.217.160.0/20", + "172.217.192.0/19", + "172.217.32.0/20", + "172.253.112.0/20", + "172.253.56.0/21", + "172.255.62.10/31", + "173.194.0.0/16", + "173.228.155.0/24", + "173.228.157.0/24", + "173.239.96.0/19", + "173.45.18.0/24", + "175.126.189.0/24", + "175.158.24.43/32", + "176.34.21.58/32", + "177.153.0.128/25", + "177.153.23.241/32", + "177.153.23.242/32", + "177.153.24.32/27", + "177.153.24.64/27", + "177.153.25.32/27", + "177.153.28.0/24", + "177.154.139.160/27", + "177.23.48.0/21", + "178.22.92.0/23", + "178.239.80.0/20", + "178.249.98.17/32", + "178.77.121.128/26", + "179.124.32.224/28", + "179.188.27.0/26", + "179.188.27.128/26", + "179.188.30.0/24", + "179.190.28.142/31", + "179.190.28.144/31", + "179.190.28.146/32", + "179.190.28.16/28", + "179.190.29.44/30", + "179.190.29.48/32", + "18.194.254.142/32", + "18.208.124.128/25", + "180.149.134.111/32", + "180.149.135.147/32", + "180.149.153.121/32", + "180.150.142.123/32", + "180.150.142.124/32", + "180.150.154.88/32", + "180.150.154.92/31", + "180.163.24.128/25", + "180.168.204.0/22", + "182.22.109.0/24", + "182.22.12.0/24", + "182.22.124.0/24", + "182.22.20.0/26", + "182.22.32.0/19", + "182.22.90.0/23", + "183.2.187.0/25", + "183.2.218.32/28", + "183.232.56.48/28", + "183.3.226.0/24", + "183.3.255.0/24", + "183.60.52.0/24", + "183.60.61.0/24", + "183.60.8.0/24", + "183.61.185.0/24", + "183.61.51.0/24", + "183.61.52.0/24", + "183.62.104.0/24", + "183.79.0.0/16", + "184.105.206.0/24", + "184.105.67.0/24", + "184.150.200.0/24", + "185.12.80.0/22", + "185.136.188.88/32", + "185.136.189.88/32", + "185.189.236.0/22", + "185.211.120.0/22", + "185.250.236.0/22", + "185.3.161.12/32", + "185.3.161.7/32", + "185.3.161.8/32", + "185.34.84.0/24", + "185.5.136.0/22", + "185.70.40.0/23", + "185.70.43.0/24", + "186.202.147.96/27", + "186.202.173.96/27", + "186.202.4.42/32", + "186.234.128.16/29", + "187.17.116.0/24", + "187.191.73.0/24", + "187.31.0.0/24", + "187.31.64.0/24", + "188.125.69.0/24", + "188.165.51.139/32", + "188.172.128.0/20", + "188.93.56.0/24", + "188.93.58.0/23", + "189.125.144.0/24", + "189.56.23.224/27", + "189.89.107.198/32", + "190.216.57.112/28", + "191.252.12.224/27", + "191.252.14.0/25", + "191.252.30.0/24", + "191.252.36.0/24", + "191.252.40.0/25", + "191.252.57.0/25", + "191.252.7.128/25", + "191.252.79.0/25", + "192.161.144.0/20", + "192.237.158.0/23", + "192.254.112.0/20", + "192.64.236.0/23", + "192.64.238.0/24", + "192.76.84.0/24", + "193.109.81.0/24", + "193.113.108.40/31", + "193.131.254.22/32", + "193.38.82.91/32", + "193.38.82.92/32", + "194.42.96.0/24", + "195.13.11.0/24", + "195.13.15.224/28", + "195.130.132.32/27", + "195.130.137.64/27", + "195.130.225.0/24", + "195.140.184.104/29", + "195.140.184.112/32", + "195.140.184.231/32", + "195.140.184.232/29", + "195.140.184.240/29", + "195.211.128.0/22", + "195.238.20.0/24", + "195.238.22.0/24", + "195.238.4.0/22", + "195.62.75.16/29", + "198.2.128.0/18", + "198.21.0.0/21", + "198.23.5.0/24", + "198.37.144.0/20", + "198.61.254.0/23", + "199.10.31.235/32", + "199.10.31.236/32", + "199.127.232.0/22", + "199.243.119.244/32", + "199.249.25.0/24", + "199.255.192.0/22", + "199.74.155.0/32", + "20.47.149.138/32", + "200.110.133.18/32", + "200.136.38.0/24", + "200.143.181.192/26", + "200.147.0.0/22", + "200.147.32.0/22", + "200.147.40.192/26", + "200.147.41.0/25", + "200.147.54.0/23", + "200.147.96.0/23", + "200.147.98.144/29", + "200.147.98.64/27", + "200.147.99.0/26", + "200.147.99.104/29", + "200.154.152.0/24", + "200.176.10.0/23", + "200.176.2.0/23", + "200.176.5.22/32", + "200.176.5.24/30", + "200.176.5.7/32", + "200.177.255.66/32", + "200.177.255.72/32", + "200.177.255.79/32", + "200.177.255.80/32", + "200.177.255.86/32", + "200.177.255.96/32", + "200.192.169.178/32", + "200.192.194.2/31", + "200.221.29.128/32", + "200.234.203.0/25", + "200.234.204.130/32", + "200.45.0.194/31", + "200.45.0.196/30", + "200.45.0.210/31", + "200.45.0.212/30", + "200.45.0.216/30", + "200.45.0.220/32", + "200.45.1.130/31", + "200.45.1.132/30", + "200.45.1.136/29", + "200.45.1.144/31", + "200.45.1.146/32", + "200.45.190.252/32", + "200.45.190.254/32", + "200.45.2.29/32", + "200.45.48.16/30", + "200.45.48.2/31", + "200.45.48.20/31", + "200.45.48.22/32", + "200.45.48.4/30", + "200.45.48.8/29", + "200.45.5.53/32", + "200.45.5.54/32", + "200.45.5.56/31", + "200.45.5.59/32", + "200.45.5.60/32", + "200.47.123.3/32", + "200.57.141.0/24", + "200.57.160.144/29", + "200.63.129.139/32", + "200.63.160.30/32", + "200.63.160.43/32", + "200.70.35.231/32", + "200.98.217.0/24", + "2001:1978:1103:6::/64", + "2001:4860:4000::/36", + "2001:558:fd00:56::/64", + "2001:558:fd01:2bb4::/64", + "2001:558:fd02:2446::/64", + "2001:558:fe16:19:96:114:154:0/112", + "2001:558:fe21:29:69:252:207:0/112", + "201.7.176.0/20", + "201.76.51.240/28", + "201.76.63.0/24", + "202.106.182.0/24", + "202.106.187.128/26", + "202.108.248.0/24", + "202.108.3.0/24", + "202.108.5.0/24", + "202.108.6.40/32", + "202.108.7.0/24", + "202.130.139.21/32", + "202.131.24.128/26", + "202.131.27.0/24", + "202.179.177.0/24", + "202.93.77.0/24", + "202.93.78.0/24", + "202.95.98.237/32", + "203.104.170.0/23", + "203.133.180.0/23", + "203.166.79.21/32", + "203.166.79.23/32", + "203.166.79.24/32", + "203.188.197.0/24", + "203.188.200.0/24", + "203.188.201.12/30", + "203.205.140.128/25", + "203.205.146.128/25", + "203.205.160.0/24", + "203.205.176.0/24", + "203.205.195.0/24", + "203.205.210.0/25", + "203.205.221.0/24", + "203.205.249.0/24", + "203.205.250.0/23", + "203.216.234.0/24", + "203.216.240.0/24", + "203.216.249.0/26", + "203.226.253.0/24", + "203.226.255.0/24", + "204.12.69.0/25", + "204.141.32.0/23", + "204.141.42.0/23", + "204.153.120.0/23", + "204.187.87.0/24", + "204.209.208.144/28", + "204.209.208.40/31", + "204.29.186.0/23", + "204.93.64.116/30", + "204.93.80.116/30", + "204.93.95.57/32", + "205.201.128.0/20", + "206.127.27.11/32", + "206.17.168.28/32", + "206.46.0.0/16", + "206.47.72.0/24", + "206.51.26.0/24", + "206.53.144.0/20", + "207.46.116.128/29", + "207.46.117.0/24", + "207.46.132.128/27", + "207.46.198.0/25", + "207.46.200.0/27", + "207.46.4.128/25", + "207.46.50.192/26", + "207.46.58.128/25", + "207.68.169.172/30", + "207.68.176.0/26", + "207.68.176.96/27", + "207.69.189.0/24", + "207.69.195.0/24", + "207.69.200.0/24", + "208.117.48.0/20", + "208.71.40.0/24", + "208.71.42.0/24", + "208.74.204.0/22", + "208.75.120.0/22", + "208.84.242.0/23", + "208.84.244.136/32", + "208.84.244.49/32", + "208.89.13.209/32", + "208.89.14.202/32", + "208.89.14.209/32", + "209.13.119.77/32", + "209.191.107.0/24", + "209.225.28.0/24", + "209.225.29.51/32", + "209.225.8.0/24", + "209.61.151.0/24", + "209.67.128.0/23", + "209.71.208.0/24", + "209.71.212.0/24", + "209.73.179.0/24", + "209.85.128.0/17", + "209.86.61.0/24", + "209.86.89.0/24", + "210.89.163.112/32", + "210.89.173.104/29", + "211.110.62.0/23", + "211.188.224.0/24", + "211.218.150.0/23", + "211.218.152.0/24", + "211.231.106.0/24", + "211.249.44.192/26", + "211.249.45.192/26", + "211.43.197.0/24", + "212.140.56.164/32", + "212.140.59.179/32", + "212.140.95.0/24", + "212.216.176.0/24", + "212.227.126.128/25", + "212.227.15.0/24", + "212.227.17.0/27", + "212.27.42.1/32", + "212.27.42.10/32", + "212.27.42.2/31", + "212.27.42.4/31", + "212.27.42.6/32", + "212.27.42.9/32", + "212.29.227.76/32", + "212.29.227.81/32", + "212.29.227.82/31", + "212.29.227.84/32", + "212.29.227.86/32", + "212.4.136.0/26", + "212.49.128.65/32", + "212.54.57.64/26", + "212.54.59.64/26", + "212.82.96.0/24", + "212.82.98.0/24", + "212.97.34.0/25", + "213.120.69.0/24", + "213.121.32.100/31", + "213.121.32.92/30", + "213.121.32.96/30", + "213.121.35.71/32", + "213.121.35.72/29", + "213.121.35.80/32", + "213.165.64.0/23", + "213.180.223.192/26", + "213.181.44.0/23", + "213.199.161.128/27", + "213.199.177.0/26", + "213.205.33.0/24", + "213.205.37.169/32", + "213.205.37.171/32", + "213.205.37.172/32", + "213.205.37.192/27", + "213.209.17.209/32", + "213.209.17.246/32", + "213.209.6.249/32", + "213.209.8.0/21", + "216.109.114.0/24", + "216.163.176.38/32", + "216.163.188.0/24", + "216.198.0.0/18", + "216.239.32.0/19", + "216.252.111.0/24", + "216.33.127.0/24", + "216.39.60.0/23", + "216.39.62.0/24", + "216.58.192.0/19", + "216.9.240.0/20", + "217.69.128.0/20", + "217.72.192.248/29", + "217.72.192.64/26", + "217.72.207.0/27", + "218.107.63.0/24", + "218.145.30.0/24", + "218.213.85.192/26", + "218.30.114.0/23", + "219.142.78.0/24", + "220.181.12.0/22", + "220.181.31.0/24", + "220.181.72.0/24", + "220.181.97.0/24", + "220.194.24.0/24", + "220.249.245.0/24", + "220.73.156.0/24", + "220.95.234.208/32", + "220.95.237.0/24", + "221.179.193.32/31", + "221.179.193.35/32", + "221.179.193.37/32", + "221.179.193.38/32", + "221.179.193.45/32", + "221.5.100.192/28", + "222.122.16.0/24", + "222.122.192.0/24", + "222.231.34.38/32", + "223.165.113.0/24", + "223.165.115.0/24", + "223.165.118.0/23", + "223.165.120.0/23", + "223.252.206.0/24", + "223.252.213.0/24", + "23.103.131.7/32", + "23.249.208.0/20", + "23.251.224.0/19", + "23.253.182.0/23", + "23.83.208.0/20", + "24.232.0.0/24", + "24.41.67.0/24", + "2404:6800:4000::/36", + "2407:ae80:100:1000::/63", + "2407:ae80:100:1002::/64", + "2407:ae80:300:1000::/63", + "2606:4700::6811:e138/128", + "2606:4700::6811:e238/128", + "2607:5300:203:fe1::/112", + "2607:f8b0:4000::/36", + "27.123.206.0/24", + "2800:3f0:4000::/36", + "2a00:1450:4000::/36", + "2a00:da80::/32", + "2a01:111:f400::/48", + "2a01:111:f403::/48", + "2a01:4180:4050:400::/64", + "2a01:4180:4050:800::/64", + "2a01:4180:4051:400::/64", + "2a01:4180:4051:800::/64", + "2a01:e0c:1:1599::10/126", + "2a01:e0c:1:1599::14/127", + "2a01:e0c:1:1599::18/127", + "2a02:1800:110:4::/64", + "2a02:1800:120:4::/64", + "2a02:2788:fff5:4::7/128", + "2a02:2788:fff5:4::8/128", + "2a02:598::/32", + "2a02:6b8:0:2519::/64", + "2a02:6b8::/51", + "2a02:6b8:b010:3046::/64", + "2a02:6b8:b010:7600::/56", + "2a02:6b8:b011:900::/56", + "2a02:6b8:b030:1000::/64", + "2a02:6b8:b030::/64", + "2c0f:fb50:4000::/36", + "3.210.190.0/24", + "3.93.157.0/24", + "3.96.81.40/32", + "3.97.99.32/31", + "3.97.99.42/31", + "34.195.253.204/30", + "34.195.253.208/31", + "34.219.98.66/32", + "35.154.184.19/32", + "35.158.20.192/32", + "35.164.127.234/31", + "35.164.127.236/30", + "35.173.142.173/32", + "35.190.247.0/24", + "35.191.0.0/16", + "36.51.253.0/24", + "37.140.128.0/18", + "37.9.109.0/24", + "38.147.122.66/32", + "40.107.0.0/16", + "40.112.65.63/32", + "40.92.0.0/14", + "42.120.70.0/23", + "42.159.163.81/32", + "42.159.163.82/31", + "46.19.168.0/23", + "46.228.39.0/24", + "46.232.183.0/24", + "46.255.224.0/21", + "46.4.238.128/29", + "47.43.16.0/23", + "47.43.20.0/24", + "47.88.44.32/27", + "5.100.171.0/28", + "5.255.192.0/18", + "5.45.192.0/19", + "5.45.254.0/25", + "5.61.236.0/24", + "5.61.237.0/26", + "5.61.237.128/25", + "5.61.239.143/32", + "5.61.239.144/32", + "50.31.32.0/19", + "51.4.71.62/32", + "51.4.72.0/24", + "51.4.80.0/27", + "51.5.72.0/24", + "51.5.80.0/27", + "51.77.79.158/32", + "51.81.229.180/30", + "51.81.232.218/32", + "51.81.232.240/30", + "51.81.232.244/31", + "51.81.242.150/32", + "51.81.242.153/32", + "51.81.242.66/32", + "51.81.57.58/31", + "51.81.57.60/31", + "51.81.61.60/30", + "51.81.61.64/29", + "51.89.119.103/32", + "52.100.0.0/14", + "52.17.152.5/32", + "52.19.67.100/32", + "52.198.69.159/32", + "52.198.94.238/32", + "52.205.10.60/32", + "52.221.130.187/32", + "52.237.141.173/32", + "52.251.55.143/32", + "52.31.100.154/32", + "52.34.31.144/32", + "52.36.127.248/32", + "52.52.2.81/32", + "52.56.66.10/32", + "52.59.177.22/32", + "52.62.108.212/32", + "52.62.123.207/32", + "52.74.103.30/32", + "52.74.147.196/32", + "52.79.190.38/32", + "52.8.239.192/32", + "52.8.52.154/32", + "54.174.52.0/24", + "54.174.53.128/30", + "54.174.57.0/24", + "54.174.59.0/24", + "54.174.60.0/23", + "54.174.63.0/24", + "54.179.177.220/32", + "54.204.34.129/32", + "54.204.34.130/32", + "54.206.16.166/32", + "54.206.34.216/32", + "54.207.19.206/32", + "54.207.22.56/32", + "54.208.23.82/32", + "54.208.27.227/32", + "54.214.232.113/32", + "54.219.167.112/32", + "54.236.213.237/32", + "54.238.142.250/32", + "54.240.0.0/17", + "54.243.244.52/32", + "54.245.125.39/32", + "54.254.200.128/32", + "54.254.200.92/32", + "54.36.149.183/32", + "54.38.221.122/32", + "54.65.118.226/32", + "54.65.248.153/32", + "54.92.39.34/32", + "58.246.222.128/25", + "58.248.244.64/26", + "58.250.132.0/24", + "58.250.134.0/24", + "58.250.143.128/25", + "58.63.234.0/24", + "59.111.176.0/24", + "59.111.178.0/24", + "59.36.102.0/24", + "59.36.128.0/23", + "59.36.132.0/24", + "59.78.209.0/24", + "59.82.0.0/24", + "60.191.83.0/24", + "60.28.175.192/26", + "60.28.2.0/24", + "60.28.228.0/24", + "61.247.193.71/32", + "61.247.193.72/29", + "61.247.193.80/32", + "61.247.200.0/23", + "61.74.70.0/23", + "62.191.128.210/32", + "62.216.254.4/32", + "62.239.224.234/31", + "62.239.224.236/31", + "62.239.224.98/31", + "62.24.128.0/24", + "62.24.135.0/24", + "62.24.202.0/24", + "62.24.206.137/32", + "62.24.206.138/32", + "62.24.229.0/26", + "62.241.16.251/32", + "62.241.4.11/32", + "62.241.4.118/32", + "62.241.4.12/32", + "62.241.4.129/32", + "62.241.4.153/32", + "62.241.4.155/32", + "62.241.4.164/32", + "62.241.4.185/32", + "62.241.4.25/32", + "62.241.5.126/32", + "62.241.5.16/32", + "62.241.5.253/32", + "62.241.5.74/31", + "62.241.5.78/32", + "62.241.5.95/32", + "62.7.242.135/32", + "62.7.242.136/29", + "62.8.100.195/32", + "63.27.6.0/24", + "64.135.77.0/24", + "64.135.83.0/24", + "64.136.0.0/20", + "64.136.16.0/21", + "64.136.28.0/22", + "64.136.32.0/20", + "64.136.50.0/23", + "64.136.52.0/22", + "64.147.108.0/24", + "64.147.123.17/32", + "64.147.123.18/31", + "64.147.123.20/31", + "64.147.123.24/30", + "64.147.123.28/31", + "64.181.217.128/27", + "64.209.227.0/24", + "64.210.232.0/24", + "64.233.160.0/19", + "64.34.47.128/27", + "64.34.57.192/26", + "64.59.134.0/25", + "64.59.136.128/27", + "64.69.70.192/28", + "65.154.166.0/24", + "65.20.0.0/24", + "65.39.178.0/24", + "65.39.215.0/24", + "65.54.121.120/29", + "65.54.190.0/24", + "65.54.241.0/24", + "65.54.51.64/26", + "65.54.61.64/26", + "65.55.111.0/24", + "65.55.113.64/26", + "65.55.116.0/25", + "65.55.126.0/25", + "65.55.174.0/25", + "65.55.178.128/27", + "65.55.234.192/26", + "65.55.238.128/26", + "65.55.33.64/28", + "65.55.34.0/24", + "65.55.52.224/27", + "65.55.78.128/25", + "65.55.81.48/28", + "65.55.90.0/24", + "65.55.94.0/25", + "66.102.0.0/20", + "66.111.4.221/32", + "66.111.4.222/32", + "66.111.4.224/31", + "66.111.4.229/32", + "66.111.4.230/32", + "66.111.4.25/32", + "66.111.4.26/31", + "66.111.4.28/31", + "66.163.184.0/21", + "66.174.0.0/16", + "66.196.81.0/24", + "66.220.144.128/25", + "66.220.155.0/24", + "66.220.157.0/25", + "66.249.80.0/20", + "67.195.22.0/24", + "67.195.60.0/24", + "67.195.87.0/24", + "67.223.64.0/19", + "68.114.188.0/22", + "68.142.203.0/24", + "68.142.206.0/24", + "68.142.230.0/24", + "68.171.224.0/19", + "68.230.240.0/23", + "69.147.103.0/24", + "69.169.224.0/20", + "69.171.232.0/24", + "69.171.244.0/23", + "69.252.207.0/25", + "69.63.178.128/25", + "69.63.181.0/24", + "69.63.184.0/25", + "69.72.32.0/20", + "69.78.0.0/16", + "70.37.151.128/25", + "72.14.192.0/18", + "72.30.234.152/29", + "72.30.238.0/23", + "72.51.60.130/31", + "72.51.60.132/31", + "72.51.60.134/32", + "72.51.60.139/32", + "74.125.0.0/16", + "74.202.142.0/24", + "74.208.122.0/26", + "74.208.4.192/26", + "74.208.5.64/26", + "74.6.128.0/21", + "74.82.64.0/19", + "76.223.176.0/20", + "77.238.176.0/22", + "77.75.76.0/22", + "77.88.0.0/18", + "77.93.251.192/26", + "78.144.6.128/26", + "78.33.8.111/32", + "8.39.54.0/23", + "8.40.222.0/23", + "80.0.253.64/28", + "81.104.62.32/28", + "81.19.78.0/27", + "81.19.78.96/27", + "81.19.88.0/24", + "81.19.92.32/27", + "81.3.6.160/28", + "82.165.159.0/24", + "82.165.229.130/32", + "82.165.229.31/32", + "82.165.230.21/32", + "82.165.230.22/32", + "82.57.200.0/24", + "84.96.92.0/24", + "86.64.210.153/32", + "86.64.210.154/32", + "86.64.210.19/32", + "86.64.210.3/32", + "86.64.210.5/32", + "86.64.210.79/32", + "87.248.110.0/24", + "87.250.224.0/19", + "87.253.232.0/21", + "91.134.188.129/32", + "93.158.136.48/28", + "93.17.128.0/24", + "93.94.32.0/22", + "93.94.37.0/24", + "93.94.38.0/24", + "94.100.176.0/20", + "94.125.164.65/32", + "94.245.112.0/27", + "95.108.130.0/23", + "95.108.192.0/18", + "95.163.216.38/31", + "95.163.40.8/29", + "96.102.19.32/28", + "96.102.200.0/28", + "96.103.146.48/28", + "96.114.154.128/25", + "98.136.164.0/24", + "98.136.185.0/24", + "98.136.217.0/24", + "98.136.218.0/24", + "98.137.12.0/24", + "98.137.176.58/31", + "98.137.177.247/32", + "98.137.64.0/21", + "98.138.100.0/23", + "98.138.120.0/24", + "98.138.215.0/24", + "98.138.226.0/24", + "98.138.229.0/24", + "98.138.84.0/22", + "98.138.88.0/22", + "98.139.164.0/24", + "98.139.175.0/24", + "98.139.211.0/24", + "98.139.212.0/23", + "98.139.215.0/24", + "98.139.221.0/24", + "98.139.244.0/24", + "98.142.233.64/28", + "98.142.235.128/26", + "fd18:d113:9bed::/48" + ], + "matching_attributes": [ + "ip-src", + "ip-dst", + "domain|ip" + ], + "name": "List of known SMTP sending IP ranges", + "type": "cidr", + "version": 20210613 +} diff --git a/tools/generate-crl-ip-domains.py b/tools/generate-crl-ip-domains.py index 1466742..8cced12 100755 --- a/tools/generate-crl-ip-domains.py +++ b/tools/generate-crl-ip-domains.py @@ -7,9 +7,9 @@ from OpenSSL.crypto import FILETYPE_PEM, load_certificate, X509 from pyasn1.codec.der.decoder import decode as asn1_decoder from pyasn1_modules.rfc2459 import CRLDistPointsSyntax, AuthorityInfoAccessSyntax from typing import List, Set -from dns.resolver import Resolver, NoAnswer, NXDOMAIN +from dns.resolver import NoAnswer, NXDOMAIN from dns.exception import Timeout -from generator import download_to_file, get_version, write_to_file, get_abspath_source_file +from generator import download_to_file, get_version, write_to_file, get_abspath_source_file, create_resolver def get_domain(url: str) -> str: @@ -44,9 +44,7 @@ def get_crl_ocsp_domains(cert: X509) -> List[str]: def get_ips_from_domain(domain: str) -> Set[str]: - resolver = Resolver() - resolver.timeout = 5 - resolver.lifetime = 5 + resolver = create_resolver() ips = set() @@ -65,10 +63,6 @@ def get_ips_from_domain(domain: str) -> Set[str]: def get_ips_from_domains(domains) -> Set[str]: - resolver = Resolver() - resolver.timeout = 5 - resolver.lifetime = 5 - p = multiprocessing.dummy.Pool(10) ips = set() for ips_for_domain in p.map(get_ips_from_domain, domains): diff --git a/tools/generate-google-gmail-sending-ips.py b/tools/generate-google-gmail-sending-ips.py index 46778d8..29a2a5a 100644 --- a/tools/generate-google-gmail-sending-ips.py +++ b/tools/generate-google-gmail-sending-ips.py @@ -1,48 +1,17 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -from ipaddress import ip_network, IPv4Network, IPv6Network -from dns.resolver import Resolver -from typing import List, Union -from generator import get_version, write_to_file - - -class Spf: - def _parse_spf(self, spf: str) -> dict: - output = {"include": [], "ranges": []} - for part in spf.split(" "): - if part.startswith("include:"): - output["include"].append(part.split(":", 1)[1]) - elif part.startswith("ip4:") or part.startswith("ip6:"): - output["ranges"].append(ip_network(part.split(":", 1)[1])) - return output - - def _query_spf(self, resolver: Resolver, domain: str) -> List[Union[IPv4Network, IPv6Network]]: - ranges = [] - for rdata in resolver.query(domain, "TXT"): - parsed = self._parse_spf(rdata.to_text()) - ranges += parsed["ranges"] - - for include in parsed["include"]: - ranges += self._query_spf(resolver, include) - - return ranges - - def get_list(self, domain: str) -> List[Union[IPv4Network, IPv6Network]]: - resolver = Resolver() - return self._query_spf(resolver, domain) +from generator import get_version, write_to_file, Spf, consolidate_networks, create_resolver if __name__ == '__main__': - spf = Spf() - print() - + spf = Spf(create_resolver()) warninglist = { 'name': "List of known Gmail sending IP ranges", 'version': get_version(), 'description': "List of known Gmail sending IP ranges (https://support.google.com/a/answer/27642?hl=en)", 'matching_attributes': ["ip-src", "ip-dst", "domain|ip"], 'type': 'cidr', - 'list': [str(range) for range in spf.get_list("_spf.google.com")], + 'list': consolidate_networks(spf.get_ip_ranges("gmail.com")), } write_to_file(warninglist, "google-gmail-sending-ips") diff --git a/tools/generate-smtp-sending-ips.py b/tools/generate-smtp-sending-ips.py new file mode 100644 index 0000000..1bb7690 --- /dev/null +++ b/tools/generate-smtp-sending-ips.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +import multiprocessing.dummy +from generator import get_version, write_to_file, Spf, consolidate_networks, create_resolver + +# Source: https://github.com/mailcheck/mailcheck/wiki/List-of-Popular-Domains +domains = [ + # Default domains included + "aol.com", "att.net", "comcast.net", "facebook.com", "gmail.com", "gmx.com", "googlemail.com", + "google.com", "hotmail.com", "hotmail.co.uk", "mac.com", "me.com", "mail.com", "msn.com", + "live.com", "sbcglobal.net", "verizon.net", "yahoo.com", "yahoo.co.uk", + + # Other global domains + "email.com", "fastmail.fm", "games.com", "gmx.net", "hush.com", "hushmail.com", "icloud.com", + "iname.com", "inbox.com", "lavabit.com", + "love.com", "outlook.com", "pobox.com", "protonmail.ch", "protonmail.com", "tutanota.de", "tutanota.com", + "tutamail.com", "tuta.io", + "keemail.me", "rocketmail.com", "safe-mail.net", "wow.com", "ygm.com", + "ymail.com", "zoho.com", "yandex.com", + + # United States ISP domains + "bellsouth.net", "charter.net", "cox.net", "earthlink.net", "juno.com", + + # British ISP domains + "btinternet.com", "virginmedia.com", "blueyonder.co.uk", "live.co.uk", + "ntlworld.com", "orange.net", "sky.com", "talktalk.co.uk", "tiscali.co.uk", + "virgin.net", "bt.com", + + # Domains used in Asia + "sina.com", "sina.cn", "qq.com", "naver.com", "hanmail.net", "daum.net", "nate.com", "yahoo.co.jp", "yahoo.co.kr", + "yahoo.co.id", "yahoo.co.in", "yahoo.com.sg", "yahoo.com.ph", "163.com", "yeah.net", "126.com", "21cn.com", + "aliyun.com", "foxmail.com", + + # French ISP domains + "hotmail.fr", "live.fr", "laposte.net", "yahoo.fr", "wanadoo.fr", "orange.fr", "gmx.fr", "sfr.fr", "neuf.fr", + "free.fr", + + # German ISP domains + "gmx.de", "hotmail.de", "live.de", "online.de", "t-online.de", "web.de", "yahoo.de", + + # Italian ISP domains + "libero.it", "virgilio.it", "hotmail.it", "aol.it", "tiscali.it", + "alice.it", "live.it", "yahoo.it", "email.it", "tin.it", "poste.it", "teletu.it", + + # Russian ISP domains + "bk.ru", "inbox.ru", "list.ru", "mail.ru", "rambler.ru", "yandex.by", "yandex.com", "yandex.kz", "yandex.ru", + "yandex.ua", "ya.ru", + + # Belgian ISP domains + "hotmail.be", "live.be", "skynet.be", "voo.be", "tvcablenet.be", "telenet.be", + + # Argentinian ISP domains + "hotmail.com.ar", "live.com.ar", "yahoo.com.ar", "fibertel.com.ar", "speedy.com.ar", "arnet.com.ar", + + # Domains used in Mexico + "yahoo.com.mx", "live.com.mx", "hotmail.es", "hotmail.com.mx", "prodigy.net.mx", + + # Domains used in Canada + "yahoo.ca", "hotmail.ca", "bell.net", "shaw.ca", "sympatico.ca", "rogers.com", + + # Domains used in Brazil + "yahoo.com.br", "hotmail.com.br", "outlook.com.br", "uol.com.br", "bol.com.br", "terra.com.br", "ig.com.br", + "r7.com", "zipmail.com.br", "globo.com", "globomail.com", "oi.com.br", + + # Custom extension + # Domains used in Czechia + "seznam.cz", "atlas.cz", "centrum.cz", +] + + +if __name__ == '__main__': + spf = Spf(create_resolver()) + + ranges = [] + p = multiprocessing.dummy.Pool(20) + for domain_ranges in p.map(lambda d: spf.get_ip_ranges(d), domains): + ranges.extend(domain_ranges) + + warninglist = { + 'name': "List of known SMTP sending IP ranges", + 'version': get_version(), + 'description': "List of IP ranges for known SMTP servers-", + 'matching_attributes': ["ip-src", "ip-dst", "domain|ip"], + 'type': 'cidr', + 'list': consolidate_networks(ranges), + } + + write_to_file(warninglist, "smtp-sending-ips") diff --git a/tools/generator.py b/tools/generator.py index 51c4a46..e7bdd28 100644 --- a/tools/generator.py +++ b/tools/generator.py @@ -1,14 +1,14 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - import datetime import ipaddress import json import logging from inspect import currentframe, getframeinfo, getmodulename, stack from os import mkdir, path +from typing import List, Union import requests +import dns.exception +import dns.resolver from dateutil.parser import parse as parsedate @@ -137,23 +137,111 @@ def write_to_file(warninglist, dst): def consolidate_networks(networks): - # Convert to IpNetwork + # Split to IPv4 and IPv6 ranges ipv4_networks = [] ipv6_networks = [] for network in networks: - network = ipaddress.ip_network(network) + if isinstance(network, str): + # Convert string to IpNetwork + network = ipaddress.ip_network(network) + if network.version == 4: ipv4_networks.append(network) else: ipv6_networks.append(network) - # Collapse + # Collapse ranges networks_to_keep = list(map(str, ipaddress.collapse_addresses(ipv4_networks))) networks_to_keep.extend(map(str, ipaddress.collapse_addresses(ipv6_networks))) return networks_to_keep +def create_resolver() -> dns.resolver.Resolver: + resolver = dns.resolver.Resolver(configure=False) + resolver.timeout = 30 + resolver.lifetime = 30 + resolver.cache = dns.resolver.LRUCache() + resolver.nameservers = ["193.17.47.1", "185.43.135.1"] # CZ.NIC nameservers + return resolver + + +class Spf: + def __init__(self, resolver: dns.resolver.Resolver): + self.__resolver = resolver + + def _parse_spf(self, domain: str, spf: str) -> dict: + output = {"include": [], "ranges": [], "a": [], "mx": []} + for part in spf.split(" "): + if part.startswith("include:"): + output["include"].append(part.split(":", 1)[1]) + elif part.startswith("redirect="): + output["include"].append(part.split("=", 1)[1]) + elif part == "a": + output["a"].append(domain) + elif part.startswith("a:"): + output["a"].append(part.split(":", 1)[1]) + elif part == "mx": + output["mx"].append(domain) + elif part.startswith("mx:"): + output["mx"].append(part.split(":", 1)[1]) + elif part.startswith("ip4:") or part.startswith("ip6:"): + output["ranges"].append(ipaddress.ip_network(part.split(":", 1)[1], strict=False)) + return output + + def _get_ip_for_domain(self, domain: str) -> List[Union[ipaddress.IPv4Network, ipaddress.IPv6Network]]: + ranges = [] + try: + for ip in self.__resolver.query(domain, "a"): + ranges.append(ipaddress.ip_network(str(ip))) + except (dns.resolver.NoAnswer, dns.resolver.NXDOMAIN, dns.exception.Timeout): + pass + + try: + for ip in self.__resolver.query(domain, "aaaa"): + ranges.append(ipaddress.ip_network(str(ip))) + except (dns.resolver.NoAnswer, dns.resolver.NXDOMAIN, dns.exception.Timeout): + pass + + return ranges + + def _get_mx_ips_for_domain(self, domain: str) -> List[Union[ipaddress.IPv4Network, ipaddress.IPv6Network]]: + ranges = [] + try: + for rdata in self.__resolver.query(domain, "mx"): + ranges += self._get_ip_for_domain(rdata.exchange) + except (dns.resolver.NoAnswer, dns.resolver.NXDOMAIN, dns.exception.Timeout): + pass + return ranges + + def get_ip_ranges(self, domain: str) -> List[Union[ipaddress.IPv4Network, ipaddress.IPv6Network]]: + try: + txt_records = self.__resolver.query(domain, "TXT") + except (dns.resolver.NoAnswer, dns.resolver.NXDOMAIN, dns.exception.Timeout) as e: + logging.info("Could not fetch TXT record for domain {}: {}".format(domain, str(e))) + return [] + + ranges = [] + for rdata in txt_records: + record = "".join([s.decode("utf-8") for s in rdata.strings]) + if not record.startswith("v=spf1"): + continue + + parsed = self._parse_spf(domain, record) + ranges += parsed["ranges"] + + for include in parsed["include"]: + ranges += self.get_ip_ranges(include) + + for domain in parsed["a"]: + ranges += self._get_ip_for_domain(domain) + + for mx in parsed["mx"]: + ranges += self._get_mx_ips_for_domain(mx) + + return ranges + + def main(): init_logging()