From 00d5e40ac14c19bb0a5eb74ed98171d61a7fa132 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Mon, 14 Jun 2021 23:26:44 +0200 Subject: [PATCH] new: List of known SMTP receiving IP addresses --- generate_all.sh | 2 +- lists/smtp-receiving-ips/list.json | 349 ++++++++++++++++++ tools/generate-google-gmail-sending-ips.py | 6 +- ...e-smtp-sending-ips.py => generate-smtp.py} | 31 +- tools/generator.py | 28 +- 5 files changed, 389 insertions(+), 27 deletions(-) create mode 100644 lists/smtp-receiving-ips/list.json rename tools/{generate-smtp-sending-ips.py => generate-smtp.py} (79%) diff --git a/generate_all.sh b/generate_all.sh index 6628d07..c946b8b 100755 --- a/generate_all.sh +++ b/generate_all.sh @@ -30,7 +30,7 @@ python3 generate-wikimedia.py python3 generate-second-level-tlds.py python3 generate-google-gcp.py python3 generate-google-gmail-sending-ips.py -python3 generate-smtp-sending-ips.py +python3 generate-smtp.py popd ./jq_all_the_things.sh diff --git a/lists/smtp-receiving-ips/list.json b/lists/smtp-receiving-ips/list.json new file mode 100644 index 0000000..969a029 --- /dev/null +++ b/lists/smtp-receiving-ips/list.json @@ -0,0 +1,349 @@ +{ + "description": "List of IP addresses for known SMTP servers.", + "list": [ + "104.47.17.161", + "104.47.17.97", + "104.47.18.225", + "104.47.18.97", + "104.47.22.161", + "104.47.56.161", + "104.47.57.161", + "104.47.58.161", + "104.47.73.161", + "104.47.73.33", + "106.10.248.73", + "106.10.248.74", + "108.177.119.27", + "117.53.116.15", + "121.14.32.117", + "123.126.45.192", + "123.58.177.213", + "123.58.177.214", + "123.58.177.220", + "124.83.142.118", + "124.83.142.119", + "124.83.142.123", + "124.83.142.124", + "124.83.142.247", + "124.83.142.248", + "124.83.142.249", + "124.83.142.250", + "125.209.222.14", + "125.209.238.100", + "125.209.238.137", + "140.205.94.14", + "142.250.150.26", + "142.250.150.27", + "142.250.153.26", + "142.250.157.26", + "142.250.157.27", + "144.160.159.21", + "144.160.159.22", + "144.160.235.143", + "144.160.235.144", + "15.222.199.59", + "160.92.124.66", + "162.62.116.184", + "17.42.251.10", + "17.42.251.12", + "17.56.9.17", + "17.56.9.19", + "17.57.152.14", + "17.57.152.9", + "17.57.154.23", + "17.57.154.6", + "17.57.154.7", + "173.194.202.26", + "173.194.202.27", + "173.228.157.39", + "173.228.157.40", + "173.228.157.41", + "173.228.157.42", + "176.32.102.11", + "177.153.23.241", + "177.153.23.242", + "177.154.139.187", + "18.209.118.139", + "18.237.196.55", + "18.237.235.220", + "183.61.185.83", + "185.3.161.12", + "185.70.40.102", + "185.70.41.101", + "186.202.4.42", + "188.125.72.73", + "188.125.72.74", + "193.252.22.65", + "194.25.134.72", + "194.25.134.73", + "194.25.134.8", + "194.25.134.9", + "195.130.132.8", + "195.130.132.9", + "195.238.20.25", + "195.24.76.171", + "198.58.121.58", + "200.147.36.13", + "200.147.36.15", + "200.234.204.130", + "200.45.191.213", + "200.45.191.214", + "2001:558:fe16:1b::15", + "2001:67c:15a1::ca1", + "2001:67c:15a2::cb1", + "2001:67c:15a3::cd1", + "202.93.66.247", + "202.93.66.248", + "202.93.66.249", + "202.93.66.250", + "202.93.77.238", + "202.93.77.239", + "202.93.77.240", + "202.93.77.241", + "202.93.78.239", + "202.93.78.240", + "202.93.78.241", + "202.93.78.242", + "203.205.219.57", + "203.205.219.58", + "208.84.244.133", + "208.84.244.136", + "209.71.208.7", + "209.71.212.24", + "211.231.108.175", + "211.231.108.176", + "211.231.108.46", + "211.231.108.47", + "212.227.15.10", + "212.227.15.17", + "212.227.15.40", + "212.227.15.9", + "212.227.17.4", + "212.227.17.5", + "212.227.17.8", + "212.27.42.58", + "212.27.42.59", + "212.27.48.6", + "212.27.48.7", + "212.29.227.81", + "212.29.227.82", + "212.29.227.83", + "212.29.227.84", + "212.29.227.86", + "212.54.56.11", + "213.120.69.5", + "213.121.32.100", + "213.121.32.101", + "213.121.32.92", + "213.121.32.93", + "213.121.32.94", + "213.121.32.95", + "213.121.32.96", + "213.121.32.97", + "213.121.32.98", + "213.121.32.99", + "213.121.35.71", + "213.121.35.72", + "213.121.35.73", + "213.121.35.74", + "213.121.35.75", + "213.121.35.76", + "213.121.35.77", + "213.121.35.78", + "213.121.35.79", + "213.121.35.80", + "213.205.33.244", + "213.205.33.61", + "213.205.33.62", + "213.205.33.63", + "213.205.33.64", + "213.209.1.129", + "213.209.1.130", + "216.163.176.38", + "216.163.188.54", + "217.72.192.66", + "220.181.14.135", + "220.181.14.136", + "220.181.14.137", + "220.181.14.138", + "220.181.14.139", + "220.181.14.140", + "220.181.14.141", + "220.181.14.142", + "220.181.14.143", + "220.181.14.144", + "220.181.14.145", + "220.181.14.146", + "220.181.14.147", + "220.181.14.148", + "220.181.14.149", + "220.181.14.150", + "220.181.14.154", + "220.181.14.155", + "220.181.14.156", + "220.181.14.157", + "220.181.14.158", + "220.181.14.159", + "220.181.14.160", + "220.181.14.161", + "220.181.14.162", + "220.181.14.163", + "220.181.14.164", + "220.181.15.131", + "220.181.15.132", + "220.181.15.134", + "220.181.15.135", + "220.181.15.136", + "220.181.15.137", + "220.181.15.138", + "220.181.15.140", + "220.181.15.141", + "220.181.15.142", + "220.181.15.143", + "220.181.15.144", + "220.181.15.145", + "220.181.15.147", + "220.181.15.151", + "220.181.15.152", + "220.181.15.153", + "220.181.15.154", + "220.181.15.155", + "220.181.15.191", + "220.181.15.192", + "220.181.15.193", + "220.181.15.194", + "220.181.15.195", + "220.181.15.196", + "220.181.15.197", + "220.181.15.198", + "220.181.15.199", + "220.181.15.200", + "220.181.15.202", + "220.181.15.203", + "220.181.15.206", + "220.181.15.207", + "220.181.15.209", + "24.232.0.226", + "2404:6800:4008:c13::1a", + "2404:6800:4008:c13::1b", + "2407:ae80:100:1000:123:58:178:178", + "240d:c040:1:40::133", + "2607:f8b0:400e:c00::1b", + "2a00:1450:4010:c1c::1b", + "2a00:1450:4013:c01::1a", + "2a00:1450:4013:c07::1b", + "2a02:598:2::32", + "2a02:598:2::42", + "2a02:598:a::78:32", + "2a02:598:a::78:42", + "2a02:6b8::311", + "2a03:2880:21ff:fffd:face:b00c:0:686e", + "3.96.81.40", + "31.7.241.25", + "31.7.241.26", + "34.212.80.54", + "34.220.245.67", + "34.222.93.91", + "34.223.6.127", + "35.162.106.154", + "38.147.122.66", + "39.156.6.104", + "5.180.49.35", + "5.180.49.37", + "51.81.242.150", + "51.81.242.59", + "51.81.57.58", + "51.81.57.59", + "52.73.137.222", + "54.187.110.113", + "54.190.26.211", + "54.200.93.251", + "54.218.2.65", + "54.244.49.115", + "62.149.176.151", + "62.211.72.32", + "62.24.139.14", + "62.24.139.15", + "62.24.139.42", + "62.24.202.82", + "62.24.202.83", + "62.241.4.132", + "64.135.83.10", + "64.135.83.90", + "64.136.44.37", + "64.136.52.37", + "64.147.108.50", + "64.147.108.51", + "64.147.108.55", + "64.147.123.51", + "64.147.123.52", + "66.111.4.70", + "66.111.4.71", + "66.111.4.72", + "66.111.4.73", + "66.111.4.74", + "66.111.4.75", + "66.220.149.251", + "67.195.204.72", + "67.195.204.73", + "67.195.204.74", + "67.195.204.75", + "67.195.204.77", + "67.195.204.79", + "67.195.204.80", + "67.195.204.82", + "67.195.228.106", + "67.195.228.109", + "67.195.228.110", + "67.195.228.111", + "67.195.228.74", + "67.195.228.84", + "67.195.228.86", + "67.195.228.94", + "68.87.20.5", + "72.51.60.130", + "72.51.60.131", + "72.51.60.132", + "72.51.60.133", + "72.51.60.134", + "74.125.200.26", + "74.125.200.27", + "74.208.5.20", + "74.208.5.22", + "77.75.76.32", + "77.75.76.42", + "77.75.78.32", + "77.75.78.42", + "77.88.21.249", + "77.93.251.249", + "80.12.242.9", + "81.19.78.64", + "81.19.78.65", + "81.19.78.66", + "81.19.78.67", + "81.3.6.162", + "81.3.6.165", + "82.57.200.133", + "93.17.128.123", + "93.17.128.165", + "94.100.180.104", + "94.100.180.31", + "94.242.232.66", + "96.114.157.80", + "98.136.96.74", + "98.136.96.75", + "98.136.96.76", + "98.136.96.77", + "98.136.96.91", + "98.136.96.92", + "98.136.96.93" + ], + "matching_attributes": [ + "ip-src", + "ip-dst", + "domain|ip" + ], + "name": "List of known SMTP addresses IP ranges", + "type": "cidr", + "version": 20210614 +} diff --git a/tools/generate-google-gmail-sending-ips.py b/tools/generate-google-gmail-sending-ips.py index 29a2a5a..8b44f32 100644 --- a/tools/generate-google-gmail-sending-ips.py +++ b/tools/generate-google-gmail-sending-ips.py @@ -1,17 +1,17 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -from generator import get_version, write_to_file, Spf, consolidate_networks, create_resolver +from generator import get_version, write_to_file, Dns, consolidate_networks, create_resolver if __name__ == '__main__': - spf = Spf(create_resolver()) + spf = Dns(create_resolver()) warninglist = { 'name': "List of known Gmail sending IP ranges", 'version': get_version(), 'description': "List of known Gmail sending IP ranges (https://support.google.com/a/answer/27642?hl=en)", 'matching_attributes': ["ip-src", "ip-dst", "domain|ip"], 'type': 'cidr', - 'list': consolidate_networks(spf.get_ip_ranges("gmail.com")), + 'list': consolidate_networks(spf.get_ip_ranges_from_spf("gmail.com")), } write_to_file(warninglist, "google-gmail-sending-ips") diff --git a/tools/generate-smtp-sending-ips.py b/tools/generate-smtp.py similarity index 79% rename from tools/generate-smtp-sending-ips.py rename to tools/generate-smtp.py index 1bb7690..bbab1e0 100644 --- a/tools/generate-smtp-sending-ips.py +++ b/tools/generate-smtp.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 import multiprocessing.dummy -from generator import get_version, write_to_file, Spf, consolidate_networks, create_resolver +from generator import get_version, write_to_file, Dns, consolidate_networks, create_resolver # Source: https://github.com/mailcheck/mailcheck/wiki/List-of-Popular-Domains domains = [ @@ -68,20 +68,33 @@ domains = [ if __name__ == '__main__': - spf = Spf(create_resolver()) + dns = Dns(create_resolver()) - ranges = [] - p = multiprocessing.dummy.Pool(20) - for domain_ranges in p.map(lambda d: spf.get_ip_ranges(d), domains): - ranges.extend(domain_ranges) + spf_ranges = [] + p = multiprocessing.dummy.Pool(40) + for domain_ranges in p.map(lambda d: dns.get_ip_ranges_from_spf(d), domains): + spf_ranges.extend(domain_ranges) warninglist = { 'name': "List of known SMTP sending IP ranges", 'version': get_version(), - 'description': "List of IP ranges for known SMTP servers-", + 'description': "List of IP ranges for known SMTP servers.", 'matching_attributes': ["ip-src", "ip-dst", "domain|ip"], 'type': 'cidr', - 'list': consolidate_networks(ranges), + 'list': consolidate_networks(spf_ranges), } - write_to_file(warninglist, "smtp-sending-ips") + + mx_ips = [] + for domain_ranges in p.map(lambda d: dns.get_mx_ips_for_domain(d), domains): + mx_ips.extend(domain_ranges) + + warninglist = { + 'name': "List of known SMTP receiving IP addresses", + 'version': get_version(), + 'description': "List of IP addresses for known SMTP servers.", + 'matching_attributes': ["ip-src", "ip-dst", "domain|ip"], + 'type': 'cidr', + 'list': map(str, mx_ips), + } + write_to_file(warninglist, "smtp-receiving-ips") diff --git a/tools/generator.py b/tools/generator.py index e7bdd28..86264d3 100644 --- a/tools/generator.py +++ b/tools/generator.py @@ -166,7 +166,7 @@ def create_resolver() -> dns.resolver.Resolver: return resolver -class Spf: +class Dns: def __init__(self, resolver: dns.resolver.Resolver): self.__resolver = resolver @@ -189,35 +189,35 @@ class Spf: output["ranges"].append(ipaddress.ip_network(part.split(":", 1)[1], strict=False)) return output - def _get_ip_for_domain(self, domain: str) -> List[Union[ipaddress.IPv4Network, ipaddress.IPv6Network]]: + def get_ip_for_domain(self, domain: str) -> List[Union[ipaddress.IPv4Address, ipaddress.IPv6Address]]: ranges = [] try: for ip in self.__resolver.query(domain, "a"): - ranges.append(ipaddress.ip_network(str(ip))) - except (dns.resolver.NoAnswer, dns.resolver.NXDOMAIN, dns.exception.Timeout): + ranges.append(ipaddress.IPv4Address(str(ip))) + except (dns.resolver.NoAnswer, dns.resolver.NXDOMAIN, dns.exception.Timeout, dns.resolver.NoNameservers): pass try: for ip in self.__resolver.query(domain, "aaaa"): - ranges.append(ipaddress.ip_network(str(ip))) - except (dns.resolver.NoAnswer, dns.resolver.NXDOMAIN, dns.exception.Timeout): + ranges.append(ipaddress.IPv6Address(str(ip))) + except (dns.resolver.NoAnswer, dns.resolver.NXDOMAIN, dns.exception.Timeout, dns.resolver.NoNameservers): pass return ranges - def _get_mx_ips_for_domain(self, domain: str) -> List[Union[ipaddress.IPv4Network, ipaddress.IPv6Network]]: + def get_mx_ips_for_domain(self, domain: str) -> List[Union[ipaddress.IPv4Address, ipaddress.IPv6Address]]: ranges = [] try: for rdata in self.__resolver.query(domain, "mx"): - ranges += self._get_ip_for_domain(rdata.exchange) - except (dns.resolver.NoAnswer, dns.resolver.NXDOMAIN, dns.exception.Timeout): + ranges += self.get_ip_for_domain(rdata.exchange) + except (dns.resolver.NoAnswer, dns.resolver.NXDOMAIN, dns.exception.Timeout, dns.resolver.NoNameservers): pass return ranges - def get_ip_ranges(self, domain: str) -> List[Union[ipaddress.IPv4Network, ipaddress.IPv6Network]]: + def get_ip_ranges_from_spf(self, domain: str) -> List[Union[ipaddress.IPv4Network, ipaddress.IPv6Network]]: try: txt_records = self.__resolver.query(domain, "TXT") - except (dns.resolver.NoAnswer, dns.resolver.NXDOMAIN, dns.exception.Timeout) as e: + except (dns.resolver.NoAnswer, dns.resolver.NXDOMAIN, dns.exception.Timeout, dns.resolver.NoNameservers) as e: logging.info("Could not fetch TXT record for domain {}: {}".format(domain, str(e))) return [] @@ -231,13 +231,13 @@ class Spf: ranges += parsed["ranges"] for include in parsed["include"]: - ranges += self.get_ip_ranges(include) + ranges += self.get_ip_ranges_from_spf(include) for domain in parsed["a"]: - ranges += self._get_ip_for_domain(domain) + ranges += map(ipaddress.ip_network, self.get_ip_for_domain(domain)) for mx in parsed["mx"]: - ranges += self._get_mx_ips_for_domain(mx) + ranges += map(ipaddress.ip_network, self.get_mx_ips_for_domain(mx)) return ranges