Merge pull request #190 from JakubOnderka/smtp
new: List of known SMTP sending IP rangespull/191/head v2.4.145
commit
07456b2eb4
|
@ -30,6 +30,7 @@ python3 generate-wikimedia.py
|
|||
python3 generate-second-level-tlds.py
|
||||
python3 generate-google-gcp.py
|
||||
python3 generate-google-gmail-sending-ips.py
|
||||
python3 generate-smtp.py
|
||||
popd
|
||||
|
||||
./jq_all_the_things.sh
|
||||
|
|
|
@ -0,0 +1,349 @@
|
|||
{
|
||||
"description": "List of IP addresses for known SMTP servers.",
|
||||
"list": [
|
||||
"104.47.17.161",
|
||||
"104.47.17.97",
|
||||
"104.47.18.225",
|
||||
"104.47.18.97",
|
||||
"104.47.22.161",
|
||||
"104.47.56.161",
|
||||
"104.47.57.161",
|
||||
"104.47.58.161",
|
||||
"104.47.73.161",
|
||||
"104.47.73.33",
|
||||
"106.10.248.73",
|
||||
"106.10.248.74",
|
||||
"108.177.119.27",
|
||||
"117.53.116.15",
|
||||
"121.14.32.117",
|
||||
"123.126.45.192",
|
||||
"123.58.177.213",
|
||||
"123.58.177.214",
|
||||
"123.58.177.220",
|
||||
"124.83.142.118",
|
||||
"124.83.142.119",
|
||||
"124.83.142.123",
|
||||
"124.83.142.124",
|
||||
"124.83.142.247",
|
||||
"124.83.142.248",
|
||||
"124.83.142.249",
|
||||
"124.83.142.250",
|
||||
"125.209.222.14",
|
||||
"125.209.238.100",
|
||||
"125.209.238.137",
|
||||
"140.205.94.14",
|
||||
"142.250.150.26",
|
||||
"142.250.150.27",
|
||||
"142.250.153.26",
|
||||
"142.250.157.26",
|
||||
"142.250.157.27",
|
||||
"144.160.159.21",
|
||||
"144.160.159.22",
|
||||
"144.160.235.143",
|
||||
"144.160.235.144",
|
||||
"15.222.199.59",
|
||||
"160.92.124.66",
|
||||
"162.62.116.184",
|
||||
"17.42.251.10",
|
||||
"17.42.251.12",
|
||||
"17.56.9.17",
|
||||
"17.56.9.19",
|
||||
"17.57.152.14",
|
||||
"17.57.152.9",
|
||||
"17.57.154.23",
|
||||
"17.57.154.6",
|
||||
"17.57.154.7",
|
||||
"173.194.202.26",
|
||||
"173.194.202.27",
|
||||
"173.228.157.39",
|
||||
"173.228.157.40",
|
||||
"173.228.157.41",
|
||||
"173.228.157.42",
|
||||
"176.32.102.11",
|
||||
"177.153.23.241",
|
||||
"177.153.23.242",
|
||||
"177.154.139.187",
|
||||
"18.209.118.139",
|
||||
"18.237.196.55",
|
||||
"18.237.235.220",
|
||||
"183.61.185.83",
|
||||
"185.3.161.12",
|
||||
"185.70.40.102",
|
||||
"185.70.41.101",
|
||||
"186.202.4.42",
|
||||
"188.125.72.73",
|
||||
"188.125.72.74",
|
||||
"193.252.22.65",
|
||||
"194.25.134.72",
|
||||
"194.25.134.73",
|
||||
"194.25.134.8",
|
||||
"194.25.134.9",
|
||||
"195.130.132.8",
|
||||
"195.130.132.9",
|
||||
"195.238.20.25",
|
||||
"195.24.76.171",
|
||||
"198.58.121.58",
|
||||
"200.147.36.13",
|
||||
"200.147.36.15",
|
||||
"200.234.204.130",
|
||||
"200.45.191.213",
|
||||
"200.45.191.214",
|
||||
"2001:558:fe16:1b::15",
|
||||
"2001:67c:15a1::ca1",
|
||||
"2001:67c:15a2::cb1",
|
||||
"2001:67c:15a3::cd1",
|
||||
"202.93.66.247",
|
||||
"202.93.66.248",
|
||||
"202.93.66.249",
|
||||
"202.93.66.250",
|
||||
"202.93.77.238",
|
||||
"202.93.77.239",
|
||||
"202.93.77.240",
|
||||
"202.93.77.241",
|
||||
"202.93.78.239",
|
||||
"202.93.78.240",
|
||||
"202.93.78.241",
|
||||
"202.93.78.242",
|
||||
"203.205.219.57",
|
||||
"203.205.219.58",
|
||||
"208.84.244.133",
|
||||
"208.84.244.136",
|
||||
"209.71.208.7",
|
||||
"209.71.212.24",
|
||||
"211.231.108.175",
|
||||
"211.231.108.176",
|
||||
"211.231.108.46",
|
||||
"211.231.108.47",
|
||||
"212.227.15.10",
|
||||
"212.227.15.17",
|
||||
"212.227.15.40",
|
||||
"212.227.15.9",
|
||||
"212.227.17.4",
|
||||
"212.227.17.5",
|
||||
"212.227.17.8",
|
||||
"212.27.42.58",
|
||||
"212.27.42.59",
|
||||
"212.27.48.6",
|
||||
"212.27.48.7",
|
||||
"212.29.227.81",
|
||||
"212.29.227.82",
|
||||
"212.29.227.83",
|
||||
"212.29.227.84",
|
||||
"212.29.227.86",
|
||||
"212.54.56.11",
|
||||
"213.120.69.5",
|
||||
"213.121.32.100",
|
||||
"213.121.32.101",
|
||||
"213.121.32.92",
|
||||
"213.121.32.93",
|
||||
"213.121.32.94",
|
||||
"213.121.32.95",
|
||||
"213.121.32.96",
|
||||
"213.121.32.97",
|
||||
"213.121.32.98",
|
||||
"213.121.32.99",
|
||||
"213.121.35.71",
|
||||
"213.121.35.72",
|
||||
"213.121.35.73",
|
||||
"213.121.35.74",
|
||||
"213.121.35.75",
|
||||
"213.121.35.76",
|
||||
"213.121.35.77",
|
||||
"213.121.35.78",
|
||||
"213.121.35.79",
|
||||
"213.121.35.80",
|
||||
"213.205.33.244",
|
||||
"213.205.33.61",
|
||||
"213.205.33.62",
|
||||
"213.205.33.63",
|
||||
"213.205.33.64",
|
||||
"213.209.1.129",
|
||||
"213.209.1.130",
|
||||
"216.163.176.38",
|
||||
"216.163.188.54",
|
||||
"217.72.192.66",
|
||||
"220.181.14.135",
|
||||
"220.181.14.136",
|
||||
"220.181.14.137",
|
||||
"220.181.14.138",
|
||||
"220.181.14.139",
|
||||
"220.181.14.140",
|
||||
"220.181.14.141",
|
||||
"220.181.14.142",
|
||||
"220.181.14.143",
|
||||
"220.181.14.144",
|
||||
"220.181.14.145",
|
||||
"220.181.14.146",
|
||||
"220.181.14.147",
|
||||
"220.181.14.148",
|
||||
"220.181.14.149",
|
||||
"220.181.14.150",
|
||||
"220.181.14.154",
|
||||
"220.181.14.155",
|
||||
"220.181.14.156",
|
||||
"220.181.14.157",
|
||||
"220.181.14.158",
|
||||
"220.181.14.159",
|
||||
"220.181.14.160",
|
||||
"220.181.14.161",
|
||||
"220.181.14.162",
|
||||
"220.181.14.163",
|
||||
"220.181.14.164",
|
||||
"220.181.15.131",
|
||||
"220.181.15.132",
|
||||
"220.181.15.134",
|
||||
"220.181.15.135",
|
||||
"220.181.15.136",
|
||||
"220.181.15.137",
|
||||
"220.181.15.138",
|
||||
"220.181.15.140",
|
||||
"220.181.15.141",
|
||||
"220.181.15.142",
|
||||
"220.181.15.143",
|
||||
"220.181.15.144",
|
||||
"220.181.15.145",
|
||||
"220.181.15.147",
|
||||
"220.181.15.151",
|
||||
"220.181.15.152",
|
||||
"220.181.15.153",
|
||||
"220.181.15.154",
|
||||
"220.181.15.155",
|
||||
"220.181.15.191",
|
||||
"220.181.15.192",
|
||||
"220.181.15.193",
|
||||
"220.181.15.194",
|
||||
"220.181.15.195",
|
||||
"220.181.15.196",
|
||||
"220.181.15.197",
|
||||
"220.181.15.198",
|
||||
"220.181.15.199",
|
||||
"220.181.15.200",
|
||||
"220.181.15.202",
|
||||
"220.181.15.203",
|
||||
"220.181.15.206",
|
||||
"220.181.15.207",
|
||||
"220.181.15.209",
|
||||
"24.232.0.226",
|
||||
"2404:6800:4008:c13::1a",
|
||||
"2404:6800:4008:c13::1b",
|
||||
"2407:ae80:100:1000:123:58:178:178",
|
||||
"240d:c040:1:40::133",
|
||||
"2607:f8b0:400e:c00::1b",
|
||||
"2a00:1450:4010:c1c::1b",
|
||||
"2a00:1450:4013:c01::1a",
|
||||
"2a00:1450:4013:c07::1b",
|
||||
"2a02:598:2::32",
|
||||
"2a02:598:2::42",
|
||||
"2a02:598:a::78:32",
|
||||
"2a02:598:a::78:42",
|
||||
"2a02:6b8::311",
|
||||
"2a03:2880:21ff:fffd:face:b00c:0:686e",
|
||||
"3.96.81.40",
|
||||
"31.7.241.25",
|
||||
"31.7.241.26",
|
||||
"34.212.80.54",
|
||||
"34.220.245.67",
|
||||
"34.222.93.91",
|
||||
"34.223.6.127",
|
||||
"35.162.106.154",
|
||||
"38.147.122.66",
|
||||
"39.156.6.104",
|
||||
"5.180.49.35",
|
||||
"5.180.49.37",
|
||||
"51.81.242.150",
|
||||
"51.81.242.59",
|
||||
"51.81.57.58",
|
||||
"51.81.57.59",
|
||||
"52.73.137.222",
|
||||
"54.187.110.113",
|
||||
"54.190.26.211",
|
||||
"54.200.93.251",
|
||||
"54.218.2.65",
|
||||
"54.244.49.115",
|
||||
"62.149.176.151",
|
||||
"62.211.72.32",
|
||||
"62.24.139.14",
|
||||
"62.24.139.15",
|
||||
"62.24.139.42",
|
||||
"62.24.202.82",
|
||||
"62.24.202.83",
|
||||
"62.241.4.132",
|
||||
"64.135.83.10",
|
||||
"64.135.83.90",
|
||||
"64.136.44.37",
|
||||
"64.136.52.37",
|
||||
"64.147.108.50",
|
||||
"64.147.108.51",
|
||||
"64.147.108.55",
|
||||
"64.147.123.51",
|
||||
"64.147.123.52",
|
||||
"66.111.4.70",
|
||||
"66.111.4.71",
|
||||
"66.111.4.72",
|
||||
"66.111.4.73",
|
||||
"66.111.4.74",
|
||||
"66.111.4.75",
|
||||
"66.220.149.251",
|
||||
"67.195.204.72",
|
||||
"67.195.204.73",
|
||||
"67.195.204.74",
|
||||
"67.195.204.75",
|
||||
"67.195.204.77",
|
||||
"67.195.204.79",
|
||||
"67.195.204.80",
|
||||
"67.195.204.82",
|
||||
"67.195.228.106",
|
||||
"67.195.228.109",
|
||||
"67.195.228.110",
|
||||
"67.195.228.111",
|
||||
"67.195.228.74",
|
||||
"67.195.228.84",
|
||||
"67.195.228.86",
|
||||
"67.195.228.94",
|
||||
"68.87.20.5",
|
||||
"72.51.60.130",
|
||||
"72.51.60.131",
|
||||
"72.51.60.132",
|
||||
"72.51.60.133",
|
||||
"72.51.60.134",
|
||||
"74.125.200.26",
|
||||
"74.125.200.27",
|
||||
"74.208.5.20",
|
||||
"74.208.5.22",
|
||||
"77.75.76.32",
|
||||
"77.75.76.42",
|
||||
"77.75.78.32",
|
||||
"77.75.78.42",
|
||||
"77.88.21.249",
|
||||
"77.93.251.249",
|
||||
"80.12.242.9",
|
||||
"81.19.78.64",
|
||||
"81.19.78.65",
|
||||
"81.19.78.66",
|
||||
"81.19.78.67",
|
||||
"81.3.6.162",
|
||||
"81.3.6.165",
|
||||
"82.57.200.133",
|
||||
"93.17.128.123",
|
||||
"93.17.128.165",
|
||||
"94.100.180.104",
|
||||
"94.100.180.31",
|
||||
"94.242.232.66",
|
||||
"96.114.157.80",
|
||||
"98.136.96.74",
|
||||
"98.136.96.75",
|
||||
"98.136.96.76",
|
||||
"98.136.96.77",
|
||||
"98.136.96.91",
|
||||
"98.136.96.92",
|
||||
"98.136.96.93"
|
||||
],
|
||||
"matching_attributes": [
|
||||
"ip-src",
|
||||
"ip-dst",
|
||||
"domain|ip"
|
||||
],
|
||||
"name": "List of known SMTP addresses IP ranges",
|
||||
"type": "cidr",
|
||||
"version": 20210614
|
||||
}
|
File diff suppressed because it is too large
Load Diff
|
@ -7,9 +7,9 @@ from OpenSSL.crypto import FILETYPE_PEM, load_certificate, X509
|
|||
from pyasn1.codec.der.decoder import decode as asn1_decoder
|
||||
from pyasn1_modules.rfc2459 import CRLDistPointsSyntax, AuthorityInfoAccessSyntax
|
||||
from typing import List, Set
|
||||
from dns.resolver import Resolver, NoAnswer, NXDOMAIN
|
||||
from dns.resolver import NoAnswer, NXDOMAIN
|
||||
from dns.exception import Timeout
|
||||
from generator import download_to_file, get_version, write_to_file, get_abspath_source_file
|
||||
from generator import download_to_file, get_version, write_to_file, get_abspath_source_file, create_resolver
|
||||
|
||||
|
||||
def get_domain(url: str) -> str:
|
||||
|
@ -44,9 +44,7 @@ def get_crl_ocsp_domains(cert: X509) -> List[str]:
|
|||
|
||||
|
||||
def get_ips_from_domain(domain: str) -> Set[str]:
|
||||
resolver = Resolver()
|
||||
resolver.timeout = 5
|
||||
resolver.lifetime = 5
|
||||
resolver = create_resolver()
|
||||
|
||||
ips = set()
|
||||
|
||||
|
@ -65,10 +63,6 @@ def get_ips_from_domain(domain: str) -> Set[str]:
|
|||
|
||||
|
||||
def get_ips_from_domains(domains) -> Set[str]:
|
||||
resolver = Resolver()
|
||||
resolver.timeout = 5
|
||||
resolver.lifetime = 5
|
||||
|
||||
p = multiprocessing.dummy.Pool(10)
|
||||
ips = set()
|
||||
for ips_for_domain in p.map(get_ips_from_domain, domains):
|
||||
|
|
|
@ -1,48 +1,17 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
from ipaddress import ip_network, IPv4Network, IPv6Network
|
||||
from dns.resolver import Resolver
|
||||
from typing import List, Union
|
||||
from generator import get_version, write_to_file
|
||||
|
||||
|
||||
class Spf:
|
||||
def _parse_spf(self, spf: str) -> dict:
|
||||
output = {"include": [], "ranges": []}
|
||||
for part in spf.split(" "):
|
||||
if part.startswith("include:"):
|
||||
output["include"].append(part.split(":", 1)[1])
|
||||
elif part.startswith("ip4:") or part.startswith("ip6:"):
|
||||
output["ranges"].append(ip_network(part.split(":", 1)[1]))
|
||||
return output
|
||||
|
||||
def _query_spf(self, resolver: Resolver, domain: str) -> List[Union[IPv4Network, IPv6Network]]:
|
||||
ranges = []
|
||||
for rdata in resolver.query(domain, "TXT"):
|
||||
parsed = self._parse_spf(rdata.to_text())
|
||||
ranges += parsed["ranges"]
|
||||
|
||||
for include in parsed["include"]:
|
||||
ranges += self._query_spf(resolver, include)
|
||||
|
||||
return ranges
|
||||
|
||||
def get_list(self, domain: str) -> List[Union[IPv4Network, IPv6Network]]:
|
||||
resolver = Resolver()
|
||||
return self._query_spf(resolver, domain)
|
||||
from generator import get_version, write_to_file, Dns, consolidate_networks, create_resolver
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
spf = Spf()
|
||||
print()
|
||||
|
||||
spf = Dns(create_resolver())
|
||||
warninglist = {
|
||||
'name': "List of known Gmail sending IP ranges",
|
||||
'version': get_version(),
|
||||
'description': "List of known Gmail sending IP ranges (https://support.google.com/a/answer/27642?hl=en)",
|
||||
'matching_attributes': ["ip-src", "ip-dst", "domain|ip"],
|
||||
'type': 'cidr',
|
||||
'list': [str(range) for range in spf.get_list("_spf.google.com")],
|
||||
'list': consolidate_networks(spf.get_ip_ranges_from_spf("gmail.com")),
|
||||
}
|
||||
|
||||
write_to_file(warninglist, "google-gmail-sending-ips")
|
||||
|
|
|
@ -0,0 +1,100 @@
|
|||
#!/usr/bin/env python3
|
||||
import multiprocessing.dummy
|
||||
from generator import get_version, write_to_file, Dns, consolidate_networks, create_resolver
|
||||
|
||||
# Source: https://github.com/mailcheck/mailcheck/wiki/List-of-Popular-Domains
|
||||
domains = [
|
||||
# Default domains included
|
||||
"aol.com", "att.net", "comcast.net", "facebook.com", "gmail.com", "gmx.com", "googlemail.com",
|
||||
"google.com", "hotmail.com", "hotmail.co.uk", "mac.com", "me.com", "mail.com", "msn.com",
|
||||
"live.com", "sbcglobal.net", "verizon.net", "yahoo.com", "yahoo.co.uk",
|
||||
|
||||
# Other global domains
|
||||
"email.com", "fastmail.fm", "games.com", "gmx.net", "hush.com", "hushmail.com", "icloud.com",
|
||||
"iname.com", "inbox.com", "lavabit.com",
|
||||
"love.com", "outlook.com", "pobox.com", "protonmail.ch", "protonmail.com", "tutanota.de", "tutanota.com",
|
||||
"tutamail.com", "tuta.io",
|
||||
"keemail.me", "rocketmail.com", "safe-mail.net", "wow.com", "ygm.com",
|
||||
"ymail.com", "zoho.com", "yandex.com",
|
||||
|
||||
# United States ISP domains
|
||||
"bellsouth.net", "charter.net", "cox.net", "earthlink.net", "juno.com",
|
||||
|
||||
# British ISP domains
|
||||
"btinternet.com", "virginmedia.com", "blueyonder.co.uk", "live.co.uk",
|
||||
"ntlworld.com", "orange.net", "sky.com", "talktalk.co.uk", "tiscali.co.uk",
|
||||
"virgin.net", "bt.com",
|
||||
|
||||
# Domains used in Asia
|
||||
"sina.com", "sina.cn", "qq.com", "naver.com", "hanmail.net", "daum.net", "nate.com", "yahoo.co.jp", "yahoo.co.kr",
|
||||
"yahoo.co.id", "yahoo.co.in", "yahoo.com.sg", "yahoo.com.ph", "163.com", "yeah.net", "126.com", "21cn.com",
|
||||
"aliyun.com", "foxmail.com",
|
||||
|
||||
# French ISP domains
|
||||
"hotmail.fr", "live.fr", "laposte.net", "yahoo.fr", "wanadoo.fr", "orange.fr", "gmx.fr", "sfr.fr", "neuf.fr",
|
||||
"free.fr",
|
||||
|
||||
# German ISP domains
|
||||
"gmx.de", "hotmail.de", "live.de", "online.de", "t-online.de", "web.de", "yahoo.de",
|
||||
|
||||
# Italian ISP domains
|
||||
"libero.it", "virgilio.it", "hotmail.it", "aol.it", "tiscali.it",
|
||||
"alice.it", "live.it", "yahoo.it", "email.it", "tin.it", "poste.it", "teletu.it",
|
||||
|
||||
# Russian ISP domains
|
||||
"bk.ru", "inbox.ru", "list.ru", "mail.ru", "rambler.ru", "yandex.by", "yandex.com", "yandex.kz", "yandex.ru",
|
||||
"yandex.ua", "ya.ru",
|
||||
|
||||
# Belgian ISP domains
|
||||
"hotmail.be", "live.be", "skynet.be", "voo.be", "tvcablenet.be", "telenet.be",
|
||||
|
||||
# Argentinian ISP domains
|
||||
"hotmail.com.ar", "live.com.ar", "yahoo.com.ar", "fibertel.com.ar", "speedy.com.ar", "arnet.com.ar",
|
||||
|
||||
# Domains used in Mexico
|
||||
"yahoo.com.mx", "live.com.mx", "hotmail.es", "hotmail.com.mx", "prodigy.net.mx",
|
||||
|
||||
# Domains used in Canada
|
||||
"yahoo.ca", "hotmail.ca", "bell.net", "shaw.ca", "sympatico.ca", "rogers.com",
|
||||
|
||||
# Domains used in Brazil
|
||||
"yahoo.com.br", "hotmail.com.br", "outlook.com.br", "uol.com.br", "bol.com.br", "terra.com.br", "ig.com.br",
|
||||
"r7.com", "zipmail.com.br", "globo.com", "globomail.com", "oi.com.br",
|
||||
|
||||
# Custom extension
|
||||
# Domains used in Czechia
|
||||
"seznam.cz", "atlas.cz", "centrum.cz",
|
||||
]
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
dns = Dns(create_resolver())
|
||||
|
||||
spf_ranges = []
|
||||
p = multiprocessing.dummy.Pool(40)
|
||||
for domain_ranges in p.map(lambda d: dns.get_ip_ranges_from_spf(d), domains):
|
||||
spf_ranges.extend(domain_ranges)
|
||||
|
||||
warninglist = {
|
||||
'name': "List of known SMTP sending IP ranges",
|
||||
'version': get_version(),
|
||||
'description': "List of IP ranges for known SMTP servers.",
|
||||
'matching_attributes': ["ip-src", "ip-dst", "domain|ip"],
|
||||
'type': 'cidr',
|
||||
'list': consolidate_networks(spf_ranges),
|
||||
}
|
||||
write_to_file(warninglist, "smtp-sending-ips")
|
||||
|
||||
mx_ips = []
|
||||
for domain_ranges in p.map(lambda d: dns.get_mx_ips_for_domain(d), domains):
|
||||
mx_ips.extend(domain_ranges)
|
||||
|
||||
warninglist = {
|
||||
'name': "List of known SMTP receiving IP addresses",
|
||||
'version': get_version(),
|
||||
'description': "List of IP addresses for known SMTP servers.",
|
||||
'matching_attributes': ["ip-src", "ip-dst", "domain|ip"],
|
||||
'type': 'cidr',
|
||||
'list': map(str, mx_ips),
|
||||
}
|
||||
write_to_file(warninglist, "smtp-receiving-ips")
|
|
@ -1,14 +1,14 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import datetime
|
||||
import ipaddress
|
||||
import json
|
||||
import logging
|
||||
from inspect import currentframe, getframeinfo, getmodulename, stack
|
||||
from os import mkdir, path
|
||||
from typing import List, Union
|
||||
|
||||
import requests
|
||||
import dns.exception
|
||||
import dns.resolver
|
||||
from dateutil.parser import parse as parsedate
|
||||
|
||||
|
||||
|
@ -137,23 +137,111 @@ def write_to_file(warninglist, dst):
|
|||
|
||||
|
||||
def consolidate_networks(networks):
|
||||
# Convert to IpNetwork
|
||||
# Split to IPv4 and IPv6 ranges
|
||||
ipv4_networks = []
|
||||
ipv6_networks = []
|
||||
for network in networks:
|
||||
network = ipaddress.ip_network(network)
|
||||
if isinstance(network, str):
|
||||
# Convert string to IpNetwork
|
||||
network = ipaddress.ip_network(network)
|
||||
|
||||
if network.version == 4:
|
||||
ipv4_networks.append(network)
|
||||
else:
|
||||
ipv6_networks.append(network)
|
||||
|
||||
# Collapse
|
||||
# Collapse ranges
|
||||
networks_to_keep = list(map(str, ipaddress.collapse_addresses(ipv4_networks)))
|
||||
networks_to_keep.extend(map(str, ipaddress.collapse_addresses(ipv6_networks)))
|
||||
|
||||
return networks_to_keep
|
||||
|
||||
|
||||
def create_resolver() -> dns.resolver.Resolver:
|
||||
resolver = dns.resolver.Resolver(configure=False)
|
||||
resolver.timeout = 30
|
||||
resolver.lifetime = 30
|
||||
resolver.cache = dns.resolver.LRUCache()
|
||||
resolver.nameservers = ["193.17.47.1", "185.43.135.1"] # CZ.NIC nameservers
|
||||
return resolver
|
||||
|
||||
|
||||
class Dns:
|
||||
def __init__(self, resolver: dns.resolver.Resolver):
|
||||
self.__resolver = resolver
|
||||
|
||||
def _parse_spf(self, domain: str, spf: str) -> dict:
|
||||
output = {"include": [], "ranges": [], "a": [], "mx": []}
|
||||
for part in spf.split(" "):
|
||||
if part.startswith("include:"):
|
||||
output["include"].append(part.split(":", 1)[1])
|
||||
elif part.startswith("redirect="):
|
||||
output["include"].append(part.split("=", 1)[1])
|
||||
elif part == "a":
|
||||
output["a"].append(domain)
|
||||
elif part.startswith("a:"):
|
||||
output["a"].append(part.split(":", 1)[1])
|
||||
elif part == "mx":
|
||||
output["mx"].append(domain)
|
||||
elif part.startswith("mx:"):
|
||||
output["mx"].append(part.split(":", 1)[1])
|
||||
elif part.startswith("ip4:") or part.startswith("ip6:"):
|
||||
output["ranges"].append(ipaddress.ip_network(part.split(":", 1)[1], strict=False))
|
||||
return output
|
||||
|
||||
def get_ip_for_domain(self, domain: str) -> List[Union[ipaddress.IPv4Address, ipaddress.IPv6Address]]:
|
||||
ranges = []
|
||||
try:
|
||||
for ip in self.__resolver.query(domain, "a"):
|
||||
ranges.append(ipaddress.IPv4Address(str(ip)))
|
||||
except (dns.resolver.NoAnswer, dns.resolver.NXDOMAIN, dns.exception.Timeout, dns.resolver.NoNameservers):
|
||||
pass
|
||||
|
||||
try:
|
||||
for ip in self.__resolver.query(domain, "aaaa"):
|
||||
ranges.append(ipaddress.IPv6Address(str(ip)))
|
||||
except (dns.resolver.NoAnswer, dns.resolver.NXDOMAIN, dns.exception.Timeout, dns.resolver.NoNameservers):
|
||||
pass
|
||||
|
||||
return ranges
|
||||
|
||||
def get_mx_ips_for_domain(self, domain: str) -> List[Union[ipaddress.IPv4Address, ipaddress.IPv6Address]]:
|
||||
ranges = []
|
||||
try:
|
||||
for rdata in self.__resolver.query(domain, "mx"):
|
||||
ranges += self.get_ip_for_domain(rdata.exchange)
|
||||
except (dns.resolver.NoAnswer, dns.resolver.NXDOMAIN, dns.exception.Timeout, dns.resolver.NoNameservers):
|
||||
pass
|
||||
return ranges
|
||||
|
||||
def get_ip_ranges_from_spf(self, domain: str) -> List[Union[ipaddress.IPv4Network, ipaddress.IPv6Network]]:
|
||||
try:
|
||||
txt_records = self.__resolver.query(domain, "TXT")
|
||||
except (dns.resolver.NoAnswer, dns.resolver.NXDOMAIN, dns.exception.Timeout, dns.resolver.NoNameservers) as e:
|
||||
logging.info("Could not fetch TXT record for domain {}: {}".format(domain, str(e)))
|
||||
return []
|
||||
|
||||
ranges = []
|
||||
for rdata in txt_records:
|
||||
record = "".join([s.decode("utf-8") for s in rdata.strings])
|
||||
if not record.startswith("v=spf1"):
|
||||
continue
|
||||
|
||||
parsed = self._parse_spf(domain, record)
|
||||
ranges += parsed["ranges"]
|
||||
|
||||
for include in parsed["include"]:
|
||||
ranges += self.get_ip_ranges_from_spf(include)
|
||||
|
||||
for domain in parsed["a"]:
|
||||
ranges += map(ipaddress.ip_network, self.get_ip_for_domain(domain))
|
||||
|
||||
for mx in parsed["mx"]:
|
||||
ranges += map(ipaddress.ip_network, self.get_mx_ips_for_domain(mx))
|
||||
|
||||
return ranges
|
||||
|
||||
|
||||
def main():
|
||||
init_logging()
|
||||
|
||||
|
|
Loading…
Reference in New Issue