From bcbb51d50ed33dd5aa2b185146dd531e900da182 Mon Sep 17 00:00:00 2001 From: Hendrik Baecker Date: Wed, 5 Apr 2023 14:33:08 +0200 Subject: [PATCH] Added google bot updater + list --- generate_all.sh | 1 + lists/googlebot/list.json | 102 +++++++++++++++++++++++------------ tools/generate-google-bot.py | 23 ++++++++ 3 files changed, 93 insertions(+), 33 deletions(-) create mode 100644 tools/generate-google-bot.py diff --git a/generate_all.sh b/generate_all.sh index b87890c..99c3100 100755 --- a/generate_all.sh +++ b/generate_all.sh @@ -29,6 +29,7 @@ python3 generate-vpn.py python3 generate-wikimedia.py python3 generate-second-level-tlds.py python3 generate-google-gcp.py +python3 generate-google-bot.py python3 generate-google-gmail-sending-ips.py python3 generate-smtp.py python3 generate-tenable.py diff --git a/lists/googlebot/list.json b/lists/googlebot/list.json index 64f2d1a..8e265ba 100644 --- a/lists/googlebot/list.json +++ b/lists/googlebot/list.json @@ -1,42 +1,78 @@ { - "description": "List of known Googlebot IP ranges (https://www.lifewire.com/what-is-the-ip-address-of-google-818153 )", + "description": "Google Bot IP address ranges (https://developers.google.com/search/apis/ipranges/googlebot.json)", "list": [ - "216.239.33.128/25", - "216.239.33.96/27", - "216.239.34.0/23", - "216.239.36.0/22", - "216.239.40.0/21", - "216.239.48.0/21", - "216.239.56.0/23", - "216.239.58.0/24", - "216.239.59.0/25", - "216.239.59.128/32", - "64.233.173.193/32", - "64.233.173.194/31", - "64.233.173.196/30", - "64.233.173.200/29", - "64.233.173.208/28", - "64.233.173.224/27", - "64.68.90.0/24", - "66.249.64.1/32", - "66.249.64.128/25", - "66.249.64.16/28", - "66.249.64.2/31", - "66.249.64.32/27", - "66.249.64.4/30", - "66.249.64.64/26", - "66.249.64.8/29", - "66.249.65.0/24", - "66.249.66.0/23", - "66.249.68.0/22", - "66.249.72.0/21" + "2001:4860:4801:10::/61", + "2001:4860:4801:18::/62", + "2001:4860:4801:20::/60", + "2001:4860:4801:2::/63", + "2001:4860:4801:30::/61", + "2001:4860:4801:38::/62", + "2001:4860:4801:3c::/63", + "2001:4860:4801:3e::/64", + "2001:4860:4801:40::/61", + "2001:4860:4801:48::/63", + "2001:4860:4801:4a::/64", + "2001:4860:4801:50::/63", + "2001:4860:4801:53::/64", + "2001:4860:4801:60::/60", + "2001:4860:4801:70::/61", + "2001:4860:4801:80::/62", + "2001:4860:4801:84::/63", + "2001:4860:4801:86::/64", + "2001:4860:4801:90::/62", + "2001:4860:4801::/64", + "2001:4860:4801:c::/64", + "2001:4860:4801:f::/64", + "34.100.182.96/28", + "34.101.50.144/28", + "34.118.254.0/28", + "34.118.66.0/28", + "34.126.178.96/28", + "34.146.150.144/28", + "34.147.110.144/28", + "34.151.74.144/28", + "34.152.50.64/28", + "34.154.114.144/28", + "34.155.98.32/28", + "34.165.18.176/28", + "34.175.160.64/28", + "34.176.130.16/28", + "34.22.85.0/27", + "34.64.82.64/28", + "34.65.242.112/28", + "34.80.50.80/28", + "34.88.194.0/28", + "34.89.10.80/28", + "34.89.198.80/28", + "34.96.162.48/28", + "35.247.243.240/28", + "66.249.64.0/23", + "66.249.66.0/26", + "66.249.66.128/27", + "66.249.66.192/27", + "66.249.66.64/27", + "66.249.68.0/26", + "66.249.68.64/27", + "66.249.69.0/24", + "66.249.70.0/24", + "66.249.71.0/25", + "66.249.71.128/26", + "66.249.71.192/27", + "66.249.72.0/23", + "66.249.74.0/25", + "66.249.74.128/27", + "66.249.75.0/24", + "66.249.76.0/24", + "66.249.77.0/25", + "66.249.77.128/27", + "66.249.79.0/24" ], "matching_attributes": [ - "ip-dst", "ip-src", + "ip-dst", "domain|ip" ], - "name": "List of known Googlebot IP ranges", + "name": "List of known Googlebot IP ranges (https://developers.google.com/search/apis/ipranges/googlebot.json)", "type": "cidr", - "version": 20190724 + "version": 20230405 } diff --git a/tools/generate-google-bot.py b/tools/generate-google-bot.py new file mode 100644 index 0000000..72fcb0a --- /dev/null +++ b/tools/generate-google-bot.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import json +from generator import download, get_version, write_to_file, consolidate_networks + + +if __name__ == '__main__': + bots = download("https://developers.google.com/search/apis/ipranges/googlebot.json") + parsed = json.loads(bots.text) + + ranges = [p["ipv4Prefix"] if "ipv4Prefix" in p else p["ipv6Prefix"] for p in parsed["prefixes"]] + + warninglist = { + 'name': 'List of known Googlebot IP ranges (https://developers.google.com/search/apis/ipranges/googlebot.json)', + 'version': get_version(), + 'description': "Google Bot IP address ranges (https://developers.google.com/search/apis/ipranges/googlebot.json)", + 'matching_attributes': ["ip-src", "ip-dst", "domain|ip"], + 'type': 'cidr', + 'list': consolidate_networks(ranges), + } + + write_to_file(warninglist, "googlebot")