diff --git a/bgpranking/asn_descriptions.py b/bgpranking/asn_descriptions.py index 7518a5f..19637d6 100644 --- a/bgpranking/asn_descriptions.py +++ b/bgpranking/asn_descriptions.py @@ -21,7 +21,7 @@ class ASNDescriptions(): safe_create_dir(self.directory) self.archives = self.directory / 'archive' safe_create_dir(self.archives) - self.url = 'http://www.cidr-report.org/as2.0/autnums.html' + self.url = 'https://www.cidr-report.org/as2.0/autnums.html' def __init_logger(self, loglevel): self.logger = logging.getLogger(f'{self.__class__.__name__}') diff --git a/bgpranking/config/bgpranking.json b/bgpranking/config/bgpranking.json index 83f1d39..ea5e28e 100644 --- a/bgpranking/config/bgpranking.json +++ b/bgpranking/config/bgpranking.json @@ -1,3 +1,3 @@ { - "ipasnhistory_url": "http://127.0.0.1:5176/" + "ipasnhistory_url": "https://ipasnhistory.circl.lu/" } diff --git a/bgpranking/dbinsert.py b/bgpranking/dbinsert.py index 3410f9c..2541fce 100644 --- a/bgpranking/dbinsert.py +++ b/bgpranking/dbinsert.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- import logging +import time from redis import StrictRedis from .libs.helpers import shutdown_requested, set_running, unset_running, get_socket_path, get_ipasn, sanity_check_ipasn @@ -29,8 +30,15 @@ class DatabaseInsert(): set_running(self.__class__.__name__) while True: - if shutdown_requested() or not self.ipasn.is_up: + if shutdown_requested(): break + try: + if not self.ipasn.is_up: + break + except Exception: + self.logger.warning('Unable to query ipasnhistory') + time.sleep(10) + continue uuids = self.redis_sanitized.spop('to_insert', 100) if not uuids: break @@ -52,7 +60,8 @@ class DatabaseInsert(): self.logger.exception('Mass query in IPASN History failed, trying again later.') # Rollback the spop self.redis_sanitized.sadd('to_insert', *uuids) - break + time.sleep(10) + continue retry = [] done = [] ardb_pipeline = self.ardb_storage.pipeline(transaction=False) diff --git a/bgpranking/sanitizer.py b/bgpranking/sanitizer.py index fe1d198..f4654f7 100644 --- a/bgpranking/sanitizer.py +++ b/bgpranking/sanitizer.py @@ -51,6 +51,10 @@ class Sanitizer(): except ValueError: self.logger.info(f"Invalid IP address: {data['ip']}") continue + except KeyError: + self.logger.info(f"Invalid entry {data}") + continue + if not ip.is_global: self.logger.info(f"The IP address {data['ip']} is not global") continue diff --git a/website/web/__init__.py b/website/web/__init__.py index c9f3b0a..a57c4b3 100644 --- a/website/web/__init__.py +++ b/website/web/__init__.py @@ -188,8 +188,13 @@ def ipasn_history_proxy(path): config, general_config_file = load_general_config() if 'ipasnhistory_url' not in config: raise MissingConfigEntry(f'"ipasnhistory_url" is missing in {general_config_file}.') - proxied_url = urljoin(config['ipasnhistory_url'], - request.full_path.replace('/ipasn_history', '')) + + path_for_ipasnhistory = request.full_path.replace('/ipasn_history', '') + if '/?' in path_for_ipasnhistory: + path_for_ipasnhistory = path_for_ipasnhistory.replace('/?', '/ip?') + print(path_for_ipasnhistory) + proxied_url = urljoin(config['ipasnhistory_url'], path_for_ipasnhistory) + print(proxied_url) if request.method in ['GET', 'HEAD']: to_return = requests.get(proxied_url).json() elif request.method == 'POST':